diff --git a/.github/scripts/command/load_dump_bench.sh b/.github/scripts/command/load_dump_bench.sh index 7bf5e4df0fdc..5abd77011acf 100755 --- a/.github/scripts/command/load_dump_bench.sh +++ b/.github/scripts/command/load_dump_bench.sh @@ -4,7 +4,6 @@ source .github/scripts/common/common.sh [[ -z "$META" ]] && META=sqlite3 [[ -z "$START_META" ]] && START_META=true -[[ -z "$BIGDIR" ]] && BIGDIR=true source .github/scripts/start_meta_engine.sh META_URL=$(get_meta_url $META) META_URL2=$(get_meta_url2 $META) @@ -51,6 +50,7 @@ do_dump_load(){ else ./juicefs load $META_URL2 $dump_file fi + ./juicefs mount $META_URL2 /tmp/jfs2 -d df -i /tmp/jfs /tmp/jfs2 iused1=$(df -i /tmp/jfs | tail -1 | awk '{print $3}') @@ -62,14 +62,13 @@ do_dump_load(){ summary2=$(./juicefs summary /tmp/jfs2/ --csv | head -n +2 | tail -n 1) [[ "$summary1" == "$summary2" ]] || (echo ": summary error: $summary1 $summary2" && exit 1) - if [[ "$BIGDIR" == "true" ]]; then - file_count=$(ls -l /tmp/jfs2/bigdir/test-dir.0-0/mdtest_tree.0/ | wc -l) - file_count=$((file_count-1)) - if [[ "$file_count" -ne "$FILE_COUNT_IN_BIGDIR" ]]; then - echo ": file_count error: $file_count" - exit 1 - fi + file_count=$(ls -l /tmp/jfs2/bigdir/test-dir.0-0/mdtest_tree.0/ | wc -l) + file_count=$((file_count-1)) + if [[ "$file_count" -ne "$FILE_COUNT_IN_BIGDIR" ]]; then + echo ": file_count error: $file_count" + exit 1 fi + ./juicefs rmr /tmp/jfs2/smalldir ls /tmp/jfs2/smalldir && echo ": ls should fail" && exit 1 || true umount_jfs /tmp/jfs2 $META_URL2 diff --git a/.github/workflows/aitests.yml b/.github/workflows/aitests.yml deleted file mode 100644 index 46813b2e4733..000000000000 --- a/.github/workflows/aitests.yml +++ /dev/null @@ -1,142 +0,0 @@ -name: "aitests" - -on: -# push: -# branches: -# - 'release-*' -# paths-ignore: -# - 'docs/**' -# pull_request: -# #The branches below must be a subset of the branches above -# branches: -# - 'release-*' -# paths-ignore: -# - 'docs/**' - schedule: - - cron: '30 20 * * *' - workflow_dispatch: - - -jobs: - aitests: - timeout-minutes: 90 - strategy: - fail-fast: false - matrix: - type: [ 'normal', 'cache'] - - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Build - timeout-minutes: 10 - uses: ./.github/actions/build - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes - - - name: Setup minio - run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - sleep 5 - - - name: Juicefs Format - run: | - sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9000/mypics \ - --access-key minioadmin \ - --secret-key minioadmin \ - redis://127.0.0.1:6379/1 pics - - - name: Juicefs Mount - run: | - if [ "${{matrix.type}}" == "normal" ]; then - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --no-usage-report - elif [ "${{matrix.type}}" == "cache" ]; then - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --cache-size 3072 --no-usage-report - else - echo "matrix.type error: ${{matrix.type}} " - fi - - python --version - - - name: Set up Conda - uses: s-weigand/setup-conda@v1 - with: - python-version: 2.7 - - - name: Install Dependency - run: | - conda --version - python --version - conda install tensorflow=1.12.0 - sudo .github/scripts/apt_install.sh libhdf5-dev - pip install h5py - pip install pillow - - - name: Prepare Data - run: | - stat /jfs/ - cd /jfs/ - git clone https://github.com/sanwan/SVHNClassifier - cd SVHNClassifier - git checkout 1a8c6f1 - mkdir data - cd data - wget -O train.tar.gz -nv http://ufldl.stanford.edu/housenumbers/train.tar.gz - wget -O test.tar.gz -nv http://ufldl.stanford.edu/housenumbers/test.tar.gz - ls -lah - time tar -zxf train.tar.gz - time tar -zxf test.tar.gz - mkdir extra - mv test/1.png extra/ - cp test/digitStruct.mat extra/ - cp test/see_bboxes.m extra/ - rm train.tar.gz - rm test.tar.gz - ls -lah - du -sh - cd .. - time python convert_to_tfrecords.py --data_dir ./data >img.log - sudo chmod 777 /var/jfsCache/ - cd /var/jfsCache/ - df -lh - - - - name: Train - run: | - cd /jfs/SVHNClassifier/ - time python train.py --data_dir ./data --train_logdir ./logs/train - cd /var/jfsCache/ - df -lh - - - name: Evaluate - run: | - cd /jfs/SVHNClassifier/ - time python eval.py --data_dir ./data --checkpoint_dir ./logs/train --eval_logdir ./logs/eval - cd /var/jfsCache/ - df -lh - - - name: log - if: always() - run: | - tail -300 /var/log/juicefs.log - grep ":" /var/log/juicefs.log && exit 1 || true - - - name: Send Slack Notification - if: failure() - uses: juicedata/slack-notify-action@main - with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" - slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" - diff --git a/.github/workflows/elastictest.yml b/.github/workflows/elastictest.yml deleted file mode 100644 index 489f7564dbb0..000000000000 --- a/.github/workflows/elastictest.yml +++ /dev/null @@ -1,104 +0,0 @@ -name: "elastictest" - -on: -# push: -# branches: -# - 'release-**' -# paths-ignore: -# - 'docs/**' -# pull_request: -# #The branches below must be a subset of the branches above -# branches: -# - 'release-**' -# paths-ignore: -# - 'docs/**' - schedule: - - cron: '30 20 * * *' - - workflow_dispatch: - inputs: - debug: - type: boolean - description: "Run the build with tmate debugging enabled" - required: false - default: false - -jobs: - elastictest: - timeout-minutes: 60 - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Build - timeout-minutes: 10 - uses: ./.github/actions/build - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes - - - name: Setup minio - run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - sleep 5 - - - name: Juicefs Format - run: | - sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9000/mypics \ - --access-key minioadmin \ - --secret-key minioadmin \ - redis://127.0.0.1:6379/1 pics - - - name: Juicefs Mount - run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /data/jfs --no-usage-report - - - name: Install Elasticsearch - run: | - wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - - sudo sh -c 'echo "deb https://artifacts.elastic.co/packages/7.x/apt stable main" > /etc/apt/sources.list.d/elastic-7.x.list' - sudo .github/scripts/apt_install.sh apt-transport-https elasticsearch - - - name: Change Datadir - run: | - sudo chmod 777 /data/jfs/ - sudo service elasticsearch stop - sudo sed -i "s?^path\.data.*?path\.data: /data/jfs/?" /etc/elasticsearch/elasticsearch.yml - sudo service elasticsearch restart - curl -X GET "localhost:9200/" - - - name: Esrally Test - run: | - sudo pip3 install esrally - sudo .github/scripts/apt_install.sh pbzip2 - esrally race --track=geopoint --target-hosts=127.0.0.1:9200 --pipeline=benchmark-only --report-file=/tmp/report.md - cat /tmp/report.md - - - name: Log - if: failure() - run: | - echo "juicefs log" - sudo tail -n 1000 /var/log/juicefs.log - grep ":" /var/log/juicefs.log && exit 1 - - - name: Send Slack Notification - if: failure() - uses: juicedata/slack-notify-action@main - with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" - slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" - - - name: Setup upterm session - if: failure() && github.event_name == 'workflow_dispatch' && github.event.inputs.debug - timeout-minutes: 60 - uses: lhotari/action-upterm@v1 \ No newline at end of file diff --git a/.github/workflows/filebench.yml b/.github/workflows/filebench.yml deleted file mode 100644 index 70916e107872..000000000000 --- a/.github/workflows/filebench.yml +++ /dev/null @@ -1,105 +0,0 @@ -name: "filebench" - -on: - push: - branches: - - 'release-**' - paths-ignore: - - 'docs/**' - - '**.md' - schedule: - - cron: '30 20 * * *' - workflow_dispatch: - - -jobs: - filebench: - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - workload: [ 'varmail', 'webserver', 'videoserver'] - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Build - timeout-minutes: 10 - uses: ./.github/actions/build - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes - - - name: Setup minio - run: | - docker run -d -p 9000:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - sleep 5 - - - name: Juicefs Format - run: | - sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9000/mypics \ - --access-key minioadmin \ - --secret-key minioadmin \ - redis://127.0.0.1:6379/1 pics - - - name: Juicefs Mount - run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /data/jfs --no-usage-report - - - name: Install Dependency - run: | - wget -O filebench.tar.gz https://github.com/filebench/filebench/releases/download/1.5-alpha3/filebench-1.5-alpha3.tar.gz - tar -zxvf filebench.tar.gz - cd filebench-1.5-alpha3 - ./configure - make - sudo make install - filebench -h - echo 0 >randomize_va_space - sudo cp randomize_va_space /proc/sys/kernel/ - echo "randomize_va_space" - cat /proc/sys/kernel/randomize_va_space - - - name: Run Workload - run: | - cd /usr/local/share/filebench/workloads/ - sudo chmod 777 * - if [ ${{matrix.workload}} == "varmail" ];then - echo "run varmail workload" - sudo sed -i "s?/tmp?/data/jfs?" varmail.f - sudo sed -i "s?run 60?run 1800?" varmail.f - cat varmail.f - sudo filebench -f varmail.f - elif [ ${{matrix.workload}} == "webserver" ];then - echo "run webserver workload" - sudo sed -i "s?/tmp?/data/jfs?" webserver.f - sudo sed -i "s?run 60?run 1800?" webserver.f - cat webserver.f - sudo filebench -f webserver.f - elif [ ${{matrix.workload}} == "videoserver" ];then - echo "run videoserver worload" - sudo sed -i "s?/tmp?/data/jfs?" videoserver.f - sudo sed -i "s?$filesize=10g?$filesize=100m?" videoserver.f - sudo sed -i "s?$numpassivevids=194?$numpassivevids=100?" videoserver.f - sudo echo "run 1800" >>videoserver.f - cat videoserver.f - sudo filebench -f videoserver.f - fi - - - name: Log - if: always() - run: | - df -lh - echo "juicefs log" - sudo tail -n 1000 /var/log/juicefs.log - grep ":" /var/log/juicefs.log && exit 1 || true diff --git a/.github/workflows/mongodb_mmap.yml b/.github/workflows/mongodb_mmap.yml deleted file mode 100644 index ed1a5ee05cec..000000000000 --- a/.github/workflows/mongodb_mmap.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: "mongodb_mmap" - -on: - push: - branches: - - 'release-**' - paths-ignore: - - 'docs/**' - pull_request: - #The branches below must be a subset of the branches above - branches: - - 'release-**' - paths-ignore: - - 'docs/**' - schedule: - - cron: '30 20 * * *' - workflow_dispatch: - inputs: - debug: - type: boolean - description: "Run the build with tmate debugging enabled" - required: false - default: false - - -jobs: - mongodb_mmap: - timeout-minutes: 60 - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Build - timeout-minutes: 10 - uses: ./.github/actions/build - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes - - - name: Juicefs Format - run: | - sudo ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 pics - - - name: Juicefs Mount - run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /jfs/ --enable-xattr --cache-size 3072 --no-usage-report - - - name: Set up Mongodb - run: | - cd /jfs/ - git clone https://github.com/sanwan/actionsoftware.git - cd actionsoftware - tar zxvf mongodb-linux-x86_64-3.0.0.tgz - cd mongodb-linux-x86_64-3.0.0 - mkdir /jfs/mongodb/ - nohup ./bin/mongod --dbpath /jfs/mongodb & - - - name: Insert Data - run: | - cd /jfs/ - wget -O ycsb.tar.gz -q https://github.com/brianfrankcooper/YCSB/releases/download/0.17.0/ycsb-0.17.0.tar.gz - tar -zxvf ycsb.tar.gz - cd ycsb-0.17.0 - sed -i "s?recordcount=1000?recordcount=1000000?" workloads/workloadf - sed -i "s?operationcount=1000?operationcount=15000?" workloads/workloadf - cat workloads/workloadf - wget https://raw.githubusercontent.com/adonis0147/YCSB/master/bin/ycsb -O bin/ycsb - time ./bin/ycsb load mongodb -s -P workloads/workloadf -threads 10 > outputLoad.txt - - - name: Read Modify and Write - run: | - cd /jfs/ycsb-0.17.0/ - ps -aux | grep mongo - echo "run read modify write" - time ./bin/ycsb run mongodb -s -P workloads/workloadf -threads 10 > outputRun.txt - - - name: Log - if: always() - run: | - echo "juicefs log" - sudo tail -n 1000 /var/log/juicefs.log - grep ":" /var/log/juicefs.log && exit 1 || true - - - name: Send Slack Notification - if: failure() - uses: juicedata/slack-notify-action@main - with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" - slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" - - - name: Setup upterm session - if: failure() && (github.event.inputs.debug == 'true' || github.run_attempt != 1) - # if: failure() - timeout-minutes: 60 - uses: lhotari/action-upterm@v1 \ No newline at end of file diff --git a/.github/workflows/mysqltest_bigtable.yml b/.github/workflows/mysqltest_bigtable.yml deleted file mode 100644 index 010d91723cc5..000000000000 --- a/.github/workflows/mysqltest_bigtable.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: "mysqltest" - -on: -# push: -# branches: -# - 'release-**' -# paths-ignore: -# - 'docs/**' -# pull_request: -# #The branches below must be a subset of the branches above -# branches: -# - 'release-**' -# paths-ignore: -# - 'docs/**' - schedule: - - cron: '30 20 * * *' - workflow_dispatch: - -jobs: - mysqltest: - timeout-minutes: 60 - strategy: - fail-fast: false - matrix: - size: [ 'big', 'small' ] - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Build - timeout-minutes: 10 - uses: ./.github/actions/build - - - name: Run Redis - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes - - - name: Juicefs Format - run: | - sudo ./juicefs format --trash-days 0 redis://127.0.0.1:6379/1 pics - - - name: Juicefs Mount - run: | - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /tmp/jfs --no-usage-report - - - - name: Change Data Dir - run: | - sudo service mysql stop - sudo sed -i "s?.*datadir.*?datadir=/tmp/jfs/mysql?" /etc/mysql/mysql.conf.d/mysqld.cnf - sudo cat /etc/mysql/mysql.conf.d/mysqld.cnf - sudo mkdir /tmp/jfs/mysql - sudo chmod 777 /var/lib/mysql/ - sudo cp -a -r /var/lib/mysql/* /tmp/jfs/mysql/ - sudo chmod 777 /tmp/jfs/mysql/ - sudo chown -R mysql:mysql /tmp/jfs/mysql/ - sudo chmod 777 /etc/apparmor.d/tunables/alias - sudo echo "alias /var/lib/mysql/ -> /tmp/jfs/mysql/," >>/etc/apparmor.d/tunables/alias - sudo service apparmor restart - sudo service mysql start - sudo service mysql status - sudo mysql -uroot -proot -e "select @@datadir;" - - - name: Change Parameters - run: | - mysql -uroot -proot -e "show variables like 'log_error';" - mysql -uroot -proot -e "set global log_error_verbosity=3;" - mysql -uroot -proot -e "show variables like 'log_error_verbosity';" - mysql -uroot -proot -e "set global max_prepared_stmt_count=500000;" - - - name: Install Sysbench - run: | - curl -s https://packagecloud.io/install/repositories/akopytov/sysbench/script.deb.sh | sudo bash - sudo .github/scripts/apt_install.sh sysbench - sudo mysql -uroot -proot -e "create database test;" - echo "mysql-host=127.0.0.1" >>/tmp/config.cfg - echo "mysql-port=3306" >>/tmp/config.cfg - echo "mysql-user=root" >>/tmp/config.cfg - echo "mysql-password=root" >>/tmp/config.cfg - echo "mysql-db=test" >>/tmp/config.cfg - echo "report-interval=10" >>/tmp/config.cfg - echo "db-driver=mysql" >>/tmp/config.cfg - - - name: Big Size Table Test - run: | - if [ "${{matrix.size}}" == "big"]; then - echo "threads=8" >>/tmp/config.cfg - TABLE_SIZE=20000 - tables=4 - else - echo "threads=16" >>/tmp/config.cfg - TABLE_SIZE=10000 - tables=8 - fi - for item in oltp_point_select oltp_read_only oltp_write_only oltp_read_write oltp_point_select; do - sudo sysbench --config-file=/tmp/config.cfg $item --tables=$tables --time=300 --table-size=$TABLE_SIZE prepare - sudo sysbench --config-file=/tmp/config.cfg $item --tables=$tables --time=300 --table-size=$TABLE_SIZE run - sudo sysbench --config-file=/tmp/config.cfg $item --tables=$tables --time=300 --table-size=$TABLE_SIZE cleanup - done - - - name: Log - if: always() - run: | - echo "juicefs log" - sudo tail -n 1000 /var/log/juicefs.log - grep ":" /var/log/juicefs.log && exit 1 || true - - - name: Send Slack Notification - if: failure() - uses: juicedata/slack-notify-action@main - with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" - slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}" diff --git a/.github/workflows/tpch_clickhouse.yml b/.github/workflows/tpch_clickhouse.yml deleted file mode 100644 index 547730d74c39..000000000000 --- a/.github/workflows/tpch_clickhouse.yml +++ /dev/null @@ -1,303 +0,0 @@ -name: "tpch_clickhouse" - -on: - push: - branches: - - 'release-**' - paths-ignore: - - 'docs/**' - pull_request: - branches: - - 'release-**' - paths-ignore: - - 'docs/**' - schedule: - - cron: '30 20 * * *' - - workflow_dispatch: - -jobs: - tpch-clickhouse: - strategy: - fail-fast: false - matrix: - type: [ 'normal', 'limitcache'] - # type: ['local'] - - runs-on: ubuntu-20.04 - steps: - - name: Checkout - uses: actions/checkout@v3 - with: - fetch-depth: 1 - - - name: Build - uses: ./.github/actions/build - - - name: Run Redis - shell: bash - run: | - sudo docker run -d --name redis -v redis-data:/data \ - -p 6379:6379 redis redis-server --appendonly yes - - - name: Setup minio - shell: bash - run: | - docker run -d -p 9010:9000 --name minio \ - -e "MINIO_ACCESS_KEY=minioadmin" \ - -e "MINIO_SECRET_KEY=minioadmin" \ - -v /tmp/data:/data \ - -v /tmp/config:/root/.minio \ - minio/minio server /data - sleep 5 - - - name: Juicefs Format - shell: bash - run: | - sudo ./juicefs format --trash-days 0 --storage minio --bucket http://127.0.0.1:9010/mypics \ - --access-key minioadmin \ - --secret-key minioadmin \ - redis://127.0.0.1:6379/1 pics - - - name: Juicefs Mount - shell: bash - run: | - if [ "${{matrix.type}}" = "limitcache" ]; then - echo "mount: type is limitcache" - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /data/jfs --cache-size 3072 --no-usage-report - elif [ "${{matrix.type}}" = "normal" ]; then - echo "mount: type is normal" - sudo ./juicefs mount -d redis://127.0.0.1:6379/1 /data/jfs --no-usage-report - elif [ "${{matrix.type}}" = "local" ]; then - echo "mount: type is local" - sudo mkdir -p /data/jfs/ - sudo chmod -R 777 /data/ - else - echo ": invalid type" - exit 1 - fi - - - name: Install ClickHouse - shell: bash - run: | - sudo .github/scripts/apt_install.sh apt-transport-https ca-certificates dirmngr - echo "deb https://packages.clickhouse.com/deb stable main" | sudo tee /etc/apt/sources.list.d/clickhouse.list - sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 - sudo DEBIAN_FRONTEND=noninteractive .github/scripts/apt_install.sh clickhouse-server clickhouse-client - - - name: Change Datadir - shell: bash - run: | - sudo chmod 777 /etc/clickhouse-server/ - sudo chmod 777 /etc/clickhouse-server/config.xml - sudo sed -i "s?/var/lib/clickhouse/?/data/jfs/?" /etc/clickhouse-server/config.xml - sudo service clickhouse-server start - sudo service clickhouse-server status - ls -lah /data/jfs/ - - - name: Generate Data - shell: bash - run: | - cd /data/jfs/ - git clone https://github.com/sanwan/chdata.git || (rm -rf chdata;git clone https://github.com/sanwan/chdata.git) - cd chdata - cat lineorder.tbl.gz* >lineorder.tbl.gz - gzip -dk lineorder.tbl.gz - git clone https://github.com/sanwan/actionfile.git || (rm -rf actionfile;git clone https://github.com/sanwan/actionfile.git) - mv actionfile/*.tbl . - rm lineorder.tbl.gz* - ls -lrth - #sudo du -sh /data/jfs/ - df -lh - - - name: Create Table - shell: bash - run: | - clickhouse-client --query "CREATE TABLE customer - ( - C_CUSTKEY UInt32, - C_NAME String, - C_ADDRESS String, - C_CITY LowCardinality(String), - C_NATION LowCardinality(String), - C_REGION LowCardinality(String), - C_PHONE String, - C_MKTSEGMENT LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY (C_CUSTKEY);" - clickhouse-client --query "CREATE TABLE lineorder - ( - LO_ORDERKEY UInt32, - LO_LINENUMBER UInt8, - LO_CUSTKEY UInt32, - LO_PARTKEY UInt32, - LO_SUPPKEY UInt32, - LO_ORDERDATE Date, - LO_ORDERPRIORITY LowCardinality(String), - LO_SHIPPRIORITY UInt8, - LO_QUANTITY UInt8, - LO_EXTENDEDPRICE UInt32, - LO_ORDTOTALPRICE UInt32, - LO_DISCOUNT UInt8, - LO_REVENUE UInt32, - LO_SUPPLYCOST UInt32, - LO_TAX UInt8, - LO_COMMITDATE Date, - LO_SHIPMODE LowCardinality(String) - ) - ENGINE = MergeTree PARTITION BY toYear(LO_ORDERDATE) ORDER BY (LO_ORDERDATE, LO_ORDERKEY);" - clickhouse-client --query "CREATE TABLE part - ( - P_PARTKEY UInt32, - P_NAME String, - P_MFGR LowCardinality(String), - P_CATEGORY LowCardinality(String), - P_BRAND LowCardinality(String), - P_COLOR LowCardinality(String), - P_TYPE LowCardinality(String), - P_SIZE UInt8, - P_CONTAINER LowCardinality(String) - ) - ENGINE = MergeTree ORDER BY P_PARTKEY;" - clickhouse-client --query "CREATE TABLE supplier - ( - S_SUPPKEY UInt32, - S_NAME String, - S_ADDRESS String, - S_CITY LowCardinality(String), - S_NATION LowCardinality(String), - S_REGION LowCardinality(String), - S_PHONE String - ) - ENGINE = MergeTree ORDER BY S_SUPPKEY;" - - - - name: Import Data - shell: bash - run: | - df -lh - cd /data/jfs/chdata/ - clickhouse-client --query "INSERT INTO customer FORMAT CSV" < customer.tbl - clickhouse-client --query "INSERT INTO part FORMAT CSV" < part.tbl - clickhouse-client --query "INSERT INTO supplier FORMAT CSV" < supplier.tbl - time clickhouse-client --query "INSERT INTO lineorder FORMAT CSV" < lineorder.tbl - df -lh - - - name: ClickHouse Test - shell: bash - run: | - clickhouse-client --query "SET max_memory_usage = 20000000000;" - time clickhouse-client --query "CREATE TABLE lineorder_flat - ENGINE = MergeTree - PARTITION BY toYear(LO_ORDERDATE) - ORDER BY (LO_ORDERDATE, LO_ORDERKEY) AS - SELECT - l.LO_ORDERKEY AS LO_ORDERKEY, - l.LO_LINENUMBER AS LO_LINENUMBER, - l.LO_CUSTKEY AS LO_CUSTKEY, - l.LO_PARTKEY AS LO_PARTKEY, - l.LO_SUPPKEY AS LO_SUPPKEY, - l.LO_ORDERDATE AS LO_ORDERDATE, - l.LO_ORDERPRIORITY AS LO_ORDERPRIORITY, - l.LO_SHIPPRIORITY AS LO_SHIPPRIORITY, - l.LO_QUANTITY AS LO_QUANTITY, - l.LO_EXTENDEDPRICE AS LO_EXTENDEDPRICE, - l.LO_ORDTOTALPRICE AS LO_ORDTOTALPRICE, - l.LO_DISCOUNT AS LO_DISCOUNT, - l.LO_REVENUE AS LO_REVENUE, - l.LO_SUPPLYCOST AS LO_SUPPLYCOST, - l.LO_TAX AS LO_TAX, - l.LO_COMMITDATE AS LO_COMMITDATE, - l.LO_SHIPMODE AS LO_SHIPMODE, - c.C_NAME AS C_NAME, - c.C_ADDRESS AS C_ADDRESS, - c.C_CITY AS C_CITY, - c.C_NATION AS C_NATION, - c.C_REGION AS C_REGION, - c.C_PHONE AS C_PHONE, - c.C_MKTSEGMENT AS C_MKTSEGMENT, - s.S_NAME AS S_NAME, - s.S_ADDRESS AS S_ADDRESS, - s.S_CITY AS S_CITY, - s.S_NATION AS S_NATION, - s.S_REGION AS S_REGION, - s.S_PHONE AS S_PHONE, - p.P_NAME AS P_NAME, - p.P_MFGR AS P_MFGR, - p.P_CATEGORY AS P_CATEGORY, - p.P_BRAND AS P_BRAND, - p.P_COLOR AS P_COLOR, - p.P_TYPE AS P_TYPE, - p.P_SIZE AS P_SIZE, - p.P_CONTAINER AS P_CONTAINER - FROM lineorder AS l - INNER JOIN customer AS c ON c.C_CUSTKEY = l.LO_CUSTKEY - INNER JOIN supplier AS s ON s.S_SUPPKEY = l.LO_SUPPKEY - INNER JOIN part AS p ON p.P_PARTKEY = l.LO_PARTKEY;" - echo "Q1.1" - time clickhouse-client --query "SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue - FROM lineorder_flat - WHERE toYear(LO_ORDERDATE) = 1993 AND LO_DISCOUNT BETWEEN 1 AND 3 AND LO_QUANTITY < 25;" - echo "Q1.2" - time clickhouse-client --query "SELECT sum(LO_EXTENDEDPRICE * LO_DISCOUNT) AS revenue - FROM lineorder_flat - WHERE toYYYYMM(LO_ORDERDATE) = 199401 AND LO_DISCOUNT BETWEEN 4 AND 6 AND LO_QUANTITY BETWEEN 26 AND 35;" - echo "Q2.1" - time clickhouse-client --query "SELECT - sum(LO_REVENUE), - toYear(LO_ORDERDATE) AS year, - P_BRAND - FROM lineorder_flat - WHERE P_CATEGORY = 'MFGR#12' AND S_REGION = 'AMERICA' - GROUP BY - year, - P_BRAND - ORDER BY - year, - P_BRAND;" - echo "Q3.1" - time clickhouse-client --query "SELECT - C_NATION, - S_NATION, - toYear(LO_ORDERDATE) AS year, - sum(LO_REVENUE) AS revenue - FROM lineorder_flat - WHERE C_REGION = 'ASIA' AND S_REGION = 'ASIA' AND year >= 1992 AND year <= 1997 - GROUP BY - C_NATION, - S_NATION, - year - ORDER BY - year ASC, - revenue DESC;" - echo "Q4.1" - time clickhouse-client --query "SELECT - toYear(LO_ORDERDATE) AS year, - C_NATION, - sum(LO_REVENUE - LO_SUPPLYCOST) AS profit - FROM lineorder_flat - WHERE C_REGION = 'AMERICA' AND S_REGION = 'AMERICA' AND (P_MFGR = 'MFGR#1' OR P_MFGR = 'MFGR#2') - GROUP BY - year, - C_NATION - ORDER BY - year ASC, - C_NATION ASC;" - df -lh - - - name: Log - shell: bash - if: always() - run: | - if [ -f /var/log/juicefs.log ]; then - echo "juicefs log" - sudo tail -n 1000 /var/log/juicefs.log - grep ":" /var/log/juicefs.log && exit 1 || true - fi - - - name: Send Slack Notification - if: failure() - uses: juicedata/slack-notify-action@main - with: - channel-id: "${{ secrets.SLACK_CHANNEL_ID_FOR_PR_CHECK_NOTIFY }}" - slack_bot_token: "${{ secrets.SLACK_BOT_TOKEN }}"