Skip to content

Commit

Permalink
Merge branch 'main' into spark-jdbc
Browse files Browse the repository at this point in the history
  • Loading branch information
liangyouze authored Jan 22, 2025
2 parents 41076d9 + e56536b commit 95463d6
Show file tree
Hide file tree
Showing 564 changed files with 32,317 additions and 4,383 deletions.
83 changes: 0 additions & 83 deletions .github/CONTRIBUTING

This file was deleted.

2 changes: 1 addition & 1 deletion .github/PULL_REQUEST_TEMPLATE
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
1. If you propose a new API, clarify the use case for a new API.
2. If you fix a bug, describe the bug.)

Fix: # (issue)
Fix: #(issue)

### Does this PR introduce _any_ user-facing change?

Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/access-control-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ jobs:
./gradlew -PtestMode=deploy -PjdbcBackend=postgresql -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :authorizations:test
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: authorizations-integrate-test-reports-${{ matrix.java-version }}
Expand Down
40 changes: 11 additions & 29 deletions .github/workflows/auto-cherry-pick.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,60 +7,42 @@ on:
types: ["closed"]

jobs:
cherry_pick_branch_0_5:
runs-on: ubuntu-latest
name: Cherry pick into branch_0.5
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.5') && github.event.pull_request.merged == true }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Cherry pick into branch-0.5
uses: carloscastrojumo/[email protected]
with:
branch: branch-0.5
labels: |
cherry-pick
reviewers: |
jerryshao
cherry_pick_branch_0_6:
cherry_pick_branch_0_7:
runs-on: ubuntu-latest
name: Cherry pick into branch_0.6
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.6') && github.event.pull_request.merged == true }}
name: Cherry pick into branch_0.7
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.7') && github.event.pull_request.merged == true }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Cherry pick into branch-0.6
- name: Cherry pick into branch-0.7
uses: carloscastrojumo/[email protected]
with:
branch: branch-0.6
branch: branch-0.7
labels: |
cherry-pick
reviewers: |
jerryshao
cherry_pick_branch_0_7:
cherry_pick_branch_0_8:
runs-on: ubuntu-latest
name: Cherry pick into branch_0.7
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.7') && github.event.pull_request.merged == true }}
name: Cherry pick into branch_0.8
if: ${{ contains(github.event.pull_request.labels.*.name, 'branch-0.8') && github.event.pull_request.merged == true }}
steps:
- name: Checkout
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Cherry pick into branch-0.7
- name: Cherry pick into branch-0.8
uses: carloscastrojumo/[email protected]
with:
branch: branch-0.7
branch: branch-0.8
labels: |
cherry-pick
reviewers: |
jerryshao
FANNG1
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
5 changes: 3 additions & 2 deletions .github/workflows/backend-integration-test-action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,11 @@ jobs:
-x :web:web:test -x :web:integration-test:test -x :clients:client-python:test -x :flink-connector:flink:test -x :spark-connector:spark-common:test
-x :spark-connector:spark-3.3:test -x :spark-connector:spark-3.4:test -x :spark-connector:spark-3.5:test
-x :spark-connector:spark-runtime-3.3:test -x :spark-connector:spark-runtime-3.4:test -x :spark-connector:spark-runtime-3.5:test
-x :authorizations:authorization-ranger:test -x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
-x :trino-connector:integration-test:test -x :trino-connector:trino-connector:test
-x :authorizations:authorization-chain:test -x :authorizations:authorization-ranger:test
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: integrate-test-reports-${{ inputs.java-version }}-${{ inputs.test-mode }}-${{ inputs.backend }}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/backend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
- meta/**
- scripts/**
- server/**
- bundles/**
- server-common/**
- build.gradle.kts
- gradle.properties
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ jobs:
./gradlew :spark-connector:spark-3.5:build -PscalaVersion=2.13 -PskipITs -PskipDockerTests=false
- name: Upload unit tests report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: failure()
with:
name: unit test report
Expand Down Expand Up @@ -129,7 +129,7 @@ jobs:
run: ./gradlew build -PskipITs -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false -x :clients:client-python:build

- name: Upload unit tests report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: failure()
with:
name: unit test report
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/cron-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ jobs:
./gradlew test -PskipTests -PtestMode=${{ matrix.test-mode }} -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
with:
name: integrate test reports
Expand Down
27 changes: 23 additions & 4 deletions .github/workflows/docker-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,11 @@ on:
description: 'Publish Docker token'
required: true
type: string
publish-latest-tag:
description: 'Whether to update the latest tag. This operation is only applicable to official releases and should not be used for Release Candidate (RC).'
required: false
type: boolean
default: false

jobs:
publish-docker-image:
Expand Down Expand Up @@ -83,6 +88,12 @@ jobs:
echo "image_type=iceberg-rest-server" >> $GITHUB_ENV
echo "image_name=apache/gravitino-iceberg-rest" >> $GITHUB_ENV
fi
if [ "${{ github.event.inputs.publish-latest-tag }}" == "true" ]; then
echo "publish_latest=true" >> $GITHUB_ENV
else
echo "publish_latest=false" >> $GITHUB_ENV
fi
- name: Check publish Docker token
run: |
Expand Down Expand Up @@ -115,8 +126,16 @@ jobs:
sudo rm -rf /usr/local/lib/android
sudo rm -rf /opt/hostedtoolcache/CodeQL
if [[ "${image_type}" == "gravitino" || "${image_type}" == "iceberg-rest-server" ]]; then
./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${{ github.event.inputs.version }} --latest
if [[ -n "${tag_name}" ]]; then
full_tag_name="${tag_name}-${{ github.event.inputs.version }}"
else
full_tag_name="${{ github.event.inputs.version }}"
fi
if [[ "${publish_latest}" == "true" ]]; then
echo "Publish tag ${full_tag_name}, and update latest too."
./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name} --latest
else
./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag "${tag_name}-${{ github.event.inputs.version }}"
fi
echo "Publish tag ${full_tag_name}."
./dev/docker/build-docker.sh --platform all --type ${image_type} --image ${image_name} --tag ${full_tag_name}
fi
2 changes: 1 addition & 1 deletion .github/workflows/flink-integration-test-action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ jobs:
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ inputs.java-version }} -PskipDockerTests=false :flink-connector:flink:test --tests "org.apache.gravitino.flink.connector.integration.test.**"
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: flink-connector-integrate-test-reports-${{ inputs.java-version }}
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/frontend-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,7 @@ jobs:
./gradlew -PskipTests -PtestMode=deploy -PjdkVersion=${{ matrix.java-version }} -PskipDockerTests=false :web:integration-test:test
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: integrate-test-reports-${{ matrix.java-version }}
Expand Down
99 changes: 99 additions & 0 deletions .github/workflows/gvfs-fuse-build-test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
name: Build gvfs-fuse and testing

# Controls when the workflow will run
on:
push:
branches: [ "main", "branch-*" ]
pull_request:
branches: [ "main", "branch-*" ]
workflow_dispatch:

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

jobs:
changes:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dorny/paths-filter@v2
id: filter
with:
filters: |
source_changes:
- .github/**
- api/**
- bin/**
- catalogs/hadoop/**
- clients/filesystem-fuse/**
- common/**
- conf/**
- core/**
- dev/**
- gradle/**
- meta/**
- scripts/**
- server/**
- server-common/**
- build.gradle.kts
- gradle.properties
- gradlew
- setting.gradle.kts
outputs:
source_changes: ${{ steps.filter.outputs.source_changes }}

# Build for AMD64 architecture
Gvfs-Build:
needs: changes
if: needs.changes.outputs.source_changes == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
architecture: [linux/amd64]
java-version: [ 17 ]
env:
PLATFORM: ${{ matrix.architecture }}
steps:
- uses: actions/checkout@v3

- uses: actions/setup-java@v4
with:
java-version: ${{ matrix.java-version }}
distribution: 'temurin'
cache: 'gradle'

- name: Set up QEMU
uses: docker/setup-qemu-action@v2

- name: Check required command
run: |
dev/ci/check_commands.sh
- name: Build Gvfs-fuse
run: |
./gradlew :clients:filesystem-fuse:build -PenableFuse=true
- name: Integration test
run: |
./gradlew build -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
./gradlew compileDistribution -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }}
cd clients/filesystem-fuse
make test-s3
make test-fuse-it
- name: Free up disk space
run: |
dev/ci/util_free_space.sh
- name: Upload tests reports
uses: actions/upload-artifact@v4
if: ${{ (failure() && steps.integrationTest.outcome == 'failure') || contains(github.event.pull_request.labels.*.name, 'upload log') }}
with:
name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }}
path: |
clients/filesystem-fuse/target/debug/fuse.log
distribution/package/logs/gravitino-server.out
distribution/package/logs/gravitino-server.log
2 changes: 1 addition & 1 deletion .github/workflows/python-integration-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ jobs:
done
- name: Upload integrate tests reports
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v4
if: ${{ failure() && steps.integrationTest.outcome == 'failure' }}
with:
name: integrate test reports
Expand Down
Loading

0 comments on commit 95463d6

Please sign in to comment.