diff --git a/.github/workflows/gvfs-fuse-build-test.yml b/.github/workflows/gvfs-fuse-build-test.yml index 4af01d82da3..4fe7b66e09d 100644 --- a/.github/workflows/gvfs-fuse-build-test.yml +++ b/.github/workflows/gvfs-fuse-build-test.yml @@ -71,10 +71,18 @@ jobs: run: | dev/ci/check_commands.sh - - name: Build and test Gravitino + - name: Build Gvfs-fuse run: | ./gradlew :clients:filesystem-fuse:build -PenableFuse=true + - name: Integration test + run: | + ./gradlew build -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }} + ./gradlew compileDistribution -x :clients:client-python:build -x test -x web -PjdkVersion=${{ matrix.java-version }} + cd clients/filesystem-fuse + make test-s3 + make test-fuse-it + - name: Free up disk space run: | dev/ci/util_free_space.sh @@ -85,5 +93,7 @@ jobs: with: name: Gvfs-fuse integrate-test-reports-${{ matrix.java-version }} path: | - clients/filesystem-fuse/build/test/log/*.log + clients/filesystem-fuse/target/debug/fuse.log + distribution/package/logs/gravitino-server.out + distribution/package/logs/gravitino-server.log diff --git a/LICENSE.bin b/LICENSE.bin index effaa4ac4a2..d1dddd52795 100644 --- a/LICENSE.bin +++ b/LICENSE.bin @@ -374,6 +374,7 @@ Apache Arrow Rome Jettison + Awaitility This product bundles various third-party components also under the Apache Software Foundation License 1.1 diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java index a554559ea5c..e1e9f6955d2 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/RangerClientExtension.java @@ -100,12 +100,14 @@ public RangerClientExtension(String hostName, String authType, String username, } } + @Override public RangerPolicy createPolicy(RangerPolicy policy) throws RangerServiceException { Preconditions.checkArgument( policy.getResources().size() > 0, "Ranger policy resources can not be empty!"); return super.createPolicy(policy); } + @Override public RangerPolicy updatePolicy(long policyId, RangerPolicy policy) throws RangerServiceException { Preconditions.checkArgument( diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java index 3a58f5c95a0..611127ec3f2 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXGroup.java @@ -60,6 +60,7 @@ public VXGroup() { * * @return formatedStr */ + @Override public String toString() { String str = "VXGroup={"; str += super.toString(); diff --git a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java index f605d987de0..3dbc2b0236b 100644 --- a/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java +++ b/authorizations/authorization-ranger/src/main/java/org/apache/gravitino/authorization/ranger/reference/VXUser.java @@ -75,6 +75,7 @@ public String getName() { * * @return formatedStr */ + @Override public String toString() { String str = "VXUser={"; str += super.toString(); diff --git a/bin/common.sh b/bin/common.sh.template similarity index 90% rename from bin/common.sh rename to bin/common.sh.template index a6f002ad91d..b81710a3fc5 100644 --- a/bin/common.sh +++ b/bin/common.sh.template @@ -42,6 +42,15 @@ if [[ -f "${GRAVITINO_CONF_DIR}/gravitino-env.sh" ]]; then . "${GRAVITINO_CONF_DIR}/gravitino-env.sh" fi +if [[ -z "${GRAVITINO_VERSION}" ]]; then + echo -e "GRAVITINO_VERSION is not set, you may need to:\n" \ + "1. Ensure that a compiled version of Gravitino is available at " \ + "\${GRAVITINO_HOME}/distribution/package. You may need to compile it first, " \ + "if you are installing the software from source code.\n" \ + "2. Execute gravitino.sh in the \${GRAVITINO_HOME}/distribution/package/bin directory." + exit 1 +fi + GRAVITINO_CLASSPATH+=":${GRAVITINO_CONF_DIR}" JVM_VERSION=8 diff --git a/bin/gravitino-iceberg-rest-server.sh b/bin/gravitino-iceberg-rest-server.sh.template similarity index 100% rename from bin/gravitino-iceberg-rest-server.sh rename to bin/gravitino-iceberg-rest-server.sh.template diff --git a/bin/gravitino.sh b/bin/gravitino.sh.template similarity index 100% rename from bin/gravitino.sh rename to bin/gravitino.sh.template diff --git a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java index 0dcf24f3a67..7b1832fe56d 100644 --- a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java +++ b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonPropertiesUtils.java @@ -32,25 +32,41 @@ public class PaimonPropertiesUtils { // will only need to set the configuration 'catalog-backend' in Gravitino and Gravitino will // change it to `catalogType` automatically and pass it to Paimon. public static final Map GRAVITINO_CONFIG_TO_PAIMON; + public static final Map PAIMON_CATALOG_CONFIG_TO_GRAVITINO; static { - Map map = new HashMap(); - map.put(PaimonConstants.CATALOG_BACKEND, PaimonConstants.CATALOG_BACKEND); - map.put(PaimonConstants.GRAVITINO_JDBC_DRIVER, PaimonConstants.GRAVITINO_JDBC_DRIVER); - map.put(PaimonConstants.GRAVITINO_JDBC_USER, PaimonConstants.PAIMON_JDBC_USER); - map.put(PaimonConstants.GRAVITINO_JDBC_PASSWORD, PaimonConstants.PAIMON_JDBC_PASSWORD); - map.put(PaimonConstants.URI, PaimonConstants.URI); - map.put(PaimonConstants.WAREHOUSE, PaimonConstants.WAREHOUSE); - map.put(PaimonConstants.CATALOG_BACKEND_NAME, PaimonConstants.CATALOG_BACKEND_NAME); + Map gravitinoConfigToPaimon = new HashMap<>(); + Map paimonCatalogConfigToGravitino = new HashMap<>(); + gravitinoConfigToPaimon.put(PaimonConstants.CATALOG_BACKEND, PaimonConstants.CATALOG_BACKEND); + gravitinoConfigToPaimon.put( + PaimonConstants.GRAVITINO_JDBC_DRIVER, PaimonConstants.GRAVITINO_JDBC_DRIVER); + gravitinoConfigToPaimon.put( + PaimonConstants.GRAVITINO_JDBC_USER, PaimonConstants.PAIMON_JDBC_USER); + gravitinoConfigToPaimon.put( + PaimonConstants.GRAVITINO_JDBC_PASSWORD, PaimonConstants.PAIMON_JDBC_PASSWORD); + gravitinoConfigToPaimon.put(PaimonConstants.URI, PaimonConstants.URI); + gravitinoConfigToPaimon.put(PaimonConstants.WAREHOUSE, PaimonConstants.WAREHOUSE); + gravitinoConfigToPaimon.put( + PaimonConstants.CATALOG_BACKEND_NAME, PaimonConstants.CATALOG_BACKEND_NAME); // S3 - map.put(S3Properties.GRAVITINO_S3_ENDPOINT, PaimonConstants.S3_ENDPOINT); - map.put(S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, PaimonConstants.S3_ACCESS_KEY); - map.put(S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, PaimonConstants.S3_SECRET_KEY); + gravitinoConfigToPaimon.put(S3Properties.GRAVITINO_S3_ENDPOINT, PaimonConstants.S3_ENDPOINT); + gravitinoConfigToPaimon.put( + S3Properties.GRAVITINO_S3_ACCESS_KEY_ID, PaimonConstants.S3_ACCESS_KEY); + gravitinoConfigToPaimon.put( + S3Properties.GRAVITINO_S3_SECRET_ACCESS_KEY, PaimonConstants.S3_SECRET_KEY); // OSS - map.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, PaimonConstants.OSS_ENDPOINT); - map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, PaimonConstants.OSS_ACCESS_KEY); - map.put(OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, PaimonConstants.OSS_SECRET_KEY); - GRAVITINO_CONFIG_TO_PAIMON = Collections.unmodifiableMap(map); + gravitinoConfigToPaimon.put(OSSProperties.GRAVITINO_OSS_ENDPOINT, PaimonConstants.OSS_ENDPOINT); + gravitinoConfigToPaimon.put( + OSSProperties.GRAVITINO_OSS_ACCESS_KEY_ID, PaimonConstants.OSS_ACCESS_KEY); + gravitinoConfigToPaimon.put( + OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET, PaimonConstants.OSS_SECRET_KEY); + GRAVITINO_CONFIG_TO_PAIMON = Collections.unmodifiableMap(gravitinoConfigToPaimon); + gravitinoConfigToPaimon.forEach( + (key, value) -> { + paimonCatalogConfigToGravitino.put(value, key); + }); + PAIMON_CATALOG_CONFIG_TO_GRAVITINO = + Collections.unmodifiableMap(paimonCatalogConfigToGravitino); } /** diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index d599a5e72f1..3108d993c1a 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -54,6 +54,7 @@ dependencies { exclude("org.fusesource.leveldbjni") } implementation(libs.slf4j.api) + implementation(libs.awaitility) compileOnly(libs.guava) diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java index 36177bea37f..6c032414be5 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogOperations.java @@ -31,6 +31,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.lang3.StringUtils; import org.apache.gravitino.Catalog; import org.apache.gravitino.Entity; @@ -71,6 +73,8 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.awaitility.Awaitility; +import org.awaitility.core.ConditionTimeoutException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -755,6 +759,35 @@ FileSystem getFileSystem(Path path, Map config) throws IOExcepti scheme, path, fileSystemProvidersMap.keySet(), fileSystemProvidersMap.values())); } - return provider.getFileSystem(path, config); + int timeoutSeconds = + (int) + propertiesMetadata + .catalogPropertiesMetadata() + .getOrDefault( + config, HadoopCatalogPropertiesMetadata.FILESYSTEM_CONNECTION_TIMEOUT_SECONDS); + try { + AtomicReference fileSystem = new AtomicReference<>(); + Awaitility.await() + .atMost(timeoutSeconds, TimeUnit.SECONDS) + .until( + () -> { + fileSystem.set(provider.getFileSystem(path, config)); + return true; + }); + return fileSystem.get(); + } catch (ConditionTimeoutException e) { + throw new IOException( + String.format( + "Failed to get FileSystem for path: %s, scheme: %s, provider: %s, config: %s within %s " + + "seconds, please check the configuration or increase the " + + "file system connection timeout time by setting catalog property: %s", + path, + scheme, + provider, + config, + timeoutSeconds, + HadoopCatalogPropertiesMetadata.FILESYSTEM_CONNECTION_TIMEOUT_SECONDS), + e); + } } } diff --git a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java index 22cf0d5b2cd..3bdc125efc8 100644 --- a/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java +++ b/catalogs/catalog-hadoop/src/main/java/org/apache/gravitino/catalog/hadoop/HadoopCatalogPropertiesMetadata.java @@ -53,6 +53,9 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada */ public static final String DEFAULT_FS_PROVIDER = "default-filesystem-provider"; + static final String FILESYSTEM_CONNECTION_TIMEOUT_SECONDS = "filesystem-conn-timeout-secs"; + static final int DEFAULT_GET_FILESYSTEM_TIMEOUT_SECONDS = 6; + public static final String BUILTIN_LOCAL_FS_PROVIDER = "builtin-local"; public static final String BUILTIN_HDFS_FS_PROVIDER = "builtin-hdfs"; @@ -82,6 +85,14 @@ public class HadoopCatalogPropertiesMetadata extends BaseCatalogPropertiesMetada false /* immutable */, BUILTIN_LOCAL_FS_PROVIDER, // please see LocalFileSystemProvider#name() false /* hidden */)) + .put( + FILESYSTEM_CONNECTION_TIMEOUT_SECONDS, + PropertyEntry.integerOptionalPropertyEntry( + FILESYSTEM_CONNECTION_TIMEOUT_SECONDS, + "Timeout to wait for to create the Hadoop file system client instance.", + false /* immutable */, + DEFAULT_GET_FILESYSTEM_TIMEOUT_SECONDS, + false /* hidden */)) // The following two are about authentication. .putAll(KERBEROS_PROPERTY_ENTRIES) .putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES) diff --git a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java index aa6348e2f71..829088f0131 100644 --- a/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java +++ b/catalogs/catalog-jdbc-doris/src/main/java/org/apache/gravitino/catalog/doris/operation/DorisTableOperations.java @@ -567,10 +567,6 @@ protected String generateAlterTableSql( alterSql.add("MODIFY COMMENT \"" + newComment + "\""); } - if (!setProperties.isEmpty()) { - alterSql.add(generateTableProperties(setProperties)); - } - if (CollectionUtils.isEmpty(alterSql)) { return ""; } @@ -602,11 +598,14 @@ private String updateColumnNullabilityDefinition( } private String generateTableProperties(List setProperties) { - return setProperties.stream() - .map( - setProperty -> - String.format("\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue())) - .collect(Collectors.joining(",\n")); + String properties = + setProperties.stream() + .map( + setProperty -> + String.format( + "\"%s\" = \"%s\"", setProperty.getProperty(), setProperty.getValue())) + .collect(Collectors.joining(",\n")); + return "set (" + properties + ")"; } private String updateColumnCommentFieldDefinition( diff --git a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java index 9288c9616bc..9d2c798ae7e 100644 --- a/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java +++ b/catalogs/catalog-jdbc-doris/src/test/java/org/apache/gravitino/catalog/doris/integration/test/CatalogDorisIT.java @@ -577,6 +577,16 @@ void testAlterDorisTable() { .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) .untilAsserted( () -> assertEquals(4, tableCatalog.loadTable(tableIdentifier).columns().length)); + + // set property + tableCatalog.alterTable(tableIdentifier, TableChange.setProperty("in_memory", "true")); + Awaitility.await() + .atMost(MAX_WAIT_IN_SECONDS, TimeUnit.SECONDS) + .pollInterval(WAIT_INTERVAL_IN_SECONDS, TimeUnit.SECONDS) + .untilAsserted( + () -> + assertEquals( + "true", tableCatalog.loadTable(tableIdentifier).properties().get("in_memory"))); } @Test diff --git a/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java index b8cc2f87233..36b4daebf9b 100644 --- a/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/operation/MysqlTableOperations.java @@ -106,6 +106,7 @@ protected String generateCreateTableSql( } } + validateIndexes(indexes, columns); appendIndexesSql(indexes, sqlBuilder); sqlBuilder.append("\n)"); @@ -642,4 +643,33 @@ private StringBuilder appendColumnDefinition(JdbcColumn column, StringBuilder sq private static String quote(String name) { return BACK_QUOTE + name + BACK_QUOTE; } + + /** + * Verify the columns in the index. + * + * @param columns jdbc column + * @param indexes table indexes + */ + private static void validateIndexes(Index[] indexes, JdbcColumn[] columns) { + Map columnMap = + Arrays.stream(columns).collect(Collectors.toMap(JdbcColumn::name, c -> c)); + for (Index index : indexes) { + if (index.type() == Index.IndexType.UNIQUE_KEY) { + // the column in the unique index must be not null + for (String[] colNames : index.fieldNames()) { + JdbcColumn column = columnMap.get(colNames[0]); + Preconditions.checkArgument( + column != null, + "Column %s in the unique index %s does not exist in the table", + colNames[0], + index.name()); + Preconditions.checkArgument( + !column.nullable(), + "Column %s in the unique index %s must be a not null column", + colNames[0], + index.name()); + } + } + } + } } diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java index a80da4795a0..9bd949b7b31 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java @@ -1037,6 +1037,27 @@ void testCreateTableIndex() { Assertions.assertEquals(2, table.index().length); Assertions.assertNotNull(table.index()[0].name()); Assertions.assertNotNull(table.index()[1].name()); + + Column notNullCol = Column.of("col_6", Types.LongType.get(), "id", true, false, null); + Exception exception = + assertThrows( + IllegalArgumentException.class, + () -> + tableCatalog.createTable( + tableIdent, + new Column[] {notNullCol}, + table_comment, + properties, + Transforms.EMPTY_TRANSFORM, + Distributions.NONE, + new SortOrder[0], + new Index[] { + Indexes.of(Index.IndexType.UNIQUE_KEY, null, new String[][] {{"col_6"}}), + })); + Assertions.assertTrue( + exception + .getMessage() + .contains("Column col_6 in the unique index null must be a not null column")); } @Test diff --git a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java index 9eac348cd91..923e20fa0c0 100644 --- a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java +++ b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/operation/TestMysqlTableOperations.java @@ -64,7 +64,7 @@ public void testOperationTable() { .withName("col_1") .withType(VARCHAR) .withComment("test_comment") - .withNullable(true) + .withNullable(false) .build()); columns.add( JdbcColumn.builder() @@ -573,7 +573,7 @@ public void testCreateAndLoadTable() { JdbcColumn.builder() .withName("col_4") .withType(Types.DateType.get()) - .withNullable(true) + .withNullable(false) .withComment("date") .withDefaultValue(Column.DEFAULT_VALUE_NOT_SET) .build()); diff --git a/catalogs/catalog-jdbc-oceanbase/src/main/java/org/apache/gravitino/catalog/oceanbase/operation/OceanBaseTableOperations.java b/catalogs/catalog-jdbc-oceanbase/src/main/java/org/apache/gravitino/catalog/oceanbase/operation/OceanBaseTableOperations.java index 77c97290927..98f2d174f1a 100644 --- a/catalogs/catalog-jdbc-oceanbase/src/main/java/org/apache/gravitino/catalog/oceanbase/operation/OceanBaseTableOperations.java +++ b/catalogs/catalog-jdbc-oceanbase/src/main/java/org/apache/gravitino/catalog/oceanbase/operation/OceanBaseTableOperations.java @@ -185,6 +185,7 @@ protected Map getTableProperties(Connection connection, String t } } + @Override protected void correctJdbcTableFields( Connection connection, String databaseName, String tableName, JdbcTable.Builder tableBuilder) throws SQLException { diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java index 7b27438d2e5..aef42044c0c 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergCatalogOperations.java @@ -58,6 +58,7 @@ import org.apache.gravitino.rel.TableCatalog; import org.apache.gravitino.rel.TableChange; import org.apache.gravitino.rel.expressions.distributions.Distribution; +import org.apache.gravitino.rel.expressions.distributions.Distributions; import org.apache.gravitino.rel.expressions.sorts.SortOrder; import org.apache.gravitino.rel.expressions.transforms.Transform; import org.apache.gravitino.rel.indexes.Index; @@ -513,6 +514,13 @@ public Table createTable( .build()) .toArray(IcebergColumn[]::new); + // Gravitino NONE distribution means the client side doesn't specify distribution, which is + // not the same as none distribution in Iceberg. + if (Distributions.NONE.equals(distribution)) { + distribution = + getIcebergDefaultDistribution(sortOrders.length > 0, partitioning.length > 0); + } + IcebergTable createdTable = IcebergTable.builder() .withName(tableIdent.name()) @@ -588,6 +596,16 @@ public void testConnection( } } + private static Distribution getIcebergDefaultDistribution( + boolean isSorted, boolean isPartitioned) { + if (isSorted) { + return Distributions.RANGE; + } else if (isPartitioned) { + return Distributions.HASH; + } + return Distributions.NONE; + } + private static String currentUser() { return PrincipalUtils.getCurrentUserName(); } diff --git a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTable.java b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTable.java index 27e3c429e7c..3f2f54c1b3e 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTable.java +++ b/catalogs/catalog-lakehouse-iceberg/src/main/java/org/apache/gravitino/catalog/lakehouse/iceberg/IcebergTable.java @@ -152,21 +152,6 @@ public static IcebergTable fromIcebergTable(TableMetadata table, String tableNam Schema schema = table.schema(); Transform[] partitionSpec = FromIcebergPartitionSpec.fromPartitionSpec(table.spec(), schema); SortOrder[] sortOrder = FromIcebergSortOrder.fromSortOrder(table.sortOrder()); - Distribution distribution = Distributions.NONE; - String distributionName = properties.get(IcebergTablePropertiesMetadata.DISTRIBUTION_MODE); - if (null != distributionName) { - switch (DistributionMode.fromName(distributionName)) { - case HASH: - distribution = Distributions.HASH; - break; - case RANGE: - distribution = Distributions.RANGE; - break; - default: - // do nothing - break; - } - } IcebergColumn[] icebergColumns = schema.columns().stream().map(ConvertUtil::fromNestedField).toArray(IcebergColumn[]::new); return IcebergTable.builder() @@ -178,7 +163,7 @@ public static IcebergTable fromIcebergTable(TableMetadata table, String tableNam .withAuditInfo(AuditInfo.EMPTY) .withPartitioning(partitionSpec) .withSortOrders(sortOrder) - .withDistribution(distribution) + .withDistribution(getDistribution(properties)) .build(); } @@ -236,4 +221,23 @@ protected IcebergTable internalBuild() { public static Builder builder() { return new Builder(); } + + private static Distribution getDistribution(Map properties) { + Distribution distribution = Distributions.NONE; + String distributionName = properties.get(IcebergTablePropertiesMetadata.DISTRIBUTION_MODE); + if (null != distributionName) { + switch (DistributionMode.fromName(distributionName)) { + case HASH: + distribution = Distributions.HASH; + break; + case RANGE: + distribution = Distributions.RANGE; + break; + default: + // do nothing + break; + } + } + return distribution; + } } diff --git a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java index fd37441b459..f0162a6ec88 100644 --- a/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java +++ b/catalogs/catalog-lakehouse-iceberg/src/test/java/org/apache/gravitino/catalog/lakehouse/iceberg/integration/test/CatalogIcebergBaseIT.java @@ -379,6 +379,76 @@ void testCreateTableWithNullComment() { Assertions.assertNull(loadTable.comment()); } + @Test + void testCreateTableWithNoneDistribution() { + // Create table from Gravitino API + Column[] columns = createColumns(); + + NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); + Distribution distribution = Distributions.NONE; + + final SortOrder[] sortOrders = + new SortOrder[] { + SortOrders.of( + NamedReference.field(ICEBERG_COL_NAME2), + SortDirection.DESCENDING, + NullOrdering.NULLS_FIRST) + }; + + Transform[] partitioning = new Transform[] {Transforms.day(columns[1].name())}; + Map properties = createProperties(); + TableCatalog tableCatalog = catalog.asTableCatalog(); + Table tableWithPartitionAndSortorder = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + sortOrders); + Assertions.assertEquals(tableName, tableWithPartitionAndSortorder.name()); + Assertions.assertEquals(Distributions.RANGE, tableWithPartitionAndSortorder.distribution()); + + Table loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals(tableName, loadTable.name()); + Assertions.assertEquals(Distributions.RANGE, loadTable.distribution()); + tableCatalog.dropTable(tableIdentifier); + + Table tableWithPartition = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + partitioning, + distribution, + new SortOrder[0]); + Assertions.assertEquals(tableName, tableWithPartition.name()); + Assertions.assertEquals(Distributions.HASH, tableWithPartition.distribution()); + + loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals(tableName, loadTable.name()); + Assertions.assertEquals(Distributions.HASH, loadTable.distribution()); + tableCatalog.dropTable(tableIdentifier); + + Table tableWithoutPartitionAndSortOrder = + tableCatalog.createTable( + tableIdentifier, + columns, + table_comment, + properties, + new Transform[0], + distribution, + new SortOrder[0]); + Assertions.assertEquals(tableName, tableWithoutPartitionAndSortOrder.name()); + Assertions.assertEquals(Distributions.NONE, tableWithoutPartitionAndSortOrder.distribution()); + + loadTable = tableCatalog.loadTable(tableIdentifier); + Assertions.assertEquals(tableName, loadTable.name()); + Assertions.assertEquals(Distributions.NONE, loadTable.distribution()); + } + @Test void testCreateAndLoadIcebergTable() { // Create table from Gravitino API @@ -968,9 +1038,9 @@ public void testTableDistribution() { columns, table_comment, properties, - partitioning, + new Transform[0], distribution, - sortOrders); + new SortOrder[0]); Table loadTable = tableCatalog.loadTable(tableIdentifier); @@ -981,8 +1051,8 @@ public void testTableDistribution() { Arrays.asList(columns), properties, distribution, - sortOrders, - partitioning, + new SortOrder[0], + new Transform[0], loadTable); Assertions.assertDoesNotThrow(() -> tableCatalog.dropTable(tableIdentifier)); @@ -1179,7 +1249,7 @@ public void testTableSortOrder() { Column[] columns = createColumns(); NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName); - Distribution distribution = Distributions.NONE; + Distribution distribution = Distributions.HASH; final SortOrder[] sortOrders = new SortOrder[] { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java new file mode 100644 index 00000000000..c0775dae966 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/ColumnCommandHandler.java @@ -0,0 +1,236 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import com.google.common.collect.Lists; +import java.util.List; +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** Handles the command execution for Columns based on command type and the command line options. */ +public class ColumnCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private final String catalog; + private final String schema; + private final String table; + private String column; + + /** + * Constructs a {@link ColumnCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public ColumnCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = getUrl(line); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + this.catalog = name.getCatalogName(); + this.schema = name.getSchemaName(); + this.table = name.getTableName(); + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(getAuth(line), userName); + + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + if (table == null) missingEntities.add(CommandEntities.TABLE); + + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); + handleListCommand(); + return; + } + + this.column = name.getColumnName(); + if (column == null) missingEntities.add(CommandEntities.COLUMN); + checkEntities(missingEntities); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + if (line.hasOption(GravitinoOptions.AUDIT)) { + gravitinoCommandLine + .newColumnAudit(url, ignore, metalake, catalog, schema, table, column) + .validate() + .handle(); + } else { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); + } + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + String position = line.getOptionValue(GravitinoOptions.POSITION); + boolean nullable = + !line.hasOption(GravitinoOptions.NULL) + || line.getOptionValue(GravitinoOptions.NULL).equals("true"); + boolean autoIncrement = + line.hasOption(GravitinoOptions.AUTO) + && line.getOptionValue(GravitinoOptions.AUTO).equals("true"); + String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); + + gravitinoCommandLine + .newAddColumn( + url, + ignore, + metalake, + catalog, + schema, + table, + column, + datatype, + comment, + position, + nullable, + autoIncrement, + defaultValue) + .validate() + .handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + gravitinoCommandLine + .newDeleteColumn(url, ignore, metalake, catalog, schema, table, column) + .validate() + .handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + if (line.hasOption(GravitinoOptions.COMMENT)) { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateColumnComment(url, ignore, metalake, catalog, schema, table, column, comment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + gravitinoCommandLine + .newUpdateColumnName(url, ignore, metalake, catalog, schema, table, column, newName) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.DATATYPE) && !line.hasOption(GravitinoOptions.DEFAULT)) { + String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); + gravitinoCommandLine + .newUpdateColumnDatatype(url, ignore, metalake, catalog, schema, table, column, datatype) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.POSITION)) { + String position = line.getOptionValue(GravitinoOptions.POSITION); + gravitinoCommandLine + .newUpdateColumnPosition(url, ignore, metalake, catalog, schema, table, column, position) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.NULL)) { + boolean nullable = line.getOptionValue(GravitinoOptions.NULL).equals("true"); + gravitinoCommandLine + .newUpdateColumnNullability( + url, ignore, metalake, catalog, schema, table, column, nullable) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.AUTO)) { + boolean autoIncrement = line.getOptionValue(GravitinoOptions.AUTO).equals("true"); + gravitinoCommandLine + .newUpdateColumnAutoIncrement( + url, ignore, metalake, catalog, schema, table, column, autoIncrement) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.DEFAULT)) { + String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); + String dataType = line.getOptionValue(GravitinoOptions.DATATYPE); + gravitinoCommandLine + .newUpdateColumnDefault( + url, ignore, metalake, catalog, schema, table, column, defaultValue, dataType) + .validate() + .handle(); + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + gravitinoCommandLine + .newListColumns(url, ignore, metalake, catalog, schema, table) + .validate() + .handle(); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java new file mode 100644 index 00000000000..dce797294d8 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FilesetCommandHandler.java @@ -0,0 +1,210 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cli; + +import com.google.common.collect.Lists; +import java.util.List; +import java.util.Map; +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** + * Handles the command execution for Filesets based on command type and the command line options. + */ +public class FilesetCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private final String catalog; + private final String schema; + private String fileset; + + /** + * Constructs a {@link FilesetCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public FilesetCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = getUrl(line); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + this.catalog = name.getCatalogName(); + this.schema = name.getSchemaName(); + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(getAuth(line), userName); + + List missingEntities = Lists.newArrayList(); + if (catalog == null) missingEntities.add(CommandEntities.CATALOG); + if (schema == null) missingEntities.add(CommandEntities.SCHEMA); + + if (CommandActions.LIST.equals(command)) { + checkEntities(missingEntities); + handleListCommand(); + return; + } + + this.fileset = name.getFilesetName(); + if (fileset == null) missingEntities.add(CommandEntities.FILESET); + checkEntities(missingEntities); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_ACTION); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + case CommandActions.REMOVE: + handleRemoveCommand(); + return true; + + case CommandActions.PROPERTIES: + handlePropertiesCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + gravitinoCommandLine + .newFilesetDetails(url, ignore, metalake, catalog, schema, fileset) + .validate() + .handle(); + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + String[] properties = line.getOptionValues(CommandActions.PROPERTIES); + Map propertyMap = new Properties().parse(properties); + gravitinoCommandLine + .newCreateFileset(url, ignore, metalake, catalog, schema, fileset, comment, propertyMap) + .validate() + .handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean force = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine + .newDeleteFileset(url, ignore, force, metalake, catalog, schema, fileset) + .validate() + .handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + String value = line.getOptionValue(GravitinoOptions.VALUE); + gravitinoCommandLine + .newSetFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property, value) + .validate() + .handle(); + } + + /** Handles the "REMOVE" command. */ + private void handleRemoveCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + gravitinoCommandLine + .newRemoveFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property) + .validate() + .handle(); + } + + /** Handles the "PROPERTIES" command. */ + private void handlePropertiesCommand() { + gravitinoCommandLine + .newListFilesetProperties(url, ignore, metalake, catalog, schema, fileset) + .validate() + .handle(); + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + gravitinoCommandLine + .newListFilesets(url, ignore, metalake, catalog, schema) + .validate() + .handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + if (line.hasOption(GravitinoOptions.COMMENT)) { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateFilesetComment(url, ignore, metalake, catalog, schema, fileset, comment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + gravitinoCommandLine + .newUpdateFilesetName(url, ignore, metalake, catalog, schema, fileset, newName) + .validate() + .handle(); + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java index 7a9481cb95b..f6de0213ed6 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/FullName.java @@ -41,6 +41,20 @@ public FullName(CommandLine line) { this.line = line; } + /** + * Retrieves the level of the full name. + * + * @return The level of the full name, or -1 if line does not contain a {@code --name} option. + */ + public int getLevel() { + if (line.hasOption(GravitinoOptions.NAME)) { + String[] names = line.getOptionValue(GravitinoOptions.NAME).split("\\."); + return names.length; + } + + return -1; + } + /** * Retrieves the metalake name from the command line options, the GRAVITINO_METALAKE environment * variable or the Gravitino config file. diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java index 675a96d36a8..d7e257a8a81 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoCommandLine.java @@ -19,20 +19,14 @@ package org.apache.gravitino.cli; -import com.google.common.base.Joiner; -import com.google.common.collect.Lists; import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.List; -import java.util.Map; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Options; -import org.apache.gravitino.cli.commands.Command; /* Gravitino Command line */ public class GravitinoCommandLine extends TestableCommandLine { @@ -41,18 +35,13 @@ public class GravitinoCommandLine extends TestableCommandLine { private final Options options; private final String entity; private final String command; - private String urlEnv; - private boolean urlSet = false; private boolean ignore = false; private String ignoreEnv; private boolean ignoreSet = false; - private String authEnv; - private boolean authSet = false; public static final String CMD = "gcli"; // recommended name public static final String DEFAULT_URL = "http://localhost:8090"; // This joiner is used to join multiple outputs to be displayed, e.g. roles or groups - private static final Joiner COMMA_JOINER = Joiner.on(", ").skipNulls(); /** * Gravitino Command line. @@ -101,14 +90,8 @@ public void handleSimpleLine() { /* Display command usage. */ if (line.hasOption(GravitinoOptions.HELP)) { displayHelp(options); - } - /* Display Gravitino client version. */ - else if (line.hasOption(GravitinoOptions.VERSION)) { - newClientVersion(getUrl(), ignore).handle(); - } - /* Display Gravitino server version. */ - else if (line.hasOption(GravitinoOptions.SERVER)) { - newServerVersion(getUrl(), ignore).handle(); + } else { + new SimpleCommandHandler(this, line, ignore).handle(); } } @@ -127,9 +110,9 @@ private void executeCommand() { if (CommandActions.HELP.equals(command)) { handleHelpCommand(); } else if (line.hasOption(GravitinoOptions.OWNER)) { - handleOwnerCommand(); + new OwnerCommandHandler(this, line, command, ignore, entity).handle(); } else if (entity.equals(CommandEntities.COLUMN)) { - handleColumnCommand(); + new ColumnCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TABLE)) { new TableCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.SCHEMA)) { @@ -137,17 +120,17 @@ private void executeCommand() { } else if (entity.equals(CommandEntities.CATALOG)) { new CatalogCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.METALAKE)) { - handleMetalakeCommand(); + new MetalakeCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TOPIC)) { new TopicCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.FILESET)) { - handleFilesetCommand(); + new FilesetCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.USER)) { - handleUserCommand(); + new UserCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.GROUP)) { new GroupCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.TAG)) { - handleTagCommand(); + new TagCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.ROLE)) { new RoleCommandHandler(this, line, command, ignore).handle(); } else if (entity.equals(CommandEntities.MODEL)) { @@ -155,387 +138,6 @@ private void executeCommand() { } } - /** - * Handles the command execution for Metalakes based on command type and the command line options. - */ - private void handleMetalakeCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); - - Command.setAuthenticationMode(auth, userName); - - if (CommandActions.LIST.equals(command)) { - newListMetalakes(url, ignore, outputFormat).validate().handle(); - return; - } - - String metalake = name.getMetalakeName(); - - switch (command) { - case CommandActions.DETAILS: - if (line.hasOption(GravitinoOptions.AUDIT)) { - newMetalakeAudit(url, ignore, metalake).validate().handle(); - } else { - newMetalakeDetails(url, ignore, outputFormat, metalake).validate().handle(); - } - break; - - case CommandActions.CREATE: - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newCreateMetalake(url, ignore, metalake, comment).validate().handle(); - break; - - case CommandActions.DELETE: - boolean force = line.hasOption(GravitinoOptions.FORCE); - newDeleteMetalake(url, ignore, force, metalake).validate().handle(); - break; - - case CommandActions.SET: - String property = line.getOptionValue(GravitinoOptions.PROPERTY); - String value = line.getOptionValue(GravitinoOptions.VALUE); - newSetMetalakeProperty(url, ignore, metalake, property, value).validate().handle(); - break; - - case CommandActions.REMOVE: - property = line.getOptionValue(GravitinoOptions.PROPERTY); - newRemoveMetalakeProperty(url, ignore, metalake, property).validate().handle(); - break; - - case CommandActions.PROPERTIES: - newListMetalakeProperties(url, ignore, metalake).validate().handle(); - break; - - case CommandActions.UPDATE: - if (line.hasOption(GravitinoOptions.ENABLE) && line.hasOption(GravitinoOptions.DISABLE)) { - System.err.println(ErrorMessages.INVALID_ENABLE_DISABLE); - Main.exit(-1); - } - if (line.hasOption(GravitinoOptions.ENABLE)) { - boolean enableAllCatalogs = line.hasOption(GravitinoOptions.ALL); - newMetalakeEnable(url, ignore, metalake, enableAllCatalogs).validate().handle(); - } - if (line.hasOption(GravitinoOptions.DISABLE)) { - newMetalakeDisable(url, ignore, metalake).validate().handle(); - } - - if (line.hasOption(GravitinoOptions.COMMENT)) { - comment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateMetalakeComment(url, ignore, metalake, comment).validate().handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - force = line.hasOption(GravitinoOptions.FORCE); - newUpdateMetalakeName(url, ignore, force, metalake, newName).validate().handle(); - } - - break; - - default: - System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); - Main.exit(-1); - break; - } - } - - /** Handles the command execution for Users based on command type and the command line options. */ - protected void handleUserCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String user = line.getOptionValue(GravitinoOptions.USER); - - Command.setAuthenticationMode(auth, userName); - - if (user == null && !CommandActions.LIST.equals(command)) { - System.err.println(ErrorMessages.MISSING_USER); - Main.exit(-1); - } - - switch (command) { - case CommandActions.DETAILS: - if (line.hasOption(GravitinoOptions.AUDIT)) { - newUserAudit(url, ignore, metalake, user).validate().handle(); - } else { - newUserDetails(url, ignore, metalake, user).validate().handle(); - } - break; - - case CommandActions.LIST: - newListUsers(url, ignore, metalake).validate().handle(); - break; - - case CommandActions.CREATE: - newCreateUser(url, ignore, metalake, user).validate().handle(); - break; - - case CommandActions.DELETE: - boolean force = line.hasOption(GravitinoOptions.FORCE); - newDeleteUser(url, ignore, force, metalake, user).validate().handle(); - break; - - case CommandActions.REVOKE: - String[] revokeRoles = line.getOptionValues(GravitinoOptions.ROLE); - for (String role : revokeRoles) { - newRemoveRoleFromUser(url, ignore, metalake, user, role).validate().handle(); - } - System.out.printf("Remove roles %s from user %s%n", COMMA_JOINER.join(revokeRoles), user); - break; - - case CommandActions.GRANT: - String[] grantRoles = line.getOptionValues(GravitinoOptions.ROLE); - for (String role : grantRoles) { - newAddRoleToUser(url, ignore, metalake, user, role).validate().handle(); - } - System.out.printf("Grant roles %s to user %s%n", COMMA_JOINER.join(grantRoles), user); - break; - - default: - System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); - Main.exit(-1); - break; - } - } - - /** Handles the command execution for Tags based on command type and the command line options. */ - protected void handleTagCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - - Command.setAuthenticationMode(auth, userName); - - String[] tags = line.getOptionValues(GravitinoOptions.TAG); - - if (tags != null) { - tags = Arrays.stream(tags).distinct().toArray(String[]::new); - } - - switch (command) { - case CommandActions.DETAILS: - newTagDetails(url, ignore, metalake, getOneTag(tags)).validate().handle(); - break; - - case CommandActions.LIST: - if (!name.hasCatalogName()) { - newListTags(url, ignore, metalake).validate().handle(); - } else { - newListEntityTags(url, ignore, metalake, name).validate().handle(); - } - break; - - case CommandActions.CREATE: - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newCreateTags(url, ignore, metalake, tags, comment).validate().handle(); - break; - - case CommandActions.DELETE: - boolean forceDelete = line.hasOption(GravitinoOptions.FORCE); - newDeleteTag(url, ignore, forceDelete, metalake, tags).validate().handle(); - break; - - case CommandActions.SET: - String propertySet = line.getOptionValue(GravitinoOptions.PROPERTY); - String valueSet = line.getOptionValue(GravitinoOptions.VALUE); - if (propertySet == null && valueSet == null) { - newTagEntity(url, ignore, metalake, name, tags).validate().handle(); - } else { - newSetTagProperty(url, ignore, metalake, getOneTag(tags), propertySet, valueSet) - .validate() - .handle(); - } - break; - - case CommandActions.REMOVE: - boolean isTag = line.hasOption(GravitinoOptions.TAG); - if (!isTag) { - boolean forceRemove = line.hasOption(GravitinoOptions.FORCE); - newRemoveAllTags(url, ignore, metalake, name, forceRemove).validate().handle(); - } else { - String propertyRemove = line.getOptionValue(GravitinoOptions.PROPERTY); - if (propertyRemove != null) { - newRemoveTagProperty(url, ignore, metalake, getOneTag(tags), propertyRemove) - .validate() - .handle(); - } else { - newUntagEntity(url, ignore, metalake, name, tags).validate().handle(); - } - } - break; - - case CommandActions.PROPERTIES: - newListTagProperties(url, ignore, metalake, getOneTag(tags)).validate().handle(); - break; - - case CommandActions.UPDATE: - if (line.hasOption(GravitinoOptions.COMMENT)) { - String updateComment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateTagComment(url, ignore, metalake, getOneTag(tags), updateComment) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - newUpdateTagName(url, ignore, metalake, getOneTag(tags), newName).validate().handle(); - } - break; - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - Main.exit(-1); - break; - } - } - - private String getOneTag(String[] tags) { - if (tags == null || tags.length > 1) { - System.err.println(ErrorMessages.MULTIPLE_TAG_COMMAND_ERROR); - Main.exit(-1); - } - return tags[0]; - } - - /** - * Handles the command execution for Columns based on command type and the command line options. - */ - private void handleColumnCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - String table = name.getTableName(); - - Command.setAuthenticationMode(auth, userName); - - List missingEntities = Lists.newArrayList(); - if (catalog == null) missingEntities.add(CommandEntities.CATALOG); - if (schema == null) missingEntities.add(CommandEntities.SCHEMA); - if (table == null) missingEntities.add(CommandEntities.TABLE); - - if (CommandActions.LIST.equals(command)) { - checkEntities(missingEntities); - newListColumns(url, ignore, metalake, catalog, schema, table).validate().handle(); - return; - } - - String column = name.getColumnName(); - if (column == null) missingEntities.add(CommandEntities.COLUMN); - checkEntities(missingEntities); - - switch (command) { - case CommandActions.DETAILS: - if (line.hasOption(GravitinoOptions.AUDIT)) { - newColumnAudit(url, ignore, metalake, catalog, schema, table, column).validate().handle(); - } else { - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - Main.exit(-1); - } - break; - - case CommandActions.CREATE: - { - String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - String position = line.getOptionValue(GravitinoOptions.POSITION); - boolean nullable = - !line.hasOption(GravitinoOptions.NULL) - || line.getOptionValue(GravitinoOptions.NULL).equals("true"); - boolean autoIncrement = - line.hasOption(GravitinoOptions.AUTO) - && line.getOptionValue(GravitinoOptions.AUTO).equals("true"); - String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); - - newAddColumn( - url, - ignore, - metalake, - catalog, - schema, - table, - column, - datatype, - comment, - position, - nullable, - autoIncrement, - defaultValue) - .validate() - .handle(); - break; - } - - case CommandActions.DELETE: - newDeleteColumn(url, ignore, metalake, catalog, schema, table, column).validate().handle(); - break; - - case CommandActions.UPDATE: - { - if (line.hasOption(GravitinoOptions.COMMENT)) { - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateColumnComment(url, ignore, metalake, catalog, schema, table, column, comment) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - newUpdateColumnName(url, ignore, metalake, catalog, schema, table, column, newName) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.DATATYPE) - && !line.hasOption(GravitinoOptions.DEFAULT)) { - String datatype = line.getOptionValue(GravitinoOptions.DATATYPE); - newUpdateColumnDatatype(url, ignore, metalake, catalog, schema, table, column, datatype) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.POSITION)) { - String position = line.getOptionValue(GravitinoOptions.POSITION); - newUpdateColumnPosition(url, ignore, metalake, catalog, schema, table, column, position) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.NULL)) { - boolean nullable = line.getOptionValue(GravitinoOptions.NULL).equals("true"); - newUpdateColumnNullability( - url, ignore, metalake, catalog, schema, table, column, nullable) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.AUTO)) { - boolean autoIncrement = line.getOptionValue(GravitinoOptions.AUTO).equals("true"); - newUpdateColumnAutoIncrement( - url, ignore, metalake, catalog, schema, table, column, autoIncrement) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.DEFAULT)) { - String defaultValue = line.getOptionValue(GravitinoOptions.DEFAULT); - String dataType = line.getOptionValue(GravitinoOptions.DATATYPE); - newUpdateColumnDefault( - url, ignore, metalake, catalog, schema, table, column, defaultValue, dataType) - .validate() - .handle(); - } - break; - } - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - Main.exit(-1); - break; - } - } - private void handleHelpCommand() { String helpFile = entity.toLowerCase() + "_help.txt"; @@ -553,226 +155,4 @@ private void handleHelpCommand() { Main.exit(-1); } } - - /** - * Handles the command execution for Objects based on command type and the command line options. - */ - private void handleOwnerCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String entityName = line.getOptionValue(GravitinoOptions.NAME); - - Command.setAuthenticationMode(auth, userName); - - switch (command) { - case CommandActions.DETAILS: - newOwnerDetails(url, ignore, metalake, entityName, entity).handle(); - break; - - case CommandActions.SET: - { - String owner = line.getOptionValue(GravitinoOptions.USER); - String group = line.getOptionValue(GravitinoOptions.GROUP); - - if (owner != null && group == null) { - newSetOwner(url, ignore, metalake, entityName, entity, owner, false).handle(); - } else if (owner == null && group != null) { - newSetOwner(url, ignore, metalake, entityName, entity, group, true).handle(); - } else { - System.err.println(ErrorMessages.INVALID_SET_COMMAND); - } - break; - } - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - break; - } - } - - /** - * Handles the command execution for filesets based on command type and the command line options. - */ - private void handleFilesetCommand() { - String url = getUrl(); - String auth = getAuth(); - String userName = line.getOptionValue(GravitinoOptions.LOGIN); - FullName name = new FullName(line); - String metalake = name.getMetalakeName(); - String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - - Command.setAuthenticationMode(auth, userName); - - List missingEntities = Lists.newArrayList(); - if (catalog == null) missingEntities.add(CommandEntities.CATALOG); - if (schema == null) missingEntities.add(CommandEntities.SCHEMA); - - // Handle CommandActions.LIST action separately as it doesn't require the `fileset` - if (CommandActions.LIST.equals(command)) { - checkEntities(missingEntities); - newListFilesets(url, ignore, metalake, catalog, schema).validate().handle(); - return; - } - - String fileset = name.getFilesetName(); - if (fileset == null) missingEntities.add(CommandEntities.FILESET); - checkEntities(missingEntities); - - switch (command) { - case CommandActions.DETAILS: - newFilesetDetails(url, ignore, metalake, catalog, schema, fileset).validate().handle(); - break; - - case CommandActions.CREATE: - { - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - String[] properties = line.getOptionValues(CommandActions.PROPERTIES); - Map propertyMap = new Properties().parse(properties); - newCreateFileset(url, ignore, metalake, catalog, schema, fileset, comment, propertyMap) - .validate() - .handle(); - break; - } - - case CommandActions.DELETE: - { - boolean force = line.hasOption(GravitinoOptions.FORCE); - newDeleteFileset(url, ignore, force, metalake, catalog, schema, fileset) - .validate() - .handle(); - break; - } - - case CommandActions.SET: - { - String property = line.getOptionValue(GravitinoOptions.PROPERTY); - String value = line.getOptionValue(GravitinoOptions.VALUE); - newSetFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property, value) - .validate() - .handle(); - break; - } - - case CommandActions.REMOVE: - { - String property = line.getOptionValue(GravitinoOptions.PROPERTY); - newRemoveFilesetProperty(url, ignore, metalake, catalog, schema, fileset, property) - .validate() - .handle(); - break; - } - - case CommandActions.PROPERTIES: - newListFilesetProperties(url, ignore, metalake, catalog, schema, fileset) - .validate() - .handle(); - break; - - case CommandActions.UPDATE: - { - if (line.hasOption(GravitinoOptions.COMMENT)) { - String comment = line.getOptionValue(GravitinoOptions.COMMENT); - newUpdateFilesetComment(url, ignore, metalake, catalog, schema, fileset, comment) - .validate() - .handle(); - } - if (line.hasOption(GravitinoOptions.RENAME)) { - String newName = line.getOptionValue(GravitinoOptions.RENAME); - newUpdateFilesetName(url, ignore, metalake, catalog, schema, fileset, newName) - .validate() - .handle(); - } - break; - } - - default: - System.err.println(ErrorMessages.UNSUPPORTED_ACTION); - break; - } - } - - /** - * Retrieves the Gravitinno URL from the command line options or the GRAVITINO_URL environment - * variable or the Gravitio config file. - * - * @return The Gravitinno URL, or null if not found. - */ - public String getUrl() { - GravitinoConfig config = new GravitinoConfig(null); - - // If specified on the command line use that - if (line.hasOption(GravitinoOptions.URL)) { - return line.getOptionValue(GravitinoOptions.URL); - } - - // Cache the Gravitino URL environment variable - if (urlEnv == null && !urlSet) { - urlEnv = System.getenv("GRAVITINO_URL"); - urlSet = true; - } - - // If set return the Gravitino URL environment variable - if (urlEnv != null) { - return urlEnv; - } - - // Check if the Gravitino URL is specified in the configuration file - if (config.fileExists()) { - config.read(); - String configURL = config.getGravitinoURL(); - if (configURL != null) { - return configURL; - } - } - - // Return the default localhost URL - return DEFAULT_URL; - } - - /** - * Retrieves the Gravitinno authentication from the command line options or the GRAVITINO_AUTH - * environment variable or the Gravitio config file. - * - * @return The Gravitinno authentication, or null if not found. - */ - public String getAuth() { - // If specified on the command line use that - if (line.hasOption(GravitinoOptions.SIMPLE)) { - return GravitinoOptions.SIMPLE; - } - - // Cache the Gravitino authentication type environment variable - if (authEnv == null && !authSet) { - authEnv = System.getenv("GRAVITINO_AUTH"); - authSet = true; - } - - // If set return the Gravitino authentication type environment variable - if (authEnv != null) { - return authEnv; - } - - // Check if the authentication type is specified in the configuration file - GravitinoConfig config = new GravitinoConfig(null); - if (config.fileExists()) { - config.read(); - String configAuthType = config.getGravitinoAuthType(); - if (configAuthType != null) { - return configAuthType; - } - } - - return null; - } - - private void checkEntities(List entities) { - if (!entities.isEmpty()) { - System.err.println(ErrorMessages.MISSING_ENTITIES + COMMA_JOINER.join(entities)); - Main.exit(-1); - } - } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java index aaeb8f0184f..47f9914233d 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GravitinoOptions.java @@ -109,7 +109,7 @@ public Options options() { options.addOption(createArgOption(DEFAULT, "default column value")); options.addOption(createSimpleOption("o", OWNER, "display entity owner")); options.addOption(createArgOption(COLUMNFILE, "CSV file describing columns")); - options.addOption(createSimpleOption(null, ALL, "all operation for --enable")); + options.addOption(createSimpleOption(null, ALL, "on all entities")); // model options options.addOption(createArgOption(null, URI, "model version artifact")); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/GroupCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/GroupCommandHandler.java index e336003e6b5..7542b17974b 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/GroupCommandHandler.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/GroupCommandHandler.java @@ -130,14 +130,23 @@ private void handleDeleteCommand() { /** Handles the "REVOKE" command. */ private void handleRevokeCommand() { - String[] revokeRoles = line.getOptionValues(GravitinoOptions.ROLE); - for (String role : revokeRoles) { + boolean revokeAll = line.hasOption(GravitinoOptions.ALL); + if (revokeAll) { gravitinoCommandLine - .newRemoveRoleFromGroup(url, ignore, metalake, group, role) + .newRemoveAllRoles(url, ignore, metalake, group, CommandEntities.GROUP) .validate() .handle(); + System.out.printf("Removed all roles from group %s%n", group); + } else { + String[] revokeRoles = line.getOptionValues(GravitinoOptions.ROLE); + for (String role : revokeRoles) { + gravitinoCommandLine + .newRemoveRoleFromGroup(url, ignore, metalake, group, role) + .validate() + .handle(); + } + System.out.printf("Removed roles %s from group %s%n", COMMA_JOINER.join(revokeRoles), group); } - System.out.printf("Remove roles %s from group %s%n", COMMA_JOINER.join(revokeRoles), group); } /** Handles the "GRANT" command. */ diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java new file mode 100644 index 00000000000..993116f19f5 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/MetalakeCommandHandler.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** + * Handles the command execution for Metalakes based on command type and the command line options. + */ +public class MetalakeCommandHandler extends CommandHandler { + + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private String metalake; + + /** + * Constructs a MetalakeCommandHandler instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public MetalakeCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + this.url = getUrl(line); + } + + /** Handles the command execution logic based on the provided command. */ + public void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + FullName name = new FullName(line); + Command.setAuthenticationMode(getAuth(line), userName); + + if (CommandActions.LIST.equals(command)) { + handleListCommand(); + return; + } + + metalake = name.getMetalakeName(); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + case CommandActions.REMOVE: + handleRemoveCommand(); + return true; + + case CommandActions.PROPERTIES: + handlePropertiesCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); + gravitinoCommandLine.newListMetalakes(url, ignore, outputFormat).validate().handle(); + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + if (line.hasOption(GravitinoOptions.AUDIT)) { + gravitinoCommandLine.newMetalakeAudit(url, ignore, metalake).validate().handle(); + } else { + String outputFormat = line.getOptionValue(GravitinoOptions.OUTPUT); + gravitinoCommandLine + .newMetalakeDetails(url, ignore, outputFormat, metalake) + .validate() + .handle(); + } + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine.newCreateMetalake(url, ignore, metalake, comment).validate().handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean force = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine.newDeleteMetalake(url, ignore, force, metalake).validate().handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + String value = line.getOptionValue(GravitinoOptions.VALUE); + gravitinoCommandLine + .newSetMetalakeProperty(url, ignore, metalake, property, value) + .validate() + .handle(); + } + + /** Handles the "REMOVE" command. */ + private void handleRemoveCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + gravitinoCommandLine + .newRemoveMetalakeProperty(url, ignore, metalake, property) + .validate() + .handle(); + } + + /** Handles the "PROPERTIES" command. */ + private void handlePropertiesCommand() { + gravitinoCommandLine.newListMetalakeProperties(url, ignore, metalake).validate().handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + if (line.hasOption(GravitinoOptions.ENABLE) && line.hasOption(GravitinoOptions.DISABLE)) { + System.err.println(ErrorMessages.INVALID_ENABLE_DISABLE); + Main.exit(-1); + } + if (line.hasOption(GravitinoOptions.ENABLE)) { + boolean enableAllCatalogs = line.hasOption(GravitinoOptions.ALL); + gravitinoCommandLine + .newMetalakeEnable(url, ignore, metalake, enableAllCatalogs) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.DISABLE)) { + gravitinoCommandLine.newMetalakeDisable(url, ignore, metalake).validate().handle(); + } + + if (line.hasOption(GravitinoOptions.COMMENT)) { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateMetalakeComment(url, ignore, metalake, comment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + boolean force = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine + .newUpdateMetalakeName(url, ignore, force, metalake, newName) + .validate() + .handle(); + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java new file mode 100644 index 00000000000..7e41fb478ae --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/OwnerCommandHandler.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** Handles the command execution for Owner based on command type and the command line options. */ +public class OwnerCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private final String entityName; + private final String owner; + private final String group; + private final String entity; + + /** + * Constructs a {@link OwnerCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + * @param entity The entity to execute the command on. + */ + public OwnerCommandHandler( + GravitinoCommandLine gravitinoCommandLine, + CommandLine line, + String command, + boolean ignore, + String entity) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = getUrl(line); + this.owner = line.getOptionValue(GravitinoOptions.USER); + this.group = line.getOptionValue(GravitinoOptions.GROUP); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + this.entityName = name.getName(); + this.entity = entity; + } + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(getAuth(line), userName); + + if (entityName == null && !CommandEntities.METALAKE.equals(entity)) { + System.err.println(ErrorMessages.MISSING_NAME); + Main.exit(-1); + } + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + gravitinoCommandLine + .newOwnerDetails(url, ignore, metalake, entityName, entity) + .validate() + .handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + if (owner != null && group == null) { + gravitinoCommandLine + .newSetOwner(url, ignore, metalake, entityName, entity, owner, false) + .validate() + .handle(); + } else if (owner == null && group != null) { + gravitinoCommandLine + .newSetOwner(url, ignore, metalake, entityName, entity, group, true) + .validate() + .handle(); + } else { + System.err.println(ErrorMessages.INVALID_SET_COMMAND); + Main.exit(-1); + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/SimpleCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/SimpleCommandHandler.java new file mode 100644 index 00000000000..48aca9f9569 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/SimpleCommandHandler.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; + +/** Handles the command execution for simple command based on the command line options. */ +public class SimpleCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final boolean ignore; + + /** + * Constructs a {@link SimpleCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param ignore Ignore server version mismatch. + */ + public SimpleCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.ignore = ignore; + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + if (line.hasOption(GravitinoOptions.VERSION)) { + gravitinoCommandLine.newClientVersion(getUrl(line), ignore).validate().handle(); + } else if (line.hasOption(GravitinoOptions.SERVER)) { + gravitinoCommandLine.newServerVersion(getUrl(line), ignore).validate().handle(); + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java new file mode 100644 index 00000000000..e274c271f9c --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TagCommandHandler.java @@ -0,0 +1,207 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.cli; + +import java.util.Arrays; +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +public class TagCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private String[] tags; + private String metalake; + + public TagCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + this.url = getUrl(line); + this.tags = line.getOptionValues(GravitinoOptions.TAG); + + if (tags != null) { + tags = Arrays.stream(tags).distinct().toArray(String[]::new); + } + } + + @Override + public void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + FullName name = new FullName(line); + Command.setAuthenticationMode(getAuth(line), userName); + + metalake = name.getMetalakeName(); + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.LIST: + handleListCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.SET: + handleSetCommand(); + return true; + + case CommandActions.REMOVE: + handleRemoveCommand(); + return true; + + case CommandActions.PROPERTIES: + handlePropertiesCommand(); + return true; + + case CommandActions.UPDATE: + handleUpdateCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + FullName name = new FullName(line); + if (!name.hasCatalogName()) { + gravitinoCommandLine.newListTags(url, ignore, metalake).validate().handle(); + } else { + gravitinoCommandLine.newListEntityTags(url, ignore, metalake, name).validate().handle(); + } + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + gravitinoCommandLine.newTagDetails(url, ignore, metalake, getOneTag(tags)).validate().handle(); + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + String comment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine.newCreateTags(url, ignore, metalake, tags, comment).validate().handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean forceDelete = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine.newDeleteTag(url, ignore, forceDelete, metalake, tags).validate().handle(); + } + + /** Handles the "SET" command. */ + private void handleSetCommand() { + String property = line.getOptionValue(GravitinoOptions.PROPERTY); + String value = line.getOptionValue(GravitinoOptions.VALUE); + if (property == null && value == null) { + gravitinoCommandLine + .newTagEntity(url, ignore, metalake, new FullName(line), tags) + .validate() + .handle(); + } else { + gravitinoCommandLine + .newSetTagProperty(url, ignore, metalake, getOneTag(tags), property, value) + .validate() + .handle(); + } + } + + /** Handles the "REMOVE" command. */ + private void handleRemoveCommand() { + boolean isTag = line.hasOption(GravitinoOptions.TAG); + FullName name = new FullName(line); + if (!isTag) { + boolean forceRemove = line.hasOption(GravitinoOptions.FORCE); + gravitinoCommandLine + .newRemoveAllTags(url, ignore, metalake, name, forceRemove) + .validate() + .handle(); + } else { + String propertyRemove = line.getOptionValue(GravitinoOptions.PROPERTY); + if (propertyRemove != null) { + gravitinoCommandLine + .newRemoveTagProperty(url, ignore, metalake, getOneTag(tags), propertyRemove) + .validate() + .handle(); + } else { + gravitinoCommandLine.newUntagEntity(url, ignore, metalake, name, tags).validate().handle(); + } + } + } + + /** Handles the "PROPERTIES" command. */ + private void handlePropertiesCommand() { + gravitinoCommandLine + .newListTagProperties(url, ignore, metalake, getOneTag(tags)) + .validate() + .handle(); + } + + /** Handles the "UPDATE" command. */ + private void handleUpdateCommand() { + + if (line.hasOption(GravitinoOptions.COMMENT)) { + String updateComment = line.getOptionValue(GravitinoOptions.COMMENT); + gravitinoCommandLine + .newUpdateTagComment(url, ignore, metalake, getOneTag(tags), updateComment) + .validate() + .handle(); + } + if (line.hasOption(GravitinoOptions.RENAME)) { + String newName = line.getOptionValue(GravitinoOptions.RENAME); + gravitinoCommandLine + .newUpdateTagName(url, ignore, metalake, getOneTag(tags), newName) + .validate() + .handle(); + } + } + + private String getOneTag(String[] tags) { + if (tags == null || tags.length > 1) { + System.err.println(ErrorMessages.MULTIPLE_TAG_COMMAND_ERROR); + Main.exit(-1); + } + return tags[0]; + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java index d9e3b9288ee..498a0060cbd 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/TestableCommandLine.java @@ -86,6 +86,7 @@ import org.apache.gravitino.cli.commands.ModelDetails; import org.apache.gravitino.cli.commands.OwnerDetails; import org.apache.gravitino.cli.commands.RegisterModel; +import org.apache.gravitino.cli.commands.RemoveAllRoles; import org.apache.gravitino.cli.commands.RemoveAllTags; import org.apache.gravitino.cli.commands.RemoveCatalogProperty; import org.apache.gravitino.cli.commands.RemoveFilesetProperty; @@ -457,6 +458,11 @@ protected RemoveRoleFromGroup newRemoveRoleFromGroup( return new RemoveRoleFromGroup(url, ignore, metalake, group, role); } + protected RemoveAllRoles newRemoveAllRoles( + String url, boolean ignore, String metalake, String entity, String entityType) { + return new RemoveAllRoles(url, ignore, metalake, entity, entityType); + } + protected AddRoleToGroup newAddRoleToGroup( String url, boolean ignore, String metalake, String group, String role) { return new AddRoleToGroup(url, ignore, metalake, group, role); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java b/clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java new file mode 100644 index 00000000000..23698aa5cf8 --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/UserCommandHandler.java @@ -0,0 +1,183 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.gravitino.cli.commands.Command; + +/** Handles the command execution for Users based on command type and the command line options. */ +public class UserCommandHandler extends CommandHandler { + private final GravitinoCommandLine gravitinoCommandLine; + private final CommandLine line; + private final String command; + private final boolean ignore; + private final String url; + private final FullName name; + private final String metalake; + private String user; + + /** + * Constructs a {@link UserCommandHandler} instance. + * + * @param gravitinoCommandLine The Gravitino command line instance. + * @param line The command line arguments. + * @param command The command to execute. + * @param ignore Ignore server version mismatch. + */ + public UserCommandHandler( + GravitinoCommandLine gravitinoCommandLine, CommandLine line, String command, boolean ignore) { + this.gravitinoCommandLine = gravitinoCommandLine; + this.line = line; + this.command = command; + this.ignore = ignore; + + this.url = getUrl(line); + this.name = new FullName(line); + this.metalake = name.getMetalakeName(); + } + + /** Handles the command execution logic based on the provided command. */ + @Override + protected void handle() { + String userName = line.getOptionValue(GravitinoOptions.LOGIN); + Command.setAuthenticationMode(getAuth(line), userName); + + user = line.getOptionValue(GravitinoOptions.USER); + + if (user == null && !CommandActions.LIST.equals(command)) { + System.err.println(ErrorMessages.MISSING_USER); + Main.exit(-1); + } + + if (!executeCommand()) { + System.err.println(ErrorMessages.UNSUPPORTED_COMMAND); + Main.exit(-1); + } + } + + /** + * Executes the specific command based on the command type. + * + * @return true if the command is supported, false otherwise + */ + private boolean executeCommand() { + switch (command) { + case CommandActions.DETAILS: + handleDetailsCommand(); + return true; + + case CommandActions.LIST: + handleListCommand(); + return true; + + case CommandActions.CREATE: + handleCreateCommand(); + return true; + + case CommandActions.DELETE: + handleDeleteCommand(); + return true; + + case CommandActions.REVOKE: + handleRevokeCommand(); + return true; + + case CommandActions.GRANT: + handleGrantCommand(); + return true; + + default: + return false; + } + } + + /** Handles the "LIST" command. */ + private void handleListCommand() { + this.gravitinoCommandLine + .newListUsers(this.url, this.ignore, this.metalake) + .validate() + .handle(); + } + + /** Handles the "DETAILS" command. */ + private void handleDetailsCommand() { + if (line.hasOption(GravitinoOptions.AUDIT)) { + this.gravitinoCommandLine + .newUserAudit(this.url, this.ignore, this.metalake, user) + .validate() + .handle(); + } else { + this.gravitinoCommandLine + .newUserDetails(this.url, this.ignore, this.metalake, user) + .validate() + .handle(); + } + } + + /** Handles the "CREATE" command. */ + private void handleCreateCommand() { + this.gravitinoCommandLine + .newCreateUser(this.url, this.ignore, this.metalake, user) + .validate() + .handle(); + } + + /** Handles the "DELETE" command. */ + private void handleDeleteCommand() { + boolean force = line.hasOption(GravitinoOptions.FORCE); + this.gravitinoCommandLine + .newDeleteUser(this.url, this.ignore, force, this.metalake, user) + .validate() + .handle(); + } + + /** Handles the "REVOKE" command. */ + private void handleRevokeCommand() { + boolean removeAll = line.hasOption(GravitinoOptions.ALL); + if (removeAll) { + gravitinoCommandLine + .newRemoveAllRoles(url, ignore, metalake, user, CommandEntities.USER) + .validate() + .handle(); + System.out.printf("Removed all roles from user %s%n", user); + } else { + String[] revokeRoles = line.getOptionValues(GravitinoOptions.ROLE); + for (String role : revokeRoles) { + this.gravitinoCommandLine + .newRemoveRoleFromUser(this.url, this.ignore, this.metalake, user, role) + .validate() + .handle(); + } + System.out.printf("Removed roles %s from user %s%n", COMMA_JOINER.join(revokeRoles), user); + } + } + + /** Handles the "GRANT" command. */ + private void handleGrantCommand() { + String[] grantRoles = line.getOptionValues(GravitinoOptions.ROLE); + for (String role : grantRoles) { + this.gravitinoCommandLine + .newAddRoleToUser(this.url, this.ignore, this.metalake, user, role) + .validate() + .handle(); + } + System.out.printf("Add roles %s to user %s%n", COMMA_JOINER.join(grantRoles), user); + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java index 07d61dcaa7c..b76138cb5c9 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/AllMetalakeDetails.java @@ -55,6 +55,6 @@ public void handle() { String all = Joiner.on(System.lineSeparator()).join(metalakeDetails); - System.out.print(all.toString()); + System.out.print(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java index 7217d5ad3bd..58188c38a3f 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/GroupDetails.java @@ -62,6 +62,6 @@ public void handle() { String all = roles.isEmpty() ? "The group has no roles." : String.join(",", roles); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java index cded12808d9..e3bd42ae04c 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListAllTags.java @@ -55,6 +55,6 @@ public void handle() { String all = tags.length == 0 ? "No tags exist." : String.join(",", tags); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java index f3e8e0125cf..e1d8eac2086 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListColumns.java @@ -94,6 +94,6 @@ public void handle() { + System.lineSeparator()); } - System.out.print(all.toString()); + System.out.print(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java index a1c316fbdf2..90b1000fa05 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListEntityTags.java @@ -20,15 +20,18 @@ package org.apache.gravitino.cli.commands; import org.apache.gravitino.Catalog; -import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.cli.utils.FullNameUtil; import org.apache.gravitino.client.GravitinoClient; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchTableException; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.messaging.Topic; +import org.apache.gravitino.model.Model; import org.apache.gravitino.rel.Table; /* Lists all tags in a metalake. */ @@ -58,17 +61,34 @@ public void handle() { try { GravitinoClient client = buildClient(metalake); - // TODO fileset and topic - if (name.hasTableName()) { + if (name.getLevel() == 3) { String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - String table = name.getTableName(); - Table gTable = - client - .loadCatalog(catalog) - .asTableCatalog() - .loadTable(NameIdentifier.of(schema, table)); - tags = gTable.supportsTags().listTags(); + Catalog catalogObject = client.loadCatalog(catalog); + switch (catalogObject.type()) { + case RELATIONAL: + Table gTable = catalogObject.asTableCatalog().loadTable(FullNameUtil.toTable(name)); + tags = gTable.supportsTags().listTags(); + break; + + case MODEL: + Model gModel = catalogObject.asModelCatalog().getModel(FullNameUtil.toModel(name)); + tags = gModel.supportsTags().listTags(); + break; + + case FILESET: + Fileset fileset = + catalogObject.asFilesetCatalog().loadFileset(FullNameUtil.toFileset(name)); + tags = fileset.supportsTags().listTags(); + break; + + case MESSAGING: + Topic topic = catalogObject.asTopicCatalog().loadTopic(FullNameUtil.toTopic(name)); + tags = topic.supportsTags().listTags(); + break; + + default: + break; + } } else if (name.hasSchemaName()) { String catalog = name.getCatalogName(); String schema = name.getSchemaName(); @@ -93,6 +113,6 @@ public void handle() { String all = String.join(",", tags); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java index d00ba3e6ba5..cb46b7953c3 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListFilesets.java @@ -73,6 +73,6 @@ public void handle() { String all = filesets.length == 0 ? "No filesets exist." : Joiner.on(",").join(filesets); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java index a517b4daed8..844b8e21d09 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListGroups.java @@ -55,6 +55,6 @@ public void handle() { String all = groups.length == 0 ? "No groups exist." : String.join(",", groups); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListProperties.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListProperties.java index 56c8fb8ba78..a7d08ba36e2 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListProperties.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListProperties.java @@ -51,6 +51,6 @@ public void printProperties(Map properties) { all.append(property.getKey() + "," + property.getValue() + System.lineSeparator()); } - System.out.print(all.toString()); + System.out.print(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java index 2ecb35bd093..760fe21e3bc 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListRoles.java @@ -55,6 +55,6 @@ public void handle() { String all = roles.length == 0 ? "No roles exist." : String.join(",", roles); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java index 110a6477a62..695f44d42d7 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListSchema.java @@ -62,6 +62,6 @@ public void handle() { String all = schemas.length == 0 ? "No schemas exist." : Joiner.on(",").join(schemas); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java index 41a71e87c00..515fb28678c 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListTables.java @@ -66,6 +66,6 @@ public void handle() { ? "No tables exist." : Joiner.on(System.lineSeparator()).join(tableNames); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java index a70176dcfcb..3df4b7ca6bc 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/ListUsers.java @@ -53,8 +53,8 @@ public void handle() { exitWithError(exp.getMessage()); } - String all = String.join(",", users); + String all = users.length == 0 ? "No users exist." : String.join(",", users); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllRoles.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllRoles.java new file mode 100644 index 00000000000..eb7b354223d --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveAllRoles.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.commands; + +import java.util.List; +import org.apache.gravitino.authorization.Group; +import org.apache.gravitino.authorization.User; +import org.apache.gravitino.cli.CommandEntities; +import org.apache.gravitino.cli.ErrorMessages; +import org.apache.gravitino.client.GravitinoClient; +import org.apache.gravitino.exceptions.NoSuchGroupException; +import org.apache.gravitino.exceptions.NoSuchMetalakeException; +import org.apache.gravitino.exceptions.NoSuchUserException; + +/** Removes all roles from a group or user. */ +public class RemoveAllRoles extends Command { + protected final String metalake; + protected final String entity; + protected final String entityType; + + /** + * Removes all roles from a group or user. + * + * @param url The URL of the Gravitino server. + * @param ignoreVersions If true don't check the client/server versions match. + * @param metalake The name of the metalake. + * @param entity the name of the group or user. + * @param entityType The type of the entity (group or user). + */ + public RemoveAllRoles( + String url, boolean ignoreVersions, String metalake, String entity, String entityType) { + super(url, ignoreVersions); + this.metalake = metalake; + this.entity = entity; + this.entityType = entityType; + } + + /** Removes all roles from a group or user. */ + @Override + public void handle() { + if (CommandEntities.GROUP.equals(entityType)) { + revokeAllRolesFromGroup(); + } else { + revokeAllRolesFromUser(); + } + } + + /** Removes all roles from a group. */ + private void revokeAllRolesFromGroup() { + List roles; + try { + GravitinoClient client = buildClient(metalake); + Group group = client.getGroup(entity); + roles = group.roles(); + client.revokeRolesFromGroup(roles, entity); + } catch (NoSuchMetalakeException e) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (NoSuchGroupException e) { + exitWithError(ErrorMessages.UNKNOWN_GROUP); + } catch (Exception e) { + exitWithError(e.getMessage()); + } + } + + /** Removes all roles from a user. */ + private void revokeAllRolesFromUser() { + List roles; + try { + GravitinoClient client = buildClient(metalake); + User user = client.getUser(entity); + roles = user.roles(); + client.revokeRolesFromUser(roles, entity); + } catch (NoSuchMetalakeException e) { + exitWithError(ErrorMessages.UNKNOWN_METALAKE); + } catch (NoSuchUserException e) { + exitWithError(ErrorMessages.UNKNOWN_USER); + } catch (Exception e) { + exitWithError(e.getMessage()); + } + } +} diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java index dd7ccc0e79d..7f2f7e2b2c4 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/RemoveRoleFromGroup.java @@ -34,7 +34,7 @@ public class RemoveRoleFromGroup extends Command { protected String role; /** - * Removes a role from a group. + * Remove a role from a group. * * @param url The URL of the Gravitino server. * @param ignoreVersions If true don't check the client/server versions match. @@ -50,7 +50,7 @@ public RemoveRoleFromGroup( this.role = role; } - /** Adds a role to a group. */ + /** Remove a role from a group. */ @Override public void handle() { try { diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java index 4a06918850d..3b97778818e 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/TagEntity.java @@ -20,16 +20,19 @@ package org.apache.gravitino.cli.commands; import org.apache.gravitino.Catalog; -import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.cli.utils.FullNameUtil; import org.apache.gravitino.client.GravitinoClient; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchTableException; import org.apache.gravitino.exceptions.TagAlreadyAssociatedException; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.messaging.Topic; +import org.apache.gravitino.model.Model; import org.apache.gravitino.rel.Table; public class TagEntity extends Command { @@ -63,18 +66,42 @@ public void handle() { try { GravitinoClient client = buildClient(metalake); - // TODO fileset and topic - if (name.hasTableName()) { + if (name.getLevel() == 3) { String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - String table = name.getTableName(); - Table gTable = - client - .loadCatalog(catalog) - .asTableCatalog() - .loadTable(NameIdentifier.of(schema, table)); - tagsToAdd = gTable.supportsTags().associateTags(tags, null); - entity = table; + Catalog catalogObject = client.loadCatalog(catalog); + switch (catalogObject.type()) { + case RELATIONAL: + String table = name.getTableName(); + entity = table; + Table gTable = catalogObject.asTableCatalog().loadTable(FullNameUtil.toTable(name)); + tagsToAdd = gTable.supportsTags().associateTags(tags, null); + break; + + case MODEL: + String model = name.getModelName(); + entity = model; + Model gModel = catalogObject.asModelCatalog().getModel(FullNameUtil.toModel(name)); + tagsToAdd = gModel.supportsTags().associateTags(tags, null); + break; + + case FILESET: + String fileset = name.getFilesetName(); + entity = fileset; + Fileset gFileset = + catalogObject.asFilesetCatalog().loadFileset(FullNameUtil.toFileset(name)); + gFileset.supportsTags().associateTags(tags, null); + break; + + case MESSAGING: + String topic = name.getTopicName(); + entity = topic; + Topic gTopic = catalogObject.asTopicCatalog().loadTopic(FullNameUtil.toTopic(name)); + gTopic.supportsTags().associateTags(tags, null); + break; + + default: + break; + } } else if (name.hasSchemaName()) { String catalog = name.getCatalogName(); String schema = name.getSchemaName(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java index 3503d5eb7bf..205242135b4 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UntagEntity.java @@ -19,21 +19,22 @@ package org.apache.gravitino.cli.commands; -import com.google.common.base.Joiner; import org.apache.gravitino.Catalog; -import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Schema; import org.apache.gravitino.cli.ErrorMessages; import org.apache.gravitino.cli.FullName; +import org.apache.gravitino.cli.utils.FullNameUtil; import org.apache.gravitino.client.GravitinoClient; import org.apache.gravitino.exceptions.NoSuchCatalogException; import org.apache.gravitino.exceptions.NoSuchMetalakeException; import org.apache.gravitino.exceptions.NoSuchSchemaException; import org.apache.gravitino.exceptions.NoSuchTableException; +import org.apache.gravitino.file.Fileset; +import org.apache.gravitino.messaging.Topic; +import org.apache.gravitino.model.Model; import org.apache.gravitino.rel.Table; public class UntagEntity extends Command { - public static final Joiner COMMA_JOINER = Joiner.on(", ").skipNulls(); protected final String metalake; protected final FullName name; protected final String[] tags; @@ -64,18 +65,42 @@ public void handle() { try { GravitinoClient client = buildClient(metalake); - // TODO fileset and topic - if (name.hasTableName()) { + if (name.getLevel() == 3) { String catalog = name.getCatalogName(); - String schema = name.getSchemaName(); - String table = name.getTableName(); - Table gTable = - client - .loadCatalog(catalog) - .asTableCatalog() - .loadTable(NameIdentifier.of(schema, table)); - removeTags = gTable.supportsTags().associateTags(null, tags); - entity = table; + Catalog catalogObject = client.loadCatalog(catalog); + switch (catalogObject.type()) { + case RELATIONAL: + String table = name.getTableName(); + entity = table; + Table gTable = catalogObject.asTableCatalog().loadTable(FullNameUtil.toTable(name)); + removeTags = gTable.supportsTags().associateTags(null, tags); + break; + + case MODEL: + String model = name.getModelName(); + entity = model; + Model gModel = catalogObject.asModelCatalog().getModel(FullNameUtil.toModel(name)); + removeTags = gModel.supportsTags().associateTags(null, tags); + break; + + case FILESET: + String fileset = name.getFilesetName(); + entity = fileset; + Fileset gFileset = + catalogObject.asFilesetCatalog().loadFileset(FullNameUtil.toFileset(name)); + removeTags = gFileset.supportsTags().associateTags(null, tags); + break; + + case MESSAGING: + String topic = name.getTopicName(); + entity = topic; + Topic gTopic = catalogObject.asTopicCatalog().loadTopic(FullNameUtil.toTopic(name)); + removeTags = gTopic.supportsTags().associateTags(null, tags); + break; + + default: + break; + } } else if (name.hasSchemaName()) { String catalog = name.getCatalogName(); String schema = name.getSchemaName(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java index e37f8e6f139..d89597d1698 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/commands/UserDetails.java @@ -62,6 +62,6 @@ public void handle() { String all = roles.isEmpty() ? "The user has no roles." : String.join(",", roles); - System.out.println(all.toString()); + System.out.println(all); } } diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java b/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java index c9f502069b9..a3c99756524 100644 --- a/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/outputs/TableFormat.java @@ -111,10 +111,10 @@ static final class TableFormatImpl { Pattern.compile( "[\u1100-\u115F\u2E80-\uA4CF\uAC00-\uD7A3\uF900-\uFAFF\uFE10-\uFE19\uFE30-\uFE6F\uFF00-\uFF60\uFFE0-\uFFE6]"); private int[][] elementOutputWidths; - private final String horizontalDelimiter = "-"; - private final String verticalDelimiter = "|"; - private final String crossDelimiter = "+"; - private final String indent = " "; + private static final String horizontalDelimiter = "-"; + private static final String verticalDelimiter = "|"; + private static final String crossDelimiter = "+"; + private static final String indent = " "; public void debug() { System.out.println(); diff --git a/clients/cli/src/main/java/org/apache/gravitino/cli/utils/FullNameUtil.java b/clients/cli/src/main/java/org/apache/gravitino/cli/utils/FullNameUtil.java new file mode 100644 index 00000000000..6017b4246db --- /dev/null +++ b/clients/cli/src/main/java/org/apache/gravitino/cli/utils/FullNameUtil.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli.utils; + +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.cli.FullName; + +/** Utility class for helping with creating NameIdentifiers from {@link FullName}. */ +public class FullNameUtil { + + /** + * Returns a NameIdentifier for a model. + * + * @param fullName the {@link FullName} of the model. + * @return a NameIdentifier for the model. + */ + public static NameIdentifier toModel(FullName fullName) { + String schema = fullName.getSchemaName(); + String model = fullName.getModelName(); + + return NameIdentifier.of(schema, model); + } + + /** + * Returns a NameIdentifier for a table. + * + * @param fullName the {@link FullName} of the table. + * @return a NameIdentifier for the table. + */ + public static NameIdentifier toTable(FullName fullName) { + String schema = fullName.getSchemaName(); + String table = fullName.getTableName(); + + return NameIdentifier.of(schema, table); + } + + /** + * Returns a NameIdentifier for a fileset. + * + * @param fullName the {@link FullName} of the fileset. + * @return a NameIdentifier for the fileset. + */ + public static NameIdentifier toFileset(FullName fullName) { + String schema = fullName.getSchemaName(); + String fileset = fullName.getFilesetName(); + + return NameIdentifier.of(schema, fileset); + } + + /** + * Returns a NameIdentifier for a topic. + * + * @param fullName the {@link FullName} of the topic. + * @return a NameIdentifier for the topic. + */ + public static NameIdentifier toTopic(FullName fullName) { + String schema = fullName.getSchemaName(); + String topic = fullName.getTopicName(); + + return NameIdentifier.of(schema, topic); + } +} diff --git a/clients/cli/src/main/resources/group_help.txt b/clients/cli/src/main/resources/group_help.txt index cbac95747af..a5badc82265 100644 --- a/clients/cli/src/main/resources/group_help.txt +++ b/clients/cli/src/main/resources/group_help.txt @@ -18,3 +18,9 @@ gcli group details --group new_group --audit Delete a group gcli group delete --group new_group + +Revoke a role from a group +gcli group revoke --group new_group --role admin + +Revoke all roles from a group +gcli group revoke --group new_group --all \ No newline at end of file diff --git a/clients/cli/src/main/resources/user_help.txt b/clients/cli/src/main/resources/user_help.txt index bd08bea59fb..00add299d27 100644 --- a/clients/cli/src/main/resources/user_help.txt +++ b/clients/cli/src/main/resources/user_help.txt @@ -19,3 +19,8 @@ gcli user details --user new_user --audit Delete a user gcli user delete --user new_user +Revoke a user's role +gcli user revoke --user new_user --role admin + +Revoke all roles from a user +gcli user revoke --user new_user --all \ No newline at end of file diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFullNameUtil.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFullNameUtil.java new file mode 100644 index 00000000000..8123bb33420 --- /dev/null +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFullNameUtil.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; +import org.apache.gravitino.NameIdentifier; +import org.apache.gravitino.cli.utils.FullNameUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestFullNameUtil { + private Options options; + + @BeforeEach + public void setUp() { + Main.useExit = false; + options = new GravitinoOptions().options(); + } + + @Test + void testToModel() throws ParseException { + String[] args = {"table", "list", "-i", "--name", "Model_catalog.schema.model"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + NameIdentifier modelIdent = FullNameUtil.toModel(fullName); + Assertions.assertEquals("model", modelIdent.name()); + Assertions.assertArrayEquals(new String[] {"schema"}, modelIdent.namespace().levels()); + } + + @Test + void testToTable() throws ParseException { + String[] args = {"table", "list", "-i", "--name", "Table_catalog.schema.table"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + NameIdentifier tableIdent = FullNameUtil.toTable(fullName); + Assertions.assertEquals("table", tableIdent.name()); + Assertions.assertArrayEquals(new String[] {"schema"}, tableIdent.namespace().levels()); + } + + @Test + void testToFileset() throws ParseException { + String[] args = {"fileset", "list", "-i", "--name", "Fileset_catalog.schema.fileset"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + NameIdentifier filesetIdent = FullNameUtil.toFileset(fullName); + Assertions.assertEquals("fileset", filesetIdent.name()); + Assertions.assertArrayEquals(new String[] {"schema"}, filesetIdent.namespace().levels()); + } + + @Test + void testToTopic() throws ParseException { + String[] args = {"topic", "list", "-i", "--name", "Topic_catalog.schema.topic"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + NameIdentifier topicIdent = FullNameUtil.toTopic(fullName); + Assertions.assertEquals("topic", topicIdent.name()); + Assertions.assertArrayEquals(new String[] {"schema"}, topicIdent.namespace().levels()); + } +} diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java index f13d6e09201..c2e54c81d5e 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestFulllName.java @@ -235,4 +235,40 @@ public void testGetMetalakeWithoutMetalakeOption() throws ParseException { String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); assertEquals(errOutput, ErrorMessages.MISSING_METALAKE); } + + @Test + @SuppressWarnings("DefaultCharset") + void testGetLevelFromCatalog() throws ParseException { + String[] args = {"table", "list", "-i", "--name", "Hive_catalog"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + assertEquals(1, fullName.getLevel()); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testGetLevelFromSchema() throws ParseException { + String[] args = {"table", "list", "-i", "--name", "Hive_catalog.default"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + assertEquals(2, fullName.getLevel()); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testGetLevelFromTable() throws ParseException { + String[] args = {"table", "list", "-i", "--name", "Hive_catalog.default.sales"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + assertEquals(3, fullName.getLevel()); + } + + @Test + @SuppressWarnings("DefaultCharset") + void testGetLevelFromColumn() throws ParseException { + String[] args = {"table", "list", "-i", "--name", "Hive_catalog.default.sales.columns"}; + CommandLine commandLine = new DefaultParser().parse(options, args); + FullName fullName = new FullName(commandLine); + assertEquals(4, fullName.getLevel()); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java index ce7a8956821..cae8402d8bb 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestGroupCommands.java @@ -39,6 +39,7 @@ import org.apache.gravitino.cli.commands.GroupAudit; import org.apache.gravitino.cli.commands.GroupDetails; import org.apache.gravitino.cli.commands.ListGroups; +import org.apache.gravitino.cli.commands.RemoveAllRoles; import org.apache.gravitino.cli.commands.RemoveRoleFromGroup; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -260,6 +261,32 @@ void testRemoveRolesFromGroupCommand() { verify(mockRemoveSecondRole).handle(); } + @Test + void testRemoveAllRolesFromGroupCommand() { + RemoveAllRoles mock = mock(RemoveAllRoles.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.GROUP)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.GROUP)).thenReturn("groupA"); + when(mockCommandLine.hasOption(GravitinoOptions.ALL)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.GROUP, CommandActions.REVOKE)); + + doReturn(mock) + .when(commandLine) + .newRemoveAllRoles( + GravitinoCommandLine.DEFAULT_URL, + false, + "metalake_demo", + "groupA", + CommandEntities.GROUP); + doReturn(mock).when(mock).validate(); + commandLine.handleCommandLine(); + verify(mock).handle(); + } + @Test void testAddRolesToGroupCommand() { AddRoleToGroup mockAddFirstRole = mock(AddRoleToGroup.class); diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java index 0c2b2cf91e5..12f617380ca 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestOwnerCommands.java @@ -19,16 +19,25 @@ package org.apache.gravitino.cli; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; import org.apache.gravitino.cli.commands.OwnerDetails; import org.apache.gravitino.cli.commands.SetOwner; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -36,10 +45,23 @@ class TestOwnerCommands { private CommandLine mockCommandLine; private Options mockOptions; + private final ByteArrayOutputStream outContent = new ByteArrayOutputStream(); + private final ByteArrayOutputStream errContent = new ByteArrayOutputStream(); + private final PrintStream originalOut = System.out; + private final PrintStream originalErr = System.err; + @BeforeEach void setUp() { mockCommandLine = mock(CommandLine.class); mockOptions = mock(Options.class); + System.setOut(new PrintStream(outContent)); + System.setErr(new PrintStream(errContent)); + } + + @AfterEach + public void restoreStreams() { + System.setOut(originalOut); + System.setErr(originalErr); } @Test @@ -67,6 +89,7 @@ void testSetOwnerUserCommand() { "catalog", "admin", false); + doReturn(mockSetOwner).when(mockSetOwner).validate(); commandLine.handleCommandLine(); verify(mockSetOwner).handle(); } @@ -96,6 +119,7 @@ void testSetOwnerGroupCommand() { "catalog", "ITdept", true); + doReturn(mockSetOwner).when(mockSetOwner).validate(); commandLine.handleCommandLine(); verify(mockSetOwner).handle(); } @@ -116,7 +140,62 @@ void testOwnerDetailsCommand() { .when(commandLine) .newOwnerDetails( GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "postgres", "catalog"); + doReturn(mockOwnerDetails).when(mockOwnerDetails).validate(); commandLine.handleCommandLine(); verify(mockOwnerDetails).handle(); } + + @Test + void testOwnerDetailsCommandWithoutName() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.OWNER)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.DETAILS)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newOwnerDetails( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq(null), + eq(CommandEntities.CATALOG)); + + String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(ErrorMessages.MISSING_NAME, errOutput); + } + + @Test + void testSetOwnerUserCommandWithoutUserAndGroup() { + Main.useExit = false; + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.NAME)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.NAME)).thenReturn("postgres"); + when(mockCommandLine.hasOption(GravitinoOptions.USER)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.GROUP)).thenReturn(false); + when(mockCommandLine.hasOption(GravitinoOptions.OWNER)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.CATALOG, CommandActions.SET)); + + assertThrows(RuntimeException.class, commandLine::handleCommandLine); + verify(commandLine, never()) + .newSetOwner( + eq(GravitinoCommandLine.DEFAULT_URL), + eq(false), + eq("metalake_demo"), + eq("postgres"), + eq(CommandEntities.CATALOG), + isNull(), + eq(false)); + String errOutput = new String(errContent.toByteArray(), StandardCharsets.UTF_8).trim(); + assertEquals(ErrorMessages.INVALID_SET_COMMAND, errOutput); + } } diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestSimpleCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestSimpleCommands.java new file mode 100644 index 00000000000..044e06c58f7 --- /dev/null +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestSimpleCommands.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.gravitino.cli; + +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.Options; +import org.apache.gravitino.cli.commands.ClientVersion; +import org.apache.gravitino.cli.commands.ServerVersion; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class TestSimpleCommands { + + private CommandLine mockCommandLine; + private Options mockOptions; + + @BeforeEach + void setUp() { + mockCommandLine = mock(CommandLine.class); + mockOptions = mock(Options.class); + } + + @Test + void testServerVersion() { + ServerVersion mockServerVersion = mock(ServerVersion.class); + when(mockCommandLine.hasOption(GravitinoOptions.SERVER)).thenReturn(true); + GravitinoCommandLine commandLine = + spy(new GravitinoCommandLine(mockCommandLine, mockOptions, null, null)); + + doReturn(mockServerVersion) + .when(commandLine) + .newServerVersion(GravitinoCommandLine.DEFAULT_URL, false); + doReturn(mockServerVersion).when(mockServerVersion).validate(); + commandLine.handleSimpleLine(); + verify(mockServerVersion).handle(); + } + + @Test + void testClientVersion() { + ClientVersion mockClientVersion = mock(ClientVersion.class); + when(mockCommandLine.hasOption(GravitinoOptions.VERSION)).thenReturn(true); + GravitinoCommandLine commandLine = + spy(new GravitinoCommandLine(mockCommandLine, mockOptions, null, null)); + + doReturn(mockClientVersion) + .when(commandLine) + .newClientVersion(GravitinoCommandLine.DEFAULT_URL, false); + doReturn(mockClientVersion).when(mockClientVersion).validate(); + commandLine.handleSimpleLine(); + verify(mockClientVersion).handle(); + } +} diff --git a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java index c7612f6c870..47f003c96b4 100644 --- a/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java +++ b/clients/cli/src/test/java/org/apache/gravitino/cli/TestUserCommands.java @@ -37,6 +37,7 @@ import org.apache.gravitino.cli.commands.CreateUser; import org.apache.gravitino.cli.commands.DeleteUser; import org.apache.gravitino.cli.commands.ListUsers; +import org.apache.gravitino.cli.commands.RemoveAllRoles; import org.apache.gravitino.cli.commands.RemoveRoleFromUser; import org.apache.gravitino.cli.commands.UserAudit; import org.apache.gravitino.cli.commands.UserDetails; @@ -261,6 +262,27 @@ void testRemoveRolesFromUserCommand() { verify(mockRemoveFirstRole).handle(); } + @Test + void removeAllRolesFromUserCommand() { + RemoveAllRoles mockRemove = mock(RemoveAllRoles.class); + when(mockCommandLine.hasOption(GravitinoOptions.METALAKE)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.METALAKE)).thenReturn("metalake_demo"); + when(mockCommandLine.hasOption(GravitinoOptions.USER)).thenReturn(true); + when(mockCommandLine.getOptionValue(GravitinoOptions.USER)).thenReturn("user"); + when(mockCommandLine.hasOption(GravitinoOptions.ALL)).thenReturn(true); + GravitinoCommandLine commandLine = + spy( + new GravitinoCommandLine( + mockCommandLine, mockOptions, CommandEntities.USER, CommandActions.REVOKE)); + doReturn(mockRemove) + .when(commandLine) + .newRemoveAllRoles( + GravitinoCommandLine.DEFAULT_URL, false, "metalake_demo", "user", CommandEntities.USER); + doReturn(mockRemove).when(mockRemove).validate(); + commandLine.handleCommandLine(); + verify(mockRemove).handle(); + } + @Test void testAddRolesToUserCommand() { AddRoleToUser mockAddFirstRole = mock(AddRoleToUser.class); diff --git a/clients/client-python/LICENSE b/clients/client-python/LICENSE new file mode 100644 index 00000000000..42c856d10b1 --- /dev/null +++ b/clients/client-python/LICENSE @@ -0,0 +1,214 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + The Web UI also bundles various third-party components also under + different licenses, please see web/LICENSE for these. + + This product bundles various third-party components also under the + Apache Software License 2.0. + + This product bundles a third-party component under the + MIT License. + + Kyligence/kylinpy + ./client-python/gravitino/utils/http_client.py + diff --git a/clients/client-python/NOTICE b/clients/client-python/NOTICE new file mode 100644 index 00000000000..c1fde5e04e3 --- /dev/null +++ b/clients/client-python/NOTICE @@ -0,0 +1,8 @@ +Apache Gravitino (incubating) +Copyright 2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The initial code for the Gravitino project was donated +to the ASF by Datastrato (https://datastrato.ai/) copyright 2023-2024. \ No newline at end of file diff --git a/clients/client-python/build.gradle.kts b/clients/client-python/build.gradle.kts index bebf536f6eb..af6cfcd2d9f 100644 --- a/clients/client-python/build.gradle.kts +++ b/clients/client-python/build.gradle.kts @@ -285,9 +285,6 @@ tasks { generatePypiProjectHomePage() delete("dist") copy { - from("${project.rootDir}/licenses") { into("licenses") } - from("${project.rootDir}/LICENSE.bin") { into("./") } - from("${project.rootDir}/NOTICE.bin") { into("./") } from("${project.rootDir}/DISCLAIMER_WIP.txt") { into("./") } into("${project.rootDir}/clients/client-python") rename { fileName -> @@ -301,9 +298,6 @@ tasks { doLast { delete("README.md") - delete("licenses") - delete("LICENSE") - delete("NOTICE") delete("DISCLAIMER_WIP.txt") } } diff --git a/clients/client-python/gravitino/filesystem/gvfs_config.py b/clients/client-python/gravitino/filesystem/gvfs_config.py index 6fbd8a99d18..34db72adee0 100644 --- a/clients/client-python/gravitino/filesystem/gvfs_config.py +++ b/clients/client-python/gravitino/filesystem/gvfs_config.py @@ -42,8 +42,8 @@ class GVFSConfig: GVFS_FILESYSTEM_OSS_SECRET_KEY = "oss_secret_access_key" GVFS_FILESYSTEM_OSS_ENDPOINT = "oss_endpoint" - GVFS_FILESYSTEM_AZURE_ACCOUNT_NAME = "abs_account_name" - GVFS_FILESYSTEM_AZURE_ACCOUNT_KEY = "abs_account_key" + GVFS_FILESYSTEM_AZURE_ACCOUNT_NAME = "azure_storage_account_name" + GVFS_FILESYSTEM_AZURE_ACCOUNT_KEY = "azure_storage_account_key" # This configuration marks the expired time of the credential. For instance, if the credential # fetched from Gravitino server has expired time of 3600 seconds, and the credential_expired_time_ration is 0.5 diff --git a/clients/client-python/licenses/kylinpy.txt b/clients/client-python/licenses/kylinpy.txt new file mode 100644 index 00000000000..580127c7327 --- /dev/null +++ b/clients/client-python/licenses/kylinpy.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Dhamu + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/clients/client-python/setup.py b/clients/client-python/setup.py index 108ad0226f1..878e74a1d00 100644 --- a/clients/client-python/setup.py +++ b/clients/client-python/setup.py @@ -27,7 +27,7 @@ setup( name="apache-gravitino", description="Python lib/client for Apache Gravitino", - version="0.8.0.dev0", + version="0.9.0.dev0", long_description=long_description, long_description_content_type="text/markdown", author="Apache Software Foundation", diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile index f4a4cef20ae..86dd2f22152 100644 --- a/clients/filesystem-fuse/Makefile +++ b/clients/filesystem-fuse/Makefile @@ -62,6 +62,12 @@ doc-test: unit-test: doc-test cargo test --no-fail-fast --lib --all-features --workspace +test-fuse-it: + @bash ./tests/bin/run_fuse_testers.sh test + +test-s3: + @bash ./tests/bin/run_s3fs_testers.sh test + test: doc-test cargo test --no-fail-fast --all-targets --all-features --workspace diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 944181246d5..d1d8e7605df 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -334,13 +334,22 @@ impl RawFileSystem for DefaultRawFileSystem { file.flush().await } - async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { + async fn close_file(&self, file_id: u64, fh: u64) -> Result<()> { + let file_entry = self.get_file_entry(file_id).await; + let opened_file = self .opened_file_manager .remove(fh) .ok_or(Errno::from(libc::EBADF))?; - let mut file = opened_file.lock().await; - file.close().await + + // todo: need to handle racing condition and corner case when the file has been deleted. + if file_entry.is_ok() { + let mut file = opened_file.lock().await; + file.close().await + } else { + // If the file has been deleted, it does not cause a leak even if it has not been closed. + Ok(()) + } } async fn read(&self, file_id: u64, fh: u64, offset: u64, size: u32) -> Result { diff --git a/clients/filesystem-fuse/src/gravitino_client.rs b/clients/filesystem-fuse/src/gravitino_client.rs index 9bdfbb2c288..1e1cd411eac 100644 --- a/clients/filesystem-fuse/src/gravitino_client.rs +++ b/clients/filesystem-fuse/src/gravitino_client.rs @@ -199,10 +199,34 @@ impl GravitinoClient { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use mockito::mock; + pub(crate) fn create_test_catalog( + name: &str, + provider: &str, + properties: HashMap, + ) -> Catalog { + Catalog { + name: name.to_string(), + catalog_type: "fileset".to_string(), + provider: provider.to_string(), + comment: "".to_string(), + properties: properties, + } + } + + pub(crate) fn create_test_fileset(name: &str, storage_location: &str) -> Fileset { + Fileset { + name: name.to_string(), + fileset_type: "managed".to_string(), + comment: "".to_string(), + storage_location: storage_location.to_string(), + properties: HashMap::default(), + } + } + #[tokio::test] async fn test_get_fileset_success() { let fileset_response = r#" diff --git a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs index 7da2f572dcc..04236dfe841 100644 --- a/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs +++ b/clients/filesystem-fuse/src/gravitino_fileset_filesystem.rs @@ -140,16 +140,27 @@ impl PathFileSystem for GravitinoFilesetFileSystem { #[cfg(test)] mod tests { - use crate::config::GravitinoConfig; + use crate::config::{AppConfig, GravitinoConfig}; + use crate::default_raw_filesystem::DefaultRawFileSystem; + use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem}; + use crate::filesystem::{FileSystemContext, PathFileSystem, RawFileSystem}; + use crate::gravitino_client::tests::{create_test_catalog, create_test_fileset}; + use crate::gravitino_client::GravitinoClient; use crate::gravitino_fileset_filesystem::GravitinoFilesetFileSystem; + use crate::gvfs_creator::create_fs_with_fileset; use crate::memory_filesystem::MemoryFileSystem; + use crate::s3_filesystem::extract_s3_config; + use crate::s3_filesystem::tests::{cleanup_s3_fs, s3_test_config}; + use crate::test_enable_with; + use crate::RUN_TEST_WITH_S3; + use std::collections::HashMap; use std::path::Path; #[tokio::test] async fn test_map_fileset_path_to_raw_path() { let fs = GravitinoFilesetFileSystem { physical_fs: Box::new(MemoryFileSystem::new().await), - client: super::GravitinoClient::new(&GravitinoConfig::default()), + client: GravitinoClient::new(&GravitinoConfig::default()), location: "/c1/fileset1".into(), }; let path = fs.gvfs_path_to_raw_path(Path::new("/a")); @@ -162,7 +173,7 @@ mod tests { async fn test_map_raw_path_to_fileset_path() { let fs = GravitinoFilesetFileSystem { physical_fs: Box::new(MemoryFileSystem::new().await), - client: super::GravitinoClient::new(&GravitinoConfig::default()), + client: GravitinoClient::new(&GravitinoConfig::default()), location: "/c1/fileset1".into(), }; let path = fs @@ -172,4 +183,68 @@ mod tests { let path = fs.raw_path_to_gvfs_path(Path::new("/c1/fileset1")).unwrap(); assert_eq!(path, Path::new("/")); } + + async fn create_fileset_fs(path: &Path, config: &AppConfig) -> GravitinoFilesetFileSystem { + let opendal_config = extract_s3_config(config); + + cleanup_s3_fs(path, &opendal_config).await; + + let bucket = opendal_config.get("bucket").expect("Bucket must exist"); + let endpoint = opendal_config.get("endpoint").expect("Endpoint must exist"); + + let catalog = create_test_catalog( + "c1", + "s3", + vec![ + ("location".to_string(), format!("s3a://{}", bucket)), + ("s3-endpoint".to_string(), endpoint.to_string()), + ] + .into_iter() + .collect::>(), + ); + let file_set_location = format!("s3a://{}{}", bucket, path.to_string_lossy()); + let file_set = create_test_fileset("fileset1", &file_set_location); + + let fs_context = FileSystemContext::default(); + let inner_fs = create_fs_with_fileset(&catalog, &file_set, config, &fs_context) + .await + .unwrap(); + GravitinoFilesetFileSystem::new( + inner_fs, + path, + GravitinoClient::new(&config.gravitino), + config, + &fs_context, + ) + .await + } + + #[tokio::test] + async fn s3_ut_test_fileset_file_system() { + test_enable_with!(RUN_TEST_WITH_S3); + + let config = s3_test_config(); + let cwd = Path::new("/gvfs_test3"); + let fs = create_fileset_fs(cwd, &config).await; + let _ = fs.init().await; + let mut tester = TestPathFileSystem::new(Path::new("/"), fs); + tester.test_path_file_system().await; + } + + #[tokio::test] + async fn s3_ut_test_fileset_with_raw_file_system() { + test_enable_with!(RUN_TEST_WITH_S3); + + let config = s3_test_config(); + let cwd = Path::new("/gvfs_test4"); + let fileset_fs = create_fileset_fs(cwd, &config).await; + let raw_fs = DefaultRawFileSystem::new( + fileset_fs, + &AppConfig::default(), + &FileSystemContext::default(), + ); + let _ = raw_fs.init().await; + let mut tester = TestRawFileSystem::new(Path::new("/"), raw_fs); + tester.test_raw_file_system().await; + } } diff --git a/clients/filesystem-fuse/src/gvfs_creator.rs b/clients/filesystem-fuse/src/gvfs_creator.rs index aac88ad9d08..88bc8a1b422 100644 --- a/clients/filesystem-fuse/src/gvfs_creator.rs +++ b/clients/filesystem-fuse/src/gvfs_creator.rs @@ -87,7 +87,7 @@ pub async fn create_gvfs_filesystem( .get_fileset(&catalog_name, &schema_name, &fileset_name) .await?; - let inner_fs = create_fs_with_fileset(&catalog, &fileset, config, fs_context)?; + let inner_fs = create_fs_with_fileset(&catalog, &fileset, config, fs_context).await?; let target_path = extract_root_path(fileset.storage_location.as_str())?; let fs = @@ -95,7 +95,7 @@ pub async fn create_gvfs_filesystem( Ok(CreateFileSystemResult::Gvfs(fs)) } -fn create_fs_with_fileset( +pub(crate) async fn create_fs_with_fileset( catalog: &Catalog, fileset: &Fileset, config: &AppConfig, @@ -104,9 +104,9 @@ fn create_fs_with_fileset( let schema = extract_filesystem_scheme(&fileset.storage_location)?; match schema { - FileSystemSchema::S3 => Ok(Box::new(S3FileSystem::new( - catalog, fileset, config, fs_context, - )?)), + FileSystemSchema::S3 => Ok(Box::new( + S3FileSystem::new(catalog, fileset, config, fs_context).await?, + )), } } diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index 31e7c7fd8e1..41a9a5335d5 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -36,6 +36,19 @@ mod opened_file_manager; mod s3_filesystem; mod utils; +#[macro_export] +macro_rules! test_enable_with { + ($env_var:expr) => { + if std::env::var($env_var).is_err() { + println!("Test skipped because {} is not set", $env_var); + return; + } + }; +} + +pub const RUN_TEST_WITH_S3: &str = "RUN_TEST_WITH_S3"; +pub const RUN_TEST_WITH_FUSE: &str = "RUN_TEST_WITH_FUSE"; + pub async fn gvfs_mount(mount_to: &str, mount_from: &str, config: &AppConfig) -> GvfsResult<()> { gvfs_fuse::mount(mount_to, mount_from, config).await } diff --git a/clients/filesystem-fuse/src/open_dal_filesystem.rs b/clients/filesystem-fuse/src/open_dal_filesystem.rs index e53fbaf6032..d32b014d1f0 100644 --- a/clients/filesystem-fuse/src/open_dal_filesystem.rs +++ b/clients/filesystem-fuse/src/open_dal_filesystem.rs @@ -261,22 +261,29 @@ fn opendal_filemode_to_filetype(mode: EntryMode) -> FileType { mod test { use crate::config::AppConfig; use crate::s3_filesystem::extract_s3_config; + use crate::s3_filesystem::tests::s3_test_config; + use crate::test_enable_with; + use crate::RUN_TEST_WITH_S3; use opendal::layers::LoggingLayer; use opendal::{services, Builder, Operator}; - #[tokio::test] - async fn test_s3_stat() { - let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap(); - let opendal_config = extract_s3_config(&config); - + fn create_opendal(config: &AppConfig) -> Operator { + let opendal_config = extract_s3_config(config); let builder = services::S3::from_map(opendal_config); // Init an operator - let op = Operator::new(builder) + Operator::new(builder) .expect("opendal create failed") .layer(LoggingLayer::default()) - .finish(); + .finish() + } + + #[tokio::test] + async fn s3_ut_test_s3_stat() { + test_enable_with!(RUN_TEST_WITH_S3); + let config = s3_test_config(); + let op = create_opendal(&config); let path = "/"; let list = op.list(path).await; if let Ok(l) = list { @@ -294,4 +301,30 @@ mod test { println!("stat error: {:?}", meta.err()); } } + + #[tokio::test] + async fn s3_ut_test_s3_delete() { + test_enable_with!(RUN_TEST_WITH_S3); + let config = s3_test_config(); + + let op = create_opendal(&config); + let path = "/s1/fileset1/gvfs_test/test_dir/test_file"; + + let meta = op.stat(path).await; + if let Ok(m) = meta { + println!("stat result: {:?}", m); + } else { + println!("stat error: {:?}", meta.err()); + } + + let result = op.remove(vec![path.to_string()]).await; + match result { + Ok(_) => { + println!("Delete successful (or no-op)."); + } + Err(e) => { + println!("Delete failed: {:?}", e); + } + } + } } diff --git a/clients/filesystem-fuse/src/s3_filesystem.rs b/clients/filesystem-fuse/src/s3_filesystem.rs index e0ca69b4ccf..35a091b3fe1 100644 --- a/clients/filesystem-fuse/src/s3_filesystem.rs +++ b/clients/filesystem-fuse/src/s3_filesystem.rs @@ -40,7 +40,7 @@ impl S3FileSystem {} impl S3FileSystem { const S3_CONFIG_PREFIX: &'static str = "s3-"; - pub(crate) fn new( + pub(crate) async fn new( catalog: &Catalog, fileset: &Fileset, config: &AppConfig, @@ -48,10 +48,20 @@ impl S3FileSystem { ) -> GvfsResult { let mut opendal_config = extract_s3_config(config); let bucket = extract_bucket(&fileset.storage_location)?; - opendal_config.insert("bucket".to_string(), bucket); + opendal_config.insert("bucket".to_string(), bucket.to_string()); - let region = Self::get_s3_region(catalog)?; - opendal_config.insert("region".to_string(), region); + let endpoint = catalog.properties.get("s3-endpoint"); + if endpoint.is_none() { + return Err(InvalidConfig.to_error("s3-endpoint is required".to_string())); + } + let endpoint = endpoint.unwrap(); + opendal_config.insert("endpoint".to_string(), endpoint.clone()); + + let region = Self::get_s3_region(catalog, &bucket).await; + if region.is_none() { + return Err(InvalidConfig.to_error("s3-region is required".to_string())); + } + opendal_config.insert("region".to_string(), region.unwrap()); let builder = S3::from_map(opendal_config); @@ -67,16 +77,13 @@ impl S3FileSystem { }) } - fn get_s3_region(catalog: &Catalog) -> GvfsResult { + async fn get_s3_region(catalog: &Catalog, bucket: &str) -> Option { if let Some(region) = catalog.properties.get("s3-region") { - Ok(region.clone()) + Some(region.clone()) } else if let Some(endpoint) = catalog.properties.get("s3-endpoint") { - extract_region(endpoint) + S3::detect_region(endpoint, bucket).await } else { - Err(InvalidConfig.to_error(format!( - "Cant not retrieve region in the Catalog {}", - catalog.name - ))) + None } } } @@ -139,25 +146,11 @@ pub(crate) fn extract_bucket(location: &str) -> GvfsResult { } } -pub(crate) fn extract_region(location: &str) -> GvfsResult { - let url = parse_location(location)?; - match url.host_str() { - Some(host) => { - let parts: Vec<&str> = host.split('.').collect(); - if parts.len() > 1 { - Ok(parts[1].to_string()) - } else { - Err(InvalidConfig.to_error(format!( - "Invalid location: expected region in host, got {}", - location - ))) - } - } - None => Err(InvalidConfig.to_error(format!( - "Invalid fileset location without bucket: {}", - location - ))), - } +pub(crate) fn extract_region(location: &str) -> Option { + parse_location(location).ok().and_then(|url| { + url.host_str() + .and_then(|host| host.split('.').nth(1).map(|part| part.to_string())) + }) } pub fn extract_s3_config(config: &AppConfig) -> HashMap { @@ -181,11 +174,13 @@ pub fn extract_s3_config(config: &AppConfig) -> HashMap { } #[cfg(test)] -mod tests { +pub(crate) mod tests { use super::*; use crate::default_raw_filesystem::DefaultRawFileSystem; use crate::filesystem::tests::{TestPathFileSystem, TestRawFileSystem}; use crate::filesystem::RawFileSystem; + use crate::test_enable_with; + use crate::RUN_TEST_WITH_S3; use opendal::layers::TimeoutLayer; use std::time::Duration; @@ -201,11 +196,11 @@ mod tests { fn test_extract_region() { let location = "http://s3.ap-southeast-2.amazonaws.com"; let result = extract_region(location); - assert!(result.is_ok()); + assert!(result.is_some()); assert_eq!(result.unwrap(), "ap-southeast-2"); } - async fn delete_dir(op: &Operator, dir_name: &str) { + pub(crate) async fn delete_dir(op: &Operator, dir_name: &str) { let childs = op.list(dir_name).await.expect("list dir failed"); for child in childs { let child_name = dir_name.to_string() + child.name(); @@ -218,13 +213,11 @@ mod tests { op.delete(dir_name).await.expect("delete dir failed"); } - async fn create_s3_fs(cwd: &Path) -> S3FileSystem { - let config = AppConfig::from_file(Some("tests/conf/gvfs_fuse_s3.toml")).unwrap(); - let opendal_config = extract_s3_config(&config); - - let fs_context = FileSystemContext::default(); - - let builder = S3::from_map(opendal_config); + pub(crate) async fn cleanup_s3_fs( + cwd: &Path, + opendal_config: &HashMap, + ) -> Operator { + let builder = S3::from_map(opendal_config.clone()); let op = Operator::new(builder) .expect("opendal create failed") .layer(LoggingLayer::default()) @@ -241,18 +234,37 @@ mod tests { op.create_dir(&file_name) .await .expect("create test dir failed"); + op + } + + async fn create_s3_fs(cwd: &Path, config: &AppConfig) -> S3FileSystem { + let opendal_config = extract_s3_config(config); + let op = cleanup_s3_fs(cwd, &opendal_config).await; + + let fs_context = FileSystemContext::default(); + let open_dal_fs = OpenDalFileSystem::new(op, config, &fs_context); - let open_dal_fs = OpenDalFileSystem::new(op, &config, &fs_context); S3FileSystem { open_dal_fs } } - #[tokio::test] - async fn test_s3_file_system() { - if std::env::var("RUN_S3_TESTS").is_err() { - return; + pub(crate) fn s3_test_config() -> AppConfig { + let mut config_file_name = "target/conf/gvfs_fuse_s3.toml"; + let source_file_name = "tests/conf/gvfs_fuse_s3.toml"; + + if !Path::new(config_file_name).exists() { + config_file_name = source_file_name; } + + AppConfig::from_file(Some(config_file_name)).unwrap() + } + + #[tokio::test] + async fn s3_ut_test_s3_file_system() { + test_enable_with!(RUN_TEST_WITH_S3); + + let config = s3_test_config(); let cwd = Path::new("/gvfs_test1"); - let fs = create_s3_fs(cwd).await; + let fs = create_s3_fs(cwd, &config).await; let _ = fs.init().await; let mut tester = TestPathFileSystem::new(cwd, fs); @@ -260,13 +272,12 @@ mod tests { } #[tokio::test] - async fn test_s3_file_system_with_raw_file_system() { - if std::env::var("RUN_S3_TESTS").is_err() { - return; - } + async fn s3_ut_test_s3_file_system_with_raw_file_system() { + test_enable_with!(RUN_TEST_WITH_S3); + let config = s3_test_config(); let cwd = Path::new("/gvfs_test2"); - let s3_fs = create_s3_fs(cwd).await; + let s3_fs = create_s3_fs(cwd, &config).await; let raw_fs = DefaultRawFileSystem::new(s3_fs, &AppConfig::default(), &FileSystemContext::default()); let _ = raw_fs.init().await; diff --git a/clients/filesystem-fuse/tests/bin/env.sh b/clients/filesystem-fuse/tests/bin/env.sh new file mode 100644 index 00000000000..c2e0b23be05 --- /dev/null +++ b/clients/filesystem-fuse/tests/bin/env.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +S3_ACCESS_KEY_ID=${S3_ACCESS_KEY_ID:-test} +S3_SECRET_ACCESS=${S3_SECRET_ACCESS:-test} +S3_REGION=${S3_REGION:-ap-southeast-2} +S3_BUCKET=${S3_BUCKET:-my-bucket} +S3_ENDPOINT=${S3_ENDPOINT:-http://127.0.0.1:4566} + +# Check required environment variables +if [[ -z "$S3_ACCESS_KEY_ID" || -z "$S3_SECRET_ACCESS" || -z "$S3_REGION" || -z "$S3_BUCKET" || -z "$S3_ENDPOINT" ]]; then + echo "Error: One or more required S3 environment variables are not set." + echo "Please set: S3_ACCESS_KEY_ID, S3_SECRET_ACCESS, S3_REGION, S3_BUCKET, S3_ENDPOINT." + exit 1 +fi + +DISABLE_LOCALSTACK=${DISABLE_LOCALSTACK:-0} +# if S3 endpoint is not default value. disable localstack +if [[ "$S3_ENDPOINT" != "http://127.0.0.1:4566" ]]; then + echo "AWS S3 endpoint detected, disabling localstack" + DISABLE_LOCALSTACK=1 +fi + +GRAVITINO_HOME=../../../.. +GRAVITINO_HOME=$(cd $GRAVITINO_HOME && pwd) +GRAVITINO_SERVER_DIR=$GRAVITINO_HOME/distribution/package +CLIENT_FUSE_DIR=$GRAVITINO_HOME/clients/filesystem-fuse + +generate_test_config() { + local config_dir + config_dir=$(dirname "$TEST_CONFIG_FILE") + mkdir -p "$config_dir" + + awk -v access_key="$S3_ACCESS_KEY_ID" \ + -v secret_key="$S3_SECRET_ACCESS" \ + -v region="$S3_REGION" \ + -v bucket="$S3_BUCKET" \ + -v endpoint="$S3_ENDPOINT" \ + 'BEGIN { in_extend_config = 0 } + /^\[extend_config\]/ { in_extend_config = 1 } + in_extend_config && /s3-access_key_id/ { $0 = "s3-access_key_id = \"" access_key "\"" } + in_extend_config && /s3-secret_access_key/ { $0 = "s3-secret_access_key = \"" secret_key "\"" } + in_extend_config && /s3-region/ { $0 = "s3-region = \"" region "\"" } + in_extend_config && /s3-bucket/ { $0 = "s3-bucket = \"" bucket "\"" } + in_extend_config && /s3-endpoint/ { $0 = "s3-endpoint = \"" endpoint "\"" } + { print }' $CLIENT_FUSE_DIR/tests/conf/gvfs_fuse_s3.toml > "$TEST_CONFIG_FILE" +} diff --git a/clients/filesystem-fuse/tests/bin/gravitino_server.sh b/clients/filesystem-fuse/tests/bin/gravitino_server.sh new file mode 100644 index 00000000000..0f9b0fdab98 --- /dev/null +++ b/clients/filesystem-fuse/tests/bin/gravitino_server.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +GRAVITINO_SERVER_URL="http://localhost:8090" + +check_gravitino_server_ready() { + local url=$1 + local retries=10 # Number of retries + local wait_time=1 # Wait time between retries (seconds) + + for ((i=1; i<=retries; i++)); do + if curl --silent --head --fail "$url/api/metalakes" >/dev/null; then + echo "Gravitino server is ready." + return 0 + else + echo "Attempt $i/$retries: Server not ready. Retrying in $wait_time seconds..." + sleep "$wait_time" + fi + done + + echo "Error: Gravitino server did not become ready after $((retries * wait_time)) seconds." + exit 1 +} + +create_resource() { + local url=$1 + local data=$2 + + response=$(curl -s -w "\n%{http_code}" -X POST -H "Accept: application/vnd.gravitino.v1+json" \ + -H "Content-Type: application/json" -d "$data" "$url") + + body=$(echo "$response" | head -n -1) + response_code=$(echo "$response" | tail -n 1) + + # Check if the response code is not 2xx + if [[ "$response_code" -lt 200 || "$response_code" -ge 300 ]]; then + echo "Error: Failed to create resource. Status code: $response_code" + echo "Response body: $body" + exit 1 + fi +} + + + +start_gravitino_server() { + echo "Starting Gravitino Server" + # copy the aws-bundle to the server + if ls $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar 1>/dev/null 2>&1; then + echo "File exists, skipping copy." + else + echo "Copying the aws-bundle to the server" + cp $GRAVITINO_HOME/bundles/aws-bundle/build/libs/gravitino-aws-bundle-*-incubating-SNAPSHOT.jar \ + $GRAVITINO_SERVER_DIR/catalogs/hadoop/libs + fi + + rm -rf $GRAVITINO_SERVER_DIR/data + $GRAVITINO_SERVER_DIR/bin/gravitino.sh restart + + check_gravitino_server_ready $GRAVITINO_SERVER_URL + + # Create metalake + create_resource "$GRAVITINO_SERVER_URL/api/metalakes" '{ + "name":"test", + "comment":"comment", + "properties":{} + }' + + # Create catalog + create_resource "$GRAVITINO_SERVER_URL/api/metalakes/test/catalogs" '{ + "name": "c1", + "type": "FILESET", + "comment": "comment", + "provider": "hadoop", + "properties": { + "location": "s3a://'"$S3_BUCKET"'", + "s3-access-key-id": "'"$S3_ACCESS_KEY_ID"'", + "s3-secret-access-key": "'"$S3_SECRET_ACCESS"'", + "s3-endpoint": "'"$S3_ENDPOINT"'", + "filesystem-providers": "s3" + } + }' + + # Create schema + create_resource "$GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas" '{ + "name":"s1", + "comment":"comment", + "properties":{} + }' + + # Create FILESET + create_resource "$GRAVITINO_SERVER_URL/api/metalakes/test/catalogs/c1/schemas/s1/filesets" '{ + "name":"fileset1", + "comment":"comment", + "properties":{} + }' +} + +stop_gravitino_server() { + $GRAVITINO_SERVER_DIR/bin/gravitino.sh stop + echo "Gravitino Server stopped" +} \ No newline at end of file diff --git a/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh new file mode 100644 index 00000000000..e706d8e2c0d --- /dev/null +++ b/clients/filesystem-fuse/tests/bin/gvfs_fuse.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +check_gvfs_fuse_ready() { + local retries=10 + local wait_time=1 + + for ((i=1; i<=retries; i++)); do + # check the $MOUNT_DIR/.gvfs_meta is exist + if [ -f "$MOUNT_DIR/.gvfs_meta" ]; then + echo "Gvfs fuse is ready." + return 0 + else + echo "Attempt $i/$retries: Gvfs fuse not ready. Retrying in $wait_time seconds..." + sleep "$wait_time" + fi + done + + echo "Error: Gvfs fuse did not become ready after $((retries * wait_time)) seconds." + tail -n 100 $CLIENT_FUSE_DIR/target/debug/fuse.log + exit 1 +} + +start_gvfs_fuse() { + MOUNT_DIR=$CLIENT_FUSE_DIR/target/gvfs + + umount $MOUNT_DIR > /dev/null 2>&1 || true + if [ ! -d "$MOUNT_DIR" ]; then + echo "Create the mount point" + mkdir -p $MOUNT_DIR + fi + + MOUNT_FROM_LOCATION=gvfs://fileset/test/c1/s1/fileset1 + + # Build the gvfs-fuse + cd $CLIENT_FUSE_DIR + make build + + echo "Starting gvfs-fuse-daemon" + $CLIENT_FUSE_DIR/target/debug/gvfs-fuse $MOUNT_DIR $MOUNT_FROM_LOCATION $TEST_CONFIG_FILE > \ + $CLIENT_FUSE_DIR/target/debug/fuse.log 2>&1 & + check_gvfs_fuse_ready + cd - +} + +stop_gvfs_fuse() { + # Stop the gvfs-fuse process if it's running + pkill -INT gvfs-fuse || true + echo "Stopping gvfs-fuse-daemon" +} \ No newline at end of file diff --git a/clients/filesystem-fuse/tests/bin/localstatck.sh b/clients/filesystem-fuse/tests/bin/localstatck.sh new file mode 100644 index 00000000000..fa4552d48a3 --- /dev/null +++ b/clients/filesystem-fuse/tests/bin/localstatck.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +start_localstack() { +if [ "$DISABLE_LOCALSTACK" -eq 1 ]; then + return +fi + + echo "Starting localstack..." + docker run -d -p 4566:4566 -p 4571:4571 --name localstack localstack/localstack + echo "Localstack started" + + docker exec localstack sh -c "\ + aws configure set aws_access_key_id $S3_ACCESS_KEY_ID && \ + aws configure set aws_secret_access_key $S3_SECRET_ACCESS && \ + aws configure set region $S3_REGION && \ + aws configure set output json" + + docker exec localstack awslocal s3 mb s3://$S3_BUCKET +} + +stop_localstack() { +if [ "$DISABLE_LOCALSTACK" -eq 1 ]; then + return +fi + + echo "Stopping localstack..." + docker stop localstack 2>/dev/null || true + docker rm localstack 2>/dev/null || true + echo "Localstack stopped" +} \ No newline at end of file diff --git a/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh new file mode 100755 index 00000000000..6dc38c48f07 --- /dev/null +++ b/clients/filesystem-fuse/tests/bin/run_fuse_testers.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +source ./env.sh +source ./gravitino_server.sh +source ./gvfs_fuse.sh +source ./localstatck.sh + +TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/debug/gvfs-fuse.toml + +start_servers() { + start_localstack + start_gravitino_server + generate_test_config + start_gvfs_fuse +} + +stop_servers() { + set +e + stop_gvfs_fuse + stop_gravitino_server + stop_localstack +} + +# Main logic based on parameters +if [ "$1" == "test" ]; then + trap stop_servers EXIT + start_servers + # Run the integration test + echo "Running tests..." + cd $CLIENT_FUSE_DIR + export RUN_TEST_WITH_FUSE=1 + cargo test --test fuse_test fuse_it_ + +elif [ "$1" == "start" ]; then + # Start the servers + echo "Starting servers..." + start_servers + +elif [ "$1" == "stop" ]; then + # Stop the servers + echo "Stopping servers..." + stop_servers + +else + echo "Usage: $0 {test|start|stop}" + exit 1 +fi + + diff --git a/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh new file mode 100644 index 00000000000..ac5f9812c93 --- /dev/null +++ b/clients/filesystem-fuse/tests/bin/run_s3fs_testers.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +source ./env.sh +source ./localstatck.sh + +TEST_CONFIG_FILE=$CLIENT_FUSE_DIR/target/conf/gvfs_fuse_s3.toml + +start_servers() { + start_localstack + generate_test_config +} + +stop_servers() { + set +e + stop_localstack +} + +# Main logic based on parameters +if [ "$1" == "test" ]; then + trap stop_servers EXIT + start_servers + # Run the integration test + echo "Running tests..." + cd $CLIENT_FUSE_DIR + export RUN_TEST_WITH_S3=1 + cargo test s3_ut_ --lib + +elif [ "$1" == "start" ]; then + # Start the servers + echo "Starting servers..." + start_servers + +elif [ "$1" == "stop" ]; then + # Stop the servers + echo "Stopping servers..." + stop_servers + +else + echo "Usage: $0 {test|start|stop}" + exit 1 +fi + + diff --git a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml index 7d182cd40df..d0ff8e5ddec 100644 --- a/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml +++ b/clients/filesystem-fuse/tests/conf/gvfs_fuse_s3.toml @@ -19,7 +19,7 @@ [fuse] file_mask= 0o600 dir_mask= 0o700 -fs_type = "memory" +fs_type = "gvfs" [fuse.properties] key1 = "value1" @@ -40,4 +40,5 @@ s3-access_key_id = "XXX_access_key" s3-secret_access_key = "XXX_secret_key" s3-region = "XXX_region" s3-bucket = "XXX_bucket" +s3-endpoint = "XXX_endpoint" diff --git a/clients/filesystem-fuse/tests/fuse_test.rs b/clients/filesystem-fuse/tests/fuse_test.rs index d06199d782e..41e385c49f1 100644 --- a/clients/filesystem-fuse/tests/fuse_test.rs +++ b/clients/filesystem-fuse/tests/fuse_test.rs @@ -19,7 +19,8 @@ use fuse3::Errno; use gvfs_fuse::config::AppConfig; -use gvfs_fuse::{gvfs_mount, gvfs_unmount}; +use gvfs_fuse::RUN_TEST_WITH_FUSE; +use gvfs_fuse::{gvfs_mount, gvfs_unmount, test_enable_with}; use log::{error, info}; use std::fs::File; use std::path::Path; @@ -85,7 +86,7 @@ impl Drop for FuseTest { } #[test] -fn test_fuse_system_with_auto() { +fn test_fuse_with_memory_fs() { tracing_subscriber::fmt().init(); panic::set_hook(Box::new(|info| { @@ -106,14 +107,21 @@ fn test_fuse_system_with_auto() { test_fuse_filesystem(mount_point); } -fn test_fuse_system_with_manual() { - test_fuse_filesystem("build/gvfs"); +#[test] +fn fuse_it_test_fuse() { + test_enable_with!(RUN_TEST_WITH_FUSE); + + test_fuse_filesystem("target/gvfs/gvfs_test"); } fn test_fuse_filesystem(mount_point: &str) { info!("Test startup"); let base_path = Path::new(mount_point); + if !file_exists(base_path) { + fs::create_dir_all(base_path).expect("Failed to create test dir"); + } + //test create file let test_file = base_path.join("test_create"); let file = File::create(&test_file).expect("Failed to create file"); @@ -124,12 +132,12 @@ fn test_fuse_filesystem(mount_point: &str) { fs::write(&test_file, "read test").expect("Failed to write file"); //test read file - let content = fs::read_to_string(test_file.clone()).expect("Failed to read file"); + let content = fs::read_to_string(&test_file).expect("Failed to read file"); assert_eq!(content, "read test", "File content mismatch"); //test delete file - fs::remove_file(test_file.clone()).expect("Failed to delete file"); - assert!(!file_exists(test_file)); + fs::remove_file(&test_file).expect("Failed to delete file"); + assert!(!file_exists(&test_file)); //test create directory let test_dir = base_path.join("test_dir"); diff --git a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java index 14b1912b4d6..444e89062a6 100644 --- a/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java +++ b/core/src/main/java/org/apache/gravitino/connector/BaseCatalog.java @@ -208,8 +208,15 @@ public void initAuthorizationPluginInstance(IsolatedClassLoader classLoader) { try { BaseAuthorization authorization = BaseAuthorization.createAuthorization(classLoader, authorizationProvider); + + // Load the authorization plugin with the class loader of the catalog. + // Because the JDBC authorization plugin may load JDBC driver using the class loader. authorizationPlugin = - authorization.newPlugin(entity.namespace().level(0), provider(), this.conf); + classLoader.withClassLoader( + cl -> + authorization.newPlugin( + entity.namespace().level(0), provider(), this.conf)); + } catch (Exception e) { LOG.error("Failed to load authorization with class loader", e); throw new RuntimeException(e); diff --git a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java index 6ac0c498c02..2a7955d08ad 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java +++ b/core/src/main/java/org/apache/gravitino/credential/CatalogCredentialContext.java @@ -47,7 +47,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || !(o instanceof CatalogCredentialContext)) { + if (!(o instanceof CatalogCredentialContext)) { return false; } return Objects.equal(userName, ((CatalogCredentialContext) o).userName); diff --git a/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java b/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java index 1d0d8f7b3b6..635f530b073 100644 --- a/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java +++ b/core/src/main/java/org/apache/gravitino/credential/CredentialCacheKey.java @@ -43,7 +43,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || !(o instanceof CredentialCacheKey)) { + if (!(o instanceof CredentialCacheKey)) { return false; } CredentialCacheKey that = (CredentialCacheKey) o; diff --git a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java index 06d17b134ba..2c015b0c569 100644 --- a/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java +++ b/core/src/main/java/org/apache/gravitino/credential/PathBasedCredentialContext.java @@ -67,7 +67,7 @@ public boolean equals(Object o) { if (this == o) { return true; } - if (o == null || !(o instanceof PathBasedCredentialContext)) { + if (!(o instanceof PathBasedCredentialContext)) { return false; } PathBasedCredentialContext that = (PathBasedCredentialContext) o; diff --git a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java index 26f31a88396..aa53b8800f8 100644 --- a/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java +++ b/core/src/main/java/org/apache/gravitino/hook/MetalakeHookDispatcher.java @@ -116,6 +116,7 @@ public void disableMetalake(NameIdentifier ident) throws NoSuchMetalakeException dispatcher.disableMetalake(ident); } + @Override public boolean dropMetalake(NameIdentifier ident) { // For metalake, we don't clear all the privileges of catalog authorization plugin. // we just remove metalake. diff --git a/core/src/main/java/org/apache/gravitino/listener/api/event/CreateTablePreEvent.java b/core/src/main/java/org/apache/gravitino/listener/api/event/CreateTablePreEvent.java index 6c01d614f3c..dd6b8cc123b 100644 --- a/core/src/main/java/org/apache/gravitino/listener/api/event/CreateTablePreEvent.java +++ b/core/src/main/java/org/apache/gravitino/listener/api/event/CreateTablePreEvent.java @@ -43,6 +43,7 @@ public TableInfo createTableRequest() { return createTableRequest; } + @Override public OperationType operationType() { return OperationType.CREATE_TABLE; } diff --git a/core/src/main/java/org/apache/gravitino/lock/LockManager.java b/core/src/main/java/org/apache/gravitino/lock/LockManager.java index 222dee8daad..d52c858cc43 100644 --- a/core/src/main/java/org/apache/gravitino/lock/LockManager.java +++ b/core/src/main/java/org/apache/gravitino/lock/LockManager.java @@ -26,6 +26,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.text.SimpleDateFormat; import java.util.List; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -136,10 +137,14 @@ void checkDeadLock(TreeLockNode node) { // If the thread is holding the lock for more than 30 seconds, we will log it. if (System.currentTimeMillis() - ts > 30000) { LOG.warn( - "Dead lock detected for thread with identifier {} on node {}, threads that holding the node: {} ", + "Thread with identifier {} holds the lock node {} for more than 30s since {}, please " + + "check if some dead lock or thread hang like io-connection hangs", threadIdentifier, node, - node.getHoldingThreadTimestamp()); + // SimpleDateFormat is not thread-safe, so we should create a new instance for + // each time + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") + .format(node.getHoldingThreadTimestamp())); } }); } diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java index abaf2c59af9..77bf3c4e285 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/CatalogMetaPostgreSQLProvider.java @@ -76,6 +76,7 @@ public String insertCatalogMetaOnDuplicateKeyUpdate(CatalogPO catalogPO) { + " deleted_at = #{catalogMeta.deletedAt}"; } + @Override public String updateCatalogMeta( @Param("newCatalogMeta") CatalogPO newCatalogPO, @Param("oldCatalogMeta") CatalogPO oldCatalogPO) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java index a95d7f09fe3..06dde29751c 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/MetalakeMetaPostgreSQLProvider.java @@ -62,6 +62,7 @@ public String insertMetalakeMetaOnDuplicateKeyUpdate(MetalakePO metalakePO) { + " deleted_at = #{metalakeMeta.deletedAt}"; } + @Override public String updateMetalakeMeta( @Param("newMetalakeMeta") MetalakePO newMetalakePO, @Param("oldMetalakeMeta") MetalakePO oldMetalakePO) { diff --git a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java index 92352bcd95a..6de57dbdc48 100644 --- a/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java +++ b/core/src/main/java/org/apache/gravitino/storage/relational/mapper/provider/postgresql/SecurableObjectPostgreSQLProvider.java @@ -32,6 +32,7 @@ import org.apache.ibatis.annotations.Param; public class SecurableObjectPostgreSQLProvider extends SecurableObjectBaseSQLProvider { + @Override public String batchSoftDeleteSecurableObjects( @Param("securableObjects") List securableObjectPOs) { return "