diff --git a/catalogs/catalog-hadoop/build.gradle.kts b/catalogs/catalog-hadoop/build.gradle.kts index 3108d993c1a..36e6934d110 100644 --- a/catalogs/catalog-hadoop/build.gradle.kts +++ b/catalogs/catalog-hadoop/build.gradle.kts @@ -52,6 +52,8 @@ dependencies { exclude("org.eclipse.jetty", "*") exclude("io.netty") exclude("org.fusesource.leveldbjni") + // Exclude protobuf-java to avoid conflict with; + exclude("com.google.protobuf", "protobuf-java") } implementation(libs.slf4j.api) implementation(libs.awaitility) diff --git a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java index 60ffe3e83c8..fe114569a44 100644 --- a/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java +++ b/core/src/main/java/org/apache/gravitino/authorization/AuthorizationUtils.java @@ -25,6 +25,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -61,6 +62,7 @@ /* The utilization class of authorization module*/ public class AuthorizationUtils { + private static final Logger LOG = LoggerFactory.getLogger(AuthorizationUtils.class); static final String USER_DOES_NOT_EXIST_MSG = "User %s does not exist in the metalake %s"; static final String GROUP_DOES_NOT_EXIST_MSG = "Group %s does not exist in the metalake %s"; @@ -417,7 +419,7 @@ private static String getHiveDefaultLocation(String metalakeName, String catalog if (defaultSchemaLocation != null && !defaultSchemaLocation.isEmpty()) { return defaultSchemaLocation; } else { - LOG.warn("Schema %s location is not found", defaultSchemaIdent); + LOG.warn("Schema {} location is not found", defaultSchemaIdent); } } @@ -445,7 +447,7 @@ public static List getMetadataObjectLocation( // The Hive default schema location is Hive warehouse directory String defaultSchemaLocation = getHiveDefaultLocation(ident.name(), catalogObj.name()); - if (defaultSchemaLocation != null && !defaultSchemaLocation.isEmpty()) { + if (StringUtils.isNotBlank(defaultSchemaLocation)) { locations.add(defaultSchemaLocation); } } @@ -462,6 +464,13 @@ public static List getMetadataObjectLocation( if (defaultSchemaLocation != null && !defaultSchemaLocation.isEmpty()) { locations.add(defaultSchemaLocation); } + } else if (catalogObj.provider().equals("hadoop")) { + String catalogLocation = catalogObj.properties().get(HiveConstants.LOCATION); + if (StringUtils.isNotBlank(catalogLocation)) { + locations.add(catalogLocation); + } else { + LOG.warn("Catalog {} location is not found", ident); + } } } break; @@ -472,19 +481,20 @@ public static List getMetadataObjectLocation( .catalogDispatcher() .loadCatalog( NameIdentifier.of(ident.namespace().level(0), ident.namespace().level(1))); - LOG.info("Catalog provider is %s", catalogObj.provider()); - if (catalogObj.provider().equals("hive")) { + LOG.info("Catalog provider is {}", catalogObj.provider()); + String catalogProvider = catalogObj.provider(); + if (Objects.equals(catalogProvider, "hive") + || Objects.equals(catalogProvider, "hadoop")) { Schema schema = GravitinoEnv.getInstance().schemaDispatcher().loadSchema(ident); if (schema.properties().containsKey(HiveConstants.LOCATION)) { String schemaLocation = schema.properties().get(HiveConstants.LOCATION); if (StringUtils.isNotBlank(schemaLocation)) { locations.add(schemaLocation); } else { - LOG.warn("Schema %s location is not found", ident); + LOG.warn("Schema '{}' location is not found", ident); } } } - // TODO: [#6133] Supports get Fileset schema location in the AuthorizationUtils } break; case TABLE: @@ -501,35 +511,32 @@ public static List getMetadataObjectLocation( if (StringUtils.isNotBlank(tableLocation)) { locations.add(tableLocation); } else { - LOG.warn("Table %s location is not found", ident); + LOG.warn("Table {} location is not found", ident); } } } } break; case FILESET: - FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); - Fileset fileset = filesetDispatcher.loadFileset(ident); - Preconditions.checkArgument( - fileset != null, String.format("Fileset %s is not found", ident)); - String filesetLocation = fileset.storageLocation(); - Preconditions.checkArgument( - filesetLocation != null, String.format("Fileset %s location is not found", ident)); - locations.add(filesetLocation); + { + FilesetDispatcher filesetDispatcher = GravitinoEnv.getInstance().filesetDispatcher(); + Fileset fileset = filesetDispatcher.loadFileset(ident); + Preconditions.checkArgument( + fileset != null, String.format("Fileset %s is not found", ident)); + String filesetLocation = fileset.storageLocation(); + Preconditions.checkArgument( + filesetLocation != null, String.format("Fileset %s location is not found", ident)); + locations.add(filesetLocation); + } break; default: throw new AuthorizationPluginException( "Failed to get location paths for metadata object %s type %s", ident, type); } } catch (Exception e) { - LOG.warn("Failed to get location paths for metadata object %s type %s", ident, type, e); + LOG.warn("Failed to get location paths for metadata object {} type {}", ident, type, e); } return locations; } - - private static NameIdentifier getObjectNameIdentifier( - String metalake, MetadataObject metadataObject) { - return NameIdentifier.parse(String.format("%s.%s", metalake, metadataObject.fullName())); - } } diff --git a/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java b/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java index 373785d539b..127010dfdd8 100644 --- a/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java +++ b/core/src/test/java/org/apache/gravitino/authorization/TestAuthorizationUtils.java @@ -27,7 +27,9 @@ import org.apache.gravitino.GravitinoEnv; import org.apache.gravitino.NameIdentifier; import org.apache.gravitino.Namespace; +import org.apache.gravitino.Schema; import org.apache.gravitino.catalog.CatalogDispatcher; +import org.apache.gravitino.catalog.SchemaDispatcher; import org.apache.gravitino.catalog.TableDispatcher; import org.apache.gravitino.exceptions.IllegalNameIdentifierException; import org.apache.gravitino.exceptions.IllegalNamespaceException; @@ -224,6 +226,7 @@ void testFilteredSecurableObjects() { void testGetMetadataObjectLocation() throws IllegalAccessException { CatalogDispatcher catalogDispatcher = Mockito.mock(CatalogDispatcher.class); TableDispatcher tableDispatcher = Mockito.mock(TableDispatcher.class); + SchemaDispatcher schemaDispatcher = Mockito.mock(SchemaDispatcher.class); Catalog catalog = Mockito.mock(Catalog.class); Table table = Mockito.mock(Table.class); @@ -234,6 +237,7 @@ void testGetMetadataObjectLocation() throws IllegalAccessException { FieldUtils.writeField(GravitinoEnv.getInstance(), "catalogDispatcher", catalogDispatcher, true); FieldUtils.writeField(GravitinoEnv.getInstance(), "tableDispatcher", tableDispatcher, true); + FieldUtils.writeField(GravitinoEnv.getInstance(), "schemaDispatcher", schemaDispatcher, true); List locations = AuthorizationUtils.getMetadataObjectLocation( @@ -246,5 +250,17 @@ void testGetMetadataObjectLocation() throws IllegalAccessException { NameIdentifier.of("catalog", "schema", "fileset"), Entity.EntityType.TABLE); Assertions.assertEquals(1, locations.size()); Assertions.assertEquals("gs://bucket/1", locations.get(0)); + + Schema schema = Mockito.mock(Schema.class); + Mockito.when(schema.properties()).thenReturn(ImmutableMap.of("location", "gs://bucket/2")); + Mockito.when(schemaDispatcher.loadSchema(Mockito.any())).thenReturn(schema); + Mockito.when(catalog.provider()).thenReturn("hadoop"); + Mockito.when(catalogDispatcher.loadCatalog(Mockito.any())).thenReturn(catalog); + + locations = + AuthorizationUtils.getMetadataObjectLocation( + NameIdentifier.of("test", "catalog", "schema"), Entity.EntityType.SCHEMA); + Assertions.assertEquals(1, locations.size()); + Assertions.assertEquals("gs://bucket/2", locations.get(0)); } }