Skip to content

Commit

Permalink
Test improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@jpisaac
jpisaac committed Feb 5, 2025
1 parent 9adc731 commit 7154375
Show file tree
Hide file tree
Showing 2 changed files with 118 additions and 12 deletions.
70 changes: 69 additions & 1 deletion phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
Expand All @@ -108,6 +109,8 @@
import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY;
import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;

Expand Down Expand Up @@ -504,7 +507,42 @@ private Pair<String, byte[]> getRowKeyMatchersFromView(PhoenixConnection connect

}

// Helper to get rowKeyMatcher from Metadata.
byte[] getRowKeyMatcherFromSyscatIndex(String tenantId, String schemaName,
String tableName, boolean useIndexTable) throws SQLException {

final String
SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL =
"SELECT ROW_KEY_MATCHER FROM SYSTEM.CATALOG " + "WHERE %s AND TABLE_SCHEM <> 'SYSTEM' AND TABLE_NAME = '%s' AND ROW_KEY_MATCHER IS NOT NULL";
final String SYS_CATALOG_IDX_ROW_KEY_MATCHER_HEADER_SQL = "SELECT \"0:ROW_KEY_MATCHER\" FROM SYSTEM.SYS_ROW_KEY_MATCHER_IDX " + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'";

try (Connection connection = DriverManager.getConnection(getUrl())) {
Statement stmt = connection.createStatement();
String
tenantClause = useIndexTable ?
(tenantId == null || tenantId.isEmpty() ?
"\":TENANT_ID\" IS NULL" :
String.format("\":TENANT_ID\" = '%s'", tenantId)) :
(tenantId == null || tenantId.isEmpty() ?
"TENANT_ID IS NULL" :
String.format("TENANT_ID = '%s'", tenantId));
String
sql = useIndexTable ?
String.format(SYS_CATALOG_IDX_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, schemaName,
tableName) :
String.format(SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL, tenantClause,
tableName);
stmt.execute(sql);
ResultSet rs = stmt.getResultSet();
byte[] matcherBytes = rs.next() ? rs.getBytes(1) : EMPTY_BYTE_ARRAY;
LOGGER.info("Row key matcher SQL: {}", sql);
LOGGER.info("Row key matcher: {}, {}",
Bytes.toStringBinary(matcherBytes),
Bytes.toStringBinary(PVarbinaryEncoded.INSTANCE.toBytes(matcherBytes)));
return PVarbinaryEncoded.INSTANCE.toBytes(matcherBytes);
}
}

// Helper to get rowKeyMatcher from Metadata.
private Pair<String, byte[]> getRowKeyMatchersFromView(PhoenixConnection connection,
PTable view) throws SQLException {
return getRowKeyMatchersFromView(connection, view.getName().getString());
Expand Down Expand Up @@ -560,6 +598,12 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v
PVarbinaryEncoded.INSTANCE.toBytes(
WhereOptimizer.getRowKeyMatcher(connection, tableName, viewStatementTable,
viewColumnConstantsToBe, isViewColumnReferencedToBe));
byte[]
rowKeyMatcher3 = getRowKeyMatcherFromSyscatIndex(view.getTenantId() != null ? view.getTenantId().getString() : null, view.getSchemaName().getString(), view.getTableName().getString(), false);

byte[]
rowKeyMatcher4 = getRowKeyMatcherFromSyscatIndex(view.getTenantId() != null ? view.getTenantId().getString() : null, view.getSchemaName().getString(), view.getTableName().getString(), true);

LOGGER.debug(String.format(
"target-view-name = %s, physical = %s, stmt-table = %s\n, " +
"row-matcher-0 = %s (syscat)\n, row-matcher-1 = %s\n, row-matcher-2 = %s\n",
Expand All @@ -571,6 +615,10 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v
Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher1) == 0);
assertTrue("RowKey matcher patterns do not match",
Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher2) == 0);
assertTrue("RowKey matcher patterns do not match",
Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher3) == 0);
assertTrue("RowKey matcher patterns do not match",
Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher4) == 0);
return rowKeyMatcher1;
}

Expand Down Expand Up @@ -808,6 +856,15 @@ public void testViewsWithExtendedPK() {
List<PDataType[]> testCases = getTestCases();
SortOrder[][] sortOrders = getSortOrders();

try (Connection conn = DriverManager.getConnection(getUrl());
Statement stmt = conn.createStatement()) {
//TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES));
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'");
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL");
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL");
conn.commit();
}

String tableName = "";
tableName = createViewHierarchy(
testCases, sortOrders, 500, 5000, 3,
Expand Down Expand Up @@ -879,9 +936,20 @@ public void testViewsWithVariousTenantIdTypes() {
@Test
public void testViewsWithoutExtendedPK() {
try {

List<PDataType[]> testCases = getTestCases();
SortOrder[][] sortOrders = getSortOrders();
String tableName = "";

try (Connection conn = DriverManager.getConnection(getUrl());
Statement stmt = conn.createStatement()) {
//TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES));
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'");
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE IS NOT NULL AND LINK_TYPE IS NULL AND ROW_KEY_MATCHER IS NOT NULL");
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL");
conn.commit();
}

tableName =
createViewHierarchy(
testCases, sortOrders, 100, 1000, 3,
Expand Down
60 changes: 49 additions & 11 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,12 +13,10 @@
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TableViewFinderResult;
import org.apache.phoenix.util.TestUtil;
import org.apache.phoenix.util.ViewUtil;
import org.junit.BeforeClass;
import org.junit.Test;
Expand Down Expand Up @@ -52,7 +50,10 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT {
static final int VIEW_TTL_300_SECS = 300;
static final int VIEW_TTL_120_SECS = 120;

static final String SYS_CATALOG_VIEW_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG "
static final String SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL = "SELECT ROW_KEY_MATCHER FROM SYSTEM.CATALOG "
+ "WHERE %s AND TABLE_SCHEM <> 'SYSTEM' AND TABLE_NAME = '%s' AND " + "ROW_KEY_MATCHER IS NOT NULL";

static final String SYS_CATALOG_VIEW_TTL_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG "
+ "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = 'v'";

static final String SYS_CATALOG_VIEW_INDEX_HEADER_SQL = "SELECT VIEW_INDEX_ID FROM SYSTEM.CATALOG "
Expand All @@ -74,7 +75,7 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT {
static final String SYS_CATALOG_IDX_VIEW_INDEX_HEADER_SQL = "SELECT \": DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE)\" FROM %s " +
"WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ;

private static RegionCoprocessorEnvironment TaskRegionEnvironment;
private static RegionCoprocessorEnvironment taskRegionEnvironment;

@BeforeClass
public static void doSetup() throws Exception {
Expand All @@ -96,7 +97,7 @@ public static void doSetup() throws Exception {
setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS,
DEFAULT_PROPERTIES.entrySet().iterator()));

TaskRegionEnvironment =
taskRegionEnvironment =
getUtility()
.getRSForFirstRegionInTable(
PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME)
Expand All @@ -105,7 +106,6 @@ public static void doSetup() throws Exception {
.findCoprocessorEnvironment(TaskRegionObserver.class.getName());



}


Expand Down Expand Up @@ -161,7 +161,7 @@ void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String sche
"TENANT_ID IS NULL" :
String.format("TENANT_ID = '%s'", tenantId);
String sql = String
.format(SYS_CATALOG_VIEW_HEADER_SQL, tenantClause, schemaName, tableName);
.format(SYS_CATALOG_VIEW_TTL_HEADER_SQL, tenantClause, schemaName, tableName);
stmt.execute(sql);
ResultSet rs = stmt.getResultSet();
if (exists) {
Expand All @@ -177,6 +177,30 @@ void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String sche
}
}

void assertSystemCatalogHasRowKeyMatcherRelatedColumns(String tenantId, String schemaName,
String tableName, boolean exists) throws SQLException {

try (Connection connection = DriverManager.getConnection(getUrl())) {
Statement stmt = connection.createStatement();
String tenantClause = tenantId == null || tenantId.isEmpty() ?
"TENANT_ID IS NULL" :
String.format("TENANT_ID = '%s'", tenantId);
String sql = String
.format(SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, tableName);
stmt.execute(sql);
ResultSet rs = stmt.getResultSet();
if (exists) {
byte[] matcherBytes = rs.next() ? rs.getBytes(1) : null;
assertNotNull(String.format("Expected rows do not match for schema = %s, table = %s",
schemaName, tableName), matcherBytes);
} else {
assertFalse(String.format("Rows do exists for schema = %s, table = %s",
schemaName, tableName), rs.next());

}
}
}

String stripQuotes(String name) {
return name.replace("\"", "");
}
Expand Down Expand Up @@ -291,9 +315,10 @@ void dropTableWithChildViews(String baseTable, int numTaskRuns) throws Exception
// Run DropChildViewsTask to complete the tasks for dropping child views. The depth of the view tree is 2,
// so we expect that this will be done in two task handling runs as each non-root level will be processed
// in one run

TaskRegionObserver.SelfHealingTask task =
new TaskRegionObserver.SelfHealingTask(
TaskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
taskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
for (int i = 0; i < numTaskRuns; i++) {
task.run();
}
Expand Down Expand Up @@ -542,12 +567,15 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception {
// Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') exists in SYSTEM. CATALOG
assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, indexOnTenantViewName,true);

assertSystemCatalogHasRowKeyMatcherRelatedColumns(tenantId, schemaName, tenantViewName,true);

try (Connection conn = DriverManager.getConnection(getUrl());
Statement stmt = conn.createStatement()) {
//TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES));
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'");
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL");
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL");

conn.commit();
}

Expand All @@ -558,40 +586,50 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception {
// Assert System Catalog index table has been created
assertSystemCatalogIndexTable("SYS_VIEW_HDR_IDX", true);
assertSystemCatalogIndexTable("SYS_VIEW_INDEX_HDR_IDX", true);
assertSystemCatalogIndexTable("SYS_ROW_KEY_MATCHER_IDX", true);
// Assert appropriate rows are inserted in the SYSTEM.CATALOG index tables
assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, true);
assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", tenantId, schemaName, tenantViewName, true);
assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, indexOnTenantViewName, true);

/**
* Testing explain plans
*/

List<String> plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'", new Properties());
List<String> plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' ", new Properties());
assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_HDR_IDX", plans.get(0));

plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties());
plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties());
assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_INDEX_HDR_IDX", plans.get(0));

plans = getExplain("select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL", new Properties());
assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER SYSTEM.SYS_ROW_KEY_MATCHER_IDX [not null]", plans.get(0));

/**
* Testing cleanup of SYS_INDEX rows after dropping tables and views
*/
LOGGER.info("Dropping base table " + fullBaseTableName);
dropTableWithChildViews(fullBaseTableName, 1);
// Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exists in SYSTEM.CATALOG
// Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exist in SYSTEM.CATALOG
assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName,
false, VIEW_TTL_120_SECS);
// Assert view header rows (ROW_KEY_MATCHER IS NOT NULL does not exist in SYSTEM.CATALOG
assertSystemCatalogHasRowKeyMatcherRelatedColumns(tenantId, schemaName, tenantViewName,false);
// Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') does not exists in SYSTEM.CATALOG
assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, tenantViewName,false);

// Assert appropriate rows are dropped/deleted in the SYSTEM.CATALOG index tables
assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, false);
assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", tenantId, schemaName, tenantViewName, false);
assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, tenantViewName, false);

dropSystemCatalogIndex("SYS_VIEW_HDR_IDX");
dropSystemCatalogIndex("SYS_ROW_KEY_MATCHER_IDX");
dropSystemCatalogIndex("SYS_VIEW_INDEX_HDR_IDX");

// Assert System Catalog index table dropped
assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_HDR_IDX", false);
assertSystemCatalogIndexTable("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", false);
assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", false);
}

Expand Down

0 comments on commit 7154375

Please sign in to comment.