Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PHOENIX-7507 Update ROW_KEY_MATCHER column type to be VARBINARY_ENCODED #2060

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
+431 −96
Open

PHOENIX-7507 Update ROW_KEY_MATCHER column type to be VARBINARY_ENCODED #2060

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6d54711
PHOENIX-7507 Update ROW_KEY_MATCHER column type to be VARBINARY_ENCODED
jpisaac Jan 10, 2025
cc6aca4
Addressed review comments
jpisaac Jan 22, 2025
3b857e7
Backward compatibility fix for VARBINARY_ENCODED columns
jpisaac Jan 29, 2025
3400fef
Removed logging
jpisaac Jan 31, 2025
1873315
Optimized backward compatibility check
jpisaac Jan 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryConstants.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,7 @@ enum JoinType {INNER, LEFT_OUTER}
MAX_LOOKBACK_AGE + " BIGINT, \n" +
CDC_INCLUDE_TABLE + " VARCHAR, \n" +
TTL + " VARCHAR, \n" +
ROW_KEY_MATCHER + " VARBINARY, \n" +
ROW_KEY_MATCHER + " VARBINARY_ENCODED, \n" +
// Column metadata (will be null for table row)
DATA_TYPE + " INTEGER," +
COLUMN_SIZE + " INTEGER," +
Expand Down
2 changes: 1 addition & 1 deletion phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3655,7 +3655,7 @@ public boolean isViewReferenced() {

if ((rowKeyMatcher == null) ||
Bytes.compareTo(rowKeyMatcher, HConstants.EMPTY_BYTE_ARRAY) == 0) {
tableUpsert.setNull(38, Types.VARBINARY);
tableUpsert.setNull(38, PDataType.VARBINARY_ENCODED_TYPE);
} else {
tableUpsert.setBytes(38, rowKeyMatcher);
}
Expand Down
67 changes: 55 additions & 12 deletions phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import org.apache.phoenix.schema.RowKeyValueAccessor;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.tuple.ResultTuple;
import org.apache.phoenix.schema.types.PBinary;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDate;
Expand All @@ -67,13 +68,17 @@
import org.apache.phoenix.schema.types.PSmallint;
import org.apache.phoenix.schema.types.PTimestamp;
import org.apache.phoenix.schema.types.PTinyint;
import org.apache.phoenix.schema.types.PVarbinary;
import org.apache.phoenix.schema.types.PVarbinaryEncoded;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.LogUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ScanUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.ViewUtil;
import org.junit.Test;
import org.slf4j.Logger;
Expand Down Expand Up @@ -153,6 +158,12 @@ private String getType(PDataType pkType) {
case Types.TIMESTAMP:
pkTypeStr = "TIMESTAMP";
break;
case Types.VARBINARY:
pkTypeStr = "VARBINARY";
break;
case PDataType.VARBINARY_ENCODED_TYPE:
pkTypeStr = "VARBINARY_ENCODED";
break;
default:
pkTypeStr = "VARCHAR(25)";
}
Expand Down Expand Up @@ -183,6 +194,15 @@ private Object getData(PDataType type) {
case Types.TIMESTAMP:
//pkTypeStr = "TIMESTAMP";
return new Timestamp(System.currentTimeMillis() + rnd.nextInt(50000));
case Types.VARBINARY:
// pkTypeStr = "VARBINARY";
case PDataType.VARBINARY_ENCODED_TYPE:
// pkTypeStr = "VARBINARY_ENCODED";
byte[] varBytes = ByteUtil.concat(
RandomStringUtils.randomAlphanumeric(25).getBytes(),
Bytes.toBytes(rnd.nextInt(50000)),
Bytes.toBytes(Math.floor(rnd.nextInt(50000) * rnd.nextDouble())));
return PVarbinary.INSTANCE.toStringLiteral(varBytes);
default:
// pkTypeStr = "VARCHAR(25)";
return RandomStringUtils.randomAlphanumeric(25);
Expand Down Expand Up @@ -271,17 +291,21 @@ private Pair<String, byte[]> createGlobalView(String tableName, int partition,
"CONSTRAINT pk PRIMARY KEY (ID1 %s, ID2 %s, ID3 %s, ROW_ID)) " +
"AS SELECT * FROM %s WHERE KP = '%s'";

cstmt.execute(String.format(VIEW_TEMPLATE, globalViewName, pkType1Str, pkType2Str,
String globalViewSQL = String.format(VIEW_TEMPLATE, globalViewName, pkType1Str, pkType2Str,
pkType3Str, pkOrders[0].name(), pkOrders[1].name(), pkOrders[2].name(),
baseTableName, partitionName));
baseTableName, partitionName);
LOGGER.info("Created global view {}", globalViewSQL);
cstmt.execute(globalViewSQL);
if (hasGlobalViewIndexes) {
String indexNamePrefix = String.format("G%s", partition);
String
GLOBAL_INDEX_TEMPLATE =
"CREATE INDEX IF NOT EXISTS %s_COL2_INDEX ON %s (COL2) " +
"INCLUDE(SYSTEM_MODSTAMP)";
cstmt.execute(
String.format(GLOBAL_INDEX_TEMPLATE, indexNamePrefix, globalViewName));
String globalViewIndexSQL =
String.format(GLOBAL_INDEX_TEMPLATE, indexNamePrefix, globalViewName);
LOGGER.info("Created global view index {}", globalViewIndexSQL);
cstmt.execute(globalViewIndexSQL);
}

return getRowKeyMatchersFromView(globalConnection.unwrap(PhoenixConnection.class),
Expand Down Expand Up @@ -320,12 +344,16 @@ private Pair<String, byte[]> createTenantView(boolean extendPK, int partition,
"CONSTRAINT pk PRIMARY KEY (ZID)) " + "AS SELECT * FROM %s %s %s";
String VIEW_WO_PK_TEMPLATE = "CREATE VIEW IF NOT EXISTS %s AS SELECT * from %s %s";
if (extendPK) {
cstmt.execute(
String viewWithPKSQL =
String.format(VIEW_WITH_PK_TEMPLATE, tenantViewName, globalViewName,
getWhereClause(pkNames, pkTypes), tenantViewOptions));
getWhereClause(pkNames, pkTypes), tenantViewOptions);
LOGGER.info("Created tenant view (WITH_PK) {}", viewWithPKSQL);
cstmt.execute(viewWithPKSQL);
} else {
cstmt.execute(String.format(VIEW_WO_PK_TEMPLATE, tenantViewName, globalViewName,
tenantViewOptions));
String viewWithoutPKSQL = String.format(VIEW_WO_PK_TEMPLATE, tenantViewName, globalViewName,
tenantViewOptions);
LOGGER.info("Created tenant view ((WO_PK)) {}", viewWithoutPKSQL);
cstmt.execute(viewWithoutPKSQL);
}
return getRowKeyMatchersFromView(tenantConnection.unwrap(PhoenixConnection.class),
tenantViewName);
Expand Down Expand Up @@ -524,12 +552,14 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v
view.getTableName().getString());
byte[]
rowKeyMatcher1 =
WhereOptimizer.getRowKeyMatcher(viewStatementContext, tableName, viewStatementTable,
whereExpression);
PVarbinaryEncoded.INSTANCE.toBytes(
WhereOptimizer.getRowKeyMatcher(viewStatementContext, tableName,
viewStatementTable, whereExpression));
byte[]
rowKeyMatcher2 =
WhereOptimizer.getRowKeyMatcher(connection, tableName, viewStatementTable,
viewColumnConstantsToBe, isViewColumnReferencedToBe);
PVarbinaryEncoded.INSTANCE.toBytes(
WhereOptimizer.getRowKeyMatcher(connection, tableName, viewStatementTable,
viewColumnConstantsToBe, isViewColumnReferencedToBe));
LOGGER.debug(String.format(
"target-view-name = %s, physical = %s, stmt-table = %s\n, " +
"row-matcher-0 = %s (syscat)\n, row-matcher-1 = %s\n, row-matcher-2 = %s\n",
Expand Down Expand Up @@ -634,6 +664,17 @@ private String getWhereClause(String[] pkNames, PDataType[] testPKTypes) {
builder.append(pkNames[b]).append(" = ")
.append(" TO_TIMESTAMP('2019-10-27T16:17:57+00:00') ");
break;
case Types.VARBINARY:
// pkTypeStr = "VARBINARY";
case 9000:
// pkTypeStr = "VARBINARY_ENCODED";
byte[] varBytes = ByteUtil.concat(
RandomStringUtils.randomAlphanumeric(25).getBytes(),
Bytes.toBytes(rnd.nextInt(50000)),
Bytes.toBytes(Math.floor(rnd.nextInt(50000) * rnd.nextDouble())));
builder.append(pkNames[b]).append(" = ")
.append(PVarbinary.INSTANCE.toStringLiteral(varBytes));
break;
default:
// pkTypeStr = "VARCHAR(25)";
builder.append(pkNames[b]).append("=").append("'")
Expand Down Expand Up @@ -755,6 +796,8 @@ private List<PDataType[]> getTestCases() {
// Test Case 7: PK1 = Varchar, PK2 = Varchar, PK3 = Integer
// last PK cannot be of variable length when creating a view on top of it
testCases.add(new PDataType[] { PVarchar.INSTANCE, PVarchar.INSTANCE, PInteger.INSTANCE });
// Test Case 8: PK1 = VARBINARY_ENCODED, PK2 = Varchar, PK3 = VARBINARY_ENCODED
testCases.add(new PDataType[] { PVarbinaryEncoded.INSTANCE, PVarchar.INSTANCE, PVarbinaryEncoded.INSTANCE });

return testCases;
}
Expand Down
Loading