PHOENIX-7170 Conditional TTL

apache · Jan 30, 2025 · 0c2fec1 · 0c2fec1
1 parent d2fe17e
commit 0c2fec1
Show file tree

Hide file tree

Showing 42 changed files with 3,510 additions and 774 deletions.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/TableTTLInfo.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/TableTTLInfo.java
@@ -18,11 +18,14 @@
 
 package org.apache.phoenix.coprocessorclient;
 
-import org.apache.hadoop.hbase.util.Bytes;
-
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
 
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.schema.TTLExpression;
+
+import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
+
 /**
  * Simple POJO class to hold TTL info
  */
@@ -31,18 +34,38 @@ public class TableTTLInfo implements Comparable {
     private final byte[] tenantId;
     private final byte[] entityName;
     private final byte[] matchPattern;
-    private final int ttl;
+    private final TTLExpression ttl;
 
+    @VisibleForTesting
     public TableTTLInfo(String physicalTableName, String tenantId, String entityName, String matchPattern, int ttl) {
         super();
         this.physicalTableName = physicalTableName.getBytes(StandardCharsets.UTF_8);
         this.tenantId = tenantId.getBytes(StandardCharsets.UTF_8);
         this.entityName = entityName.getBytes(StandardCharsets.UTF_8);
         this.matchPattern = matchPattern.getBytes(StandardCharsets.UTF_8);
-        this.ttl = ttl;
+        this.ttl = TTLExpression.create(ttl);
     }
 
+    @VisibleForTesting
     public TableTTLInfo(byte[] physicalTableName, byte[] tenantId, byte[] entityName, byte[] matchPattern, int ttl) {
+        super();
+        this.physicalTableName = physicalTableName;
+        this.tenantId = tenantId;
+        this.matchPattern = matchPattern;
+        this.entityName = entityName;
+        this.ttl = TTLExpression.create(ttl);
+    }
+
+    public TableTTLInfo(String physicalTableName, String tenantId, String entityName, String matchPattern, TTLExpression ttl) {
+        super();
+        this.physicalTableName = physicalTableName.getBytes(StandardCharsets.UTF_8);
+        this.tenantId = tenantId.getBytes(StandardCharsets.UTF_8);
+        this.entityName = entityName.getBytes(StandardCharsets.UTF_8);
+        this.matchPattern = matchPattern.getBytes(StandardCharsets.UTF_8);
+        this.ttl = ttl;
+    }
+
+    public TableTTLInfo(byte[] physicalTableName, byte[] tenantId, byte[] entityName, byte[] matchPattern, TTLExpression ttl) {
         super();
         this.physicalTableName = physicalTableName;
         this.tenantId = tenantId;
@@ -51,7 +74,7 @@ public TableTTLInfo(byte[] physicalTableName, byte[] tenantId, byte[] entityName
         this.ttl = ttl;
     }
 
-    public int getTTL() {
+    public TTLExpression getTTL() {
         return ttl;
     }
     public byte[] getTenantId() {

diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -207,7 +207,13 @@ public SQLException newException(SQLExceptionInfo info) {
     INVALID_JSON_DATA(540, "42916", "Invalid json data."),
     JSON_FRAGMENT_NOT_ALLOWED_IN_INDEX_EXPRESSION(541, "42917",
             "Functions returning JSON fragments are not allowed in Index Expression."),
+    AGGREGATE_EXPRESSION_NOT_ALLOWED_IN_CONDITIONAL_TTL(542, "42918",
+            "Aggregate expression not allowed in a conditional TTL expression."),
+    CANNOT_SET_CONDITIONAL_TTL_ON_TABLE_WITH_MULTIPLE_COLUMN_FAMILIES(543, "42919",
+            "Cannot set conditional TTL on table with multiple column families."),
 
+    CANNOT_DROP_COL_REFERENCED_IN_CONDITIONAL_TTL(544, "42920",
+            "Cannot drop column referenced in conditional TTL expression."),
     /**
      * HBase and Phoenix specific implementation defined sub-classes.
      * Column family related exceptions.

diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java b/phoenix-core-client/src/main/java/org/apache/phoenix/iterate/BaseResultIterators.java
@@ -111,7 +111,6 @@
 import org.apache.phoenix.schema.stats.GuidePostsInfo;
 import org.apache.phoenix.schema.stats.GuidePostsKey;
 import org.apache.phoenix.schema.stats.StatisticsUtil;
-import org.apache.phoenix.schema.types.PVarbinaryEncoded;
 import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
 import org.apache.phoenix.util.ByteUtil;
 import org.apache.phoenix.util.ClientUtil;
@@ -202,6 +201,13 @@ private static void initializeScan(QueryPlan plan, Integer perScanLimit, Integer
                         DEFAULT_WILDCARD_QUERY_DYNAMIC_COLS_ATTRIB);
         PTable table = tableRef.getTable();
 
+        // If the table has Conditional TTL set, then we need to add all the non PK columns
+        // referenced in the conditional TTL expression to the scan. This can influence the
+        // filters that are applied to the scan so do this before the filter analysis.
+        if (table.hasConditionalTTL()) {
+            ScanUtil.addConditionalTTLColumnsToScan(scan, context.getConnection(), table);
+        }
+
         Map<byte [], NavigableSet<byte []>> familyMap = scan.getFamilyMap();
         // Hack for PHOENIX-2067 to force raw scan over all KeyValues to fix their row keys
         if (context.getConnection().isDescVarLengthRowKeyUpgrade()) {

diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -399,6 +399,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
     public static final byte[] TTL_BYTES = Bytes.toBytes(TTL);
     public static final int TTL_NOT_DEFINED = 0;
     public static final int DEFAULT_TTL = HConstants.FOREVER;
+    public static final String FOREVER_TTL = "FOREVER";
+    public static final String NONE_TTL = "NONE";
     public static final String PHOENIX_TTL = "PHOENIX_TTL";
     public static final byte[] PHOENIX_TTL_BYTES = Bytes.toBytes(PHOENIX_TTL);
     public static final String PHOENIX_TTL_HWM = "PHOENIX_TTL_HWM";

diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -270,6 +270,8 @@
 import org.apache.phoenix.schema.SequenceAllocation;
 import org.apache.phoenix.schema.SequenceKey;
 import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.TTLExpression;
+import org.apache.phoenix.schema.LiteralTTLExpression;
 import org.apache.phoenix.schema.TableAlreadyExistsException;
 import org.apache.phoenix.schema.TableNotFoundException;
 import org.apache.phoenix.schema.TableProperty;
@@ -2999,7 +3001,7 @@ private Map<TableDescriptor, TableDescriptor> separateAndValidateProperties(PTab
         boolean willBeTransactional = false;
         boolean isOrWillBeTransactional = isTransactional;
         Integer newTTL = null;
-        Integer newPhoenixTTL = null;
+        TTLExpression newPhoenixTTL = null;
         Integer newReplicationScope = null;
         KeepDeletedCells newKeepDeletedCells = null;
         TransactionFactory.Provider txProvider = null;
@@ -3045,14 +3047,17 @@ private Map<TableDescriptor, TableDescriptor> separateAndValidateProperties(PTab
                                 //If Phoenix level TTL is enabled we are using TTL as phoenix
                                 //Table level property.
                                 if (!isPhoenixTTLEnabled()) {
-                                    newTTL = ((Number) propValue).intValue();
+                                    // only literal TTL expression
+                                    LiteralTTLExpression ttlExpr =
+                                            (LiteralTTLExpression) TableProperty.TTL.getValue(propValue);
+                                    newTTL = ttlExpr != null ? ttlExpr.getTTLValue() : null;
                                     //Even though TTL is really a HColumnProperty we treat it
                                     //specially. We enforce that all CFs have the same TTL.
                                     commonFamilyProps.put(propName, propValue);
                                 } else {
                                     //Setting this here just to check if we need to throw Exception
                                     //for Transaction's SET_TTL Feature.
-                                    newPhoenixTTL = ((Number) propValue).intValue();
+                                    newPhoenixTTL = (TTLExpression) TableProperty.TTL.getValue(propValue);
                                 }
                             } else if (propName.equals(PhoenixDatabaseMetaData.TRANSACTIONAL) && Boolean.TRUE.equals(propValue)) {
                                 willBeTransactional = isOrWillBeTransactional = true;