From 748ec9a208ad0ee1a220be6b1fd5573784783396 Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Sat, 5 Oct 2024 00:17:55 +0530 Subject: [PATCH 1/4] [Compatibility] Added EXPIREAT and PEXPIREAT command and bug fixes for EXPIRE and PEXPIRE (#666) * Added EXPIREAT and PEXPIREAT command and bug fixes for EXPIRE and PEXPIRE * Code style fix * Review comment fixes * Fixed merge conflict and fixed review comments * Changed to _unixEpochTicks --- libs/common/ConvertUtils.cs | 29 +- libs/server/API/GarnetApi.cs | 12 + libs/server/API/IGarnetApi.cs | 26 + libs/server/ExpireOption.cs | 21 +- libs/server/Resp/KeyAdminCommands.cs | 118 +++- libs/server/Resp/Parser/RespCommand.cs | 10 + libs/server/Resp/RespCommandsInfo.json | 58 ++ libs/server/Resp/RespServerSession.cs | 3 + .../Functions/MainStore/PrivateMethods.cs | 12 +- .../Functions/ObjectStore/PrivateMethods.cs | 6 +- .../Storage/Session/MainStore/MainStoreOps.cs | 51 +- .../CommandInfoUpdater/SupportedCommand.cs | 2 + test/Garnet.test/Resp/ACL/RespCommandTests.cs | 92 ++- test/Garnet.test/RespTests.cs | 587 ++++++++++++++++++ website/docs/commands/api-compatibility.md | 4 +- website/docs/commands/generic-commands.md | 57 ++ 16 files changed, 1070 insertions(+), 18 deletions(-) diff --git a/libs/common/ConvertUtils.cs b/libs/common/ConvertUtils.cs index 3228d49e55..89ef9ec763 100644 --- a/libs/common/ConvertUtils.cs +++ b/libs/common/ConvertUtils.cs @@ -2,7 +2,7 @@ // Licensed under the MIT license. using System; -using System.Diagnostics; +using System.Runtime.CompilerServices; namespace Garnet.common { @@ -11,6 +11,11 @@ namespace Garnet.common /// public static class ConvertUtils { + /// + /// Contains the number of ticks representing 1970/1/1. Value is equal to new DateTime(1970, 1, 1).Ticks + /// + private static readonly long _unixEpochTicks = DateTimeOffset.UnixEpoch.Ticks; + /// /// Convert diff ticks - utcNow.ticks to seconds. /// @@ -43,5 +48,27 @@ public static long MillisecondsFromDiffUtcNowTicks(long ticks) } return milliseconds; } + + /// + /// Converts a Unix timestamp in seconds to ticks. + /// + /// The Unix timestamp in seconds. + /// The equivalent number of ticks. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long UnixTimestampInSecondsToTicks(long unixTimestamp) + { + return unixTimestamp * TimeSpan.TicksPerSecond + _unixEpochTicks; + } + + /// + /// Converts a Unix timestamp in milliseconds to ticks. + /// + /// The Unix timestamp in milliseconds. + /// The equivalent number of ticks. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static long UnixTimestampInMillisecondsToTicks(long unixTimestamp) + { + return unixTimestamp * TimeSpan.TicksPerMillisecond + _unixEpochTicks; + } } } \ No newline at end of file diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index 2217e662db..da6f90165c 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -181,6 +181,18 @@ public GarnetStatus PEXPIRE(ArgSlice key, TimeSpan expiry, out bool timeoutSet, #endregion + #region EXPIREAT + + /// + public GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None) + => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectContext); + + /// + public GarnetStatus PEXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None) + => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectContext, milliseconds: true); + + #endregion + #region PERSIST /// public unsafe GarnetStatus PERSIST(ArgSlice key, StoreType storeType = StoreType.All) diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 7527d35865..5b5f2997c8 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -176,6 +176,32 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi #endregion + #region EXPIREAT + + /// + /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in seconds + /// + /// Key + /// Absolute Unix timestamp in seconds + /// Whether timeout was set by the call + /// Store type: main, object, or both + /// Expire option + /// + GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None); + + /// + /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in milliseconds + /// + /// Key + /// Absolute Unix timestamp in milliseconds + /// Whether timeout was set by the call + /// Store type: main, object, or both + /// Expire option + /// + GarnetStatus PEXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None); + + #endregion + #region PERSIST /// /// PERSIST diff --git a/libs/server/ExpireOption.cs b/libs/server/ExpireOption.cs index d6eb7ef8f7..65b306e342 100644 --- a/libs/server/ExpireOption.cs +++ b/libs/server/ExpireOption.cs @@ -1,32 +1,43 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT license. +using System; + namespace Garnet.server { /// /// Expire option /// + [Flags] public enum ExpireOption : byte { /// /// None /// - None, + None = 0, /// /// Set expiry only when the key has no expiry /// - NX, + NX = 1 << 0, /// /// Set expiry only when the key has an existing expiry /// - XX, + XX = 1 << 1, /// /// Set expiry only when the new expiry is greater than current one /// - GT, + GT = 1 << 2, /// /// Set expiry only when the new expiry is less than current one /// - LT + LT = 1 << 3, + /// + /// Set expiry only when the key has an existing expiry and the new expiry is greater than current one + /// + XXGT = XX | GT, + /// + /// Set expiry only when the key has an existing expiry and the new expiry is less than current one + /// + XXLT = XX | LT, } } \ No newline at end of file diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index 97ce43ae62..029e2ca16b 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -174,7 +174,7 @@ private bool NetworkEXPIRE(RespCommand command, ref TGarnetApi stora where TGarnetApi : IGarnetApi { var count = parseState.Count; - if (count < 2 || count > 3) + if (count < 2 || count > 4) { return AbortWithWrongNumberOfArguments(nameof(RespCommand.EXPIRE)); } @@ -205,6 +205,36 @@ private bool NetworkEXPIRE(RespCommand command, ref TGarnetApi stora } } + if (parseState.Count > 3) + { + if (!TryGetExpireOption(parseState.GetArgSliceByRef(3).ReadOnlySpan, out var additionExpireOption)) + { + var optionStr = parseState.GetString(3); + + while (!RespWriteUtils.WriteError($"ERR Unsupported option {optionStr}", ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (expireOption == ExpireOption.XX && (additionExpireOption == ExpireOption.GT || additionExpireOption == ExpireOption.LT)) + { + expireOption = ExpireOption.XX | additionExpireOption; + } + else if (expireOption == ExpireOption.GT && additionExpireOption == ExpireOption.XX) + { + expireOption = ExpireOption.XXGT; + } + else if (expireOption == ExpireOption.LT && additionExpireOption == ExpireOption.XX) + { + expireOption = ExpireOption.XXLT; + } + else + { + while (!RespWriteUtils.WriteError("ERR NX and XX, GT or LT options at the same time are not compatible", ref dcurr, dend)) + SendAndReset(); + } + } + var status = command == RespCommand.EXPIRE ? storageApi.EXPIRE(key, expiryMs, out var timeoutSet, StoreType.All, expireOption) : storageApi.PEXPIRE(key, expiryMs, out timeoutSet, StoreType.All, expireOption); @@ -223,6 +253,92 @@ private bool NetworkEXPIRE(RespCommand command, ref TGarnetApi stora return true; } + /// + /// Set a timeout on a key based on unix timestamp + /// + /// + /// Indicates which command to use, expire or pexpire. + /// + /// + private bool NetworkEXPIREAT(RespCommand command, ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + var count = parseState.Count; + if (count < 2 || count > 4) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.EXPIREAT)); + } + + var key = parseState.GetArgSliceByRef(0); + if (!parseState.TryGetLong(1, out var expiryTimestamp)) + { + while (!RespWriteUtils.WriteError(CmdStrings.RESP_ERR_GENERIC_VALUE_IS_NOT_INTEGER, ref dcurr, dend)) + SendAndReset(); + return true; + } + + var expireOption = ExpireOption.None; + + if (parseState.Count > 2) + { + if (!TryGetExpireOption(parseState.GetArgSliceByRef(2).ReadOnlySpan, out expireOption)) + { + var optionStr = parseState.GetString(2); + + while (!RespWriteUtils.WriteError($"ERR Unsupported option {optionStr}", ref dcurr, dend)) + SendAndReset(); + return true; + } + } + + if (parseState.Count > 3) + { + if (!TryGetExpireOption(parseState.GetArgSliceByRef(3).ReadOnlySpan, out var additionExpireOption)) + { + var optionStr = parseState.GetString(3); + + while (!RespWriteUtils.WriteError($"ERR Unsupported option {optionStr}", ref dcurr, dend)) + SendAndReset(); + return true; + } + + if (expireOption == ExpireOption.XX && (additionExpireOption == ExpireOption.GT || additionExpireOption == ExpireOption.LT)) + { + expireOption = ExpireOption.XX | additionExpireOption; + } + else if (expireOption == ExpireOption.GT && additionExpireOption == ExpireOption.XX) + { + expireOption = ExpireOption.XXGT; + } + else if (expireOption == ExpireOption.LT && additionExpireOption == ExpireOption.XX) + { + expireOption = ExpireOption.XXLT; + } + else + { + while (!RespWriteUtils.WriteError("ERR NX and XX, GT or LT options at the same time are not compatible", ref dcurr, dend)) + SendAndReset(); + } + } + + var status = command == RespCommand.EXPIREAT ? + storageApi.EXPIREAT(key, expiryTimestamp, out var timeoutSet, StoreType.All, expireOption) : + storageApi.PEXPIREAT(key, expiryTimestamp, out timeoutSet, StoreType.All, expireOption); + + if (status == GarnetStatus.OK && timeoutSet) + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_RETURN_VAL_1, ref dcurr, dend)) + SendAndReset(); + } + else + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_RETURN_VAL_0, ref dcurr, dend)) + SendAndReset(); + } + + return true; + } + /// /// PERSIST command /// diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index ab36128081..efd8cf3df6 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -85,6 +85,7 @@ public enum RespCommand : byte DECRBY, DEL, EXPIRE, + EXPIREAT, FLUSHALL, FLUSHDB, GEOADD, @@ -114,6 +115,7 @@ public enum RespCommand : byte MSETNX, PERSIST, PEXPIRE, + PEXPIREAT, PFADD, PFMERGE, PSETEX, @@ -1252,6 +1254,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.BITFIELD; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("EXPIREAT"u8) && *(ushort*)(ptr + 12) == MemoryMarshal.Read("\r\n"u8)) + { + return RespCommand.EXPIREAT; + } break; case 9: if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("SUBSCRIB"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("BE\r\n"u8)) @@ -1278,6 +1284,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.RPOPLPUSH; } + else if (*(ulong*)(ptr + 4) == MemoryMarshal.Read("PEXPIREA"u8) && *(uint*)(ptr + 11) == MemoryMarshal.Read("AT\r\n"u8)) + { + return RespCommand.PEXPIREAT; + } break; } diff --git a/libs/server/Resp/RespCommandsInfo.json b/libs/server/Resp/RespCommandsInfo.json index aed450d1d1..b6c5265760 100644 --- a/libs/server/Resp/RespCommandsInfo.json +++ b/libs/server/Resp/RespCommandsInfo.json @@ -1594,6 +1594,35 @@ ], "SubCommands": null }, + { + "Command": "EXPIREAT", + "Name": "EXPIREAT", + "IsInternal": false, + "Arity": -3, + "Flags": "Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, KeySpace, Write", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Notes": null, + "Flags": "RW, Update" + } + ], + "SubCommands": null + }, { "Command": "FAILOVER", "Name": "FAILOVER", @@ -3283,6 +3312,35 @@ ], "SubCommands": null }, + { + "Command": "PEXPIREAT", + "Name": "PEXPIREAT", + "IsInternal": false, + "Arity": -3, + "Flags": "Fast, Write", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, KeySpace, Write", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Notes": null, + "Flags": "RW, Update" + } + ], + "SubCommands": null + }, { "Command": "PFADD", "Name": "PFADD", diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index d611922a88..848a0edb3d 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -543,6 +543,9 @@ private bool ProcessBasicCommands(RespCommand cmd, ref TGarnetApi st RespCommand.RUNTXP => NetworkRUNTXP(), RespCommand.READONLY => NetworkREADONLY(), RespCommand.READWRITE => NetworkREADWRITE(), + RespCommand.EXPIREAT => NetworkEXPIREAT(RespCommand.EXPIREAT, ref storageApi), + RespCommand.PEXPIREAT => NetworkEXPIREAT(RespCommand.PEXPIREAT, ref storageApi), + _ => ProcessArrayCommands(cmd, ref storageApi) }; diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index c24563ef50..193a125f43 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -238,6 +238,7 @@ bool EvaluateExpireInPlace(ExpireOption optionType, bool expiryExists, ref SpanB o->result1 = 1; break; case ExpireOption.GT: + case ExpireOption.XXGT: bool replace = input.ExtraMetadata < value.ExtraMetadata; value.ExtraMetadata = replace ? value.ExtraMetadata : input.ExtraMetadata; if (replace) @@ -246,6 +247,7 @@ bool EvaluateExpireInPlace(ExpireOption optionType, bool expiryExists, ref SpanB o->result1 = 1; break; case ExpireOption.LT: + case ExpireOption.XXLT: replace = input.ExtraMetadata > value.ExtraMetadata; value.ExtraMetadata = replace ? value.ExtraMetadata : input.ExtraMetadata; if (replace) @@ -264,10 +266,12 @@ bool EvaluateExpireInPlace(ExpireOption optionType, bool expiryExists, ref SpanB { case ExpireOption.NX: case ExpireOption.None: + case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite return false; case ExpireOption.XX: case ExpireOption.GT: - case ExpireOption.LT: + case ExpireOption.XXGT: + case ExpireOption.XXLT: o->result1 = 0; return true; default: @@ -293,6 +297,7 @@ void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, ref Sp o->result1 = 1; break; case ExpireOption.GT: + case ExpireOption.XXGT: oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); bool replace = input.ExtraMetadata < oldValue.ExtraMetadata; newValue.ExtraMetadata = replace ? oldValue.ExtraMetadata : input.ExtraMetadata; @@ -302,6 +307,7 @@ void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, ref Sp o->result1 = 1; break; case ExpireOption.LT: + case ExpireOption.XXLT: oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); replace = input.ExtraMetadata > oldValue.ExtraMetadata; newValue.ExtraMetadata = replace ? oldValue.ExtraMetadata : input.ExtraMetadata; @@ -318,13 +324,15 @@ void EvaluateExpireCopyUpdate(ExpireOption optionType, bool expiryExists, ref Sp { case ExpireOption.NX: case ExpireOption.None: + case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite newValue.ExtraMetadata = input.ExtraMetadata; oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); o->result1 = 1; break; case ExpireOption.XX: case ExpireOption.GT: - case ExpireOption.LT: + case ExpireOption.XXGT: + case ExpireOption.XXLT: oldValue.AsReadOnlySpan().CopyTo(newValue.AsSpan()); o->result1 = 0; break; diff --git a/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs b/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs index ef5c82cc80..8a96900757 100644 --- a/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/PrivateMethods.cs @@ -150,6 +150,7 @@ static bool EvaluateObjectExpireInPlace(ExpireOption optionType, bool expiryExis o->result1 = 1; break; case ExpireOption.GT: + case ExpireOption.XXGT: bool replace = expiration < value.Expiration; value.Expiration = replace ? value.Expiration : expiration; if (replace) @@ -158,6 +159,7 @@ static bool EvaluateObjectExpireInPlace(ExpireOption optionType, bool expiryExis o->result1 = 1; break; case ExpireOption.LT: + case ExpireOption.XXLT: replace = expiration > value.Expiration; value.Expiration = replace ? value.Expiration : expiration; if (replace) @@ -175,12 +177,14 @@ static bool EvaluateObjectExpireInPlace(ExpireOption optionType, bool expiryExis { case ExpireOption.NX: case ExpireOption.None: + case ExpireOption.LT: // If expiry doesn't exist, LT should treat the current expiration as infinite value.Expiration = expiration; o->result1 = 1; break; case ExpireOption.XX: case ExpireOption.GT: - case ExpireOption.LT: + case ExpireOption.XXGT: + case ExpireOption.XXLT: o->result1 = 0; break; default: diff --git a/libs/server/Storage/Session/MainStore/MainStoreOps.cs b/libs/server/Storage/Session/MainStore/MainStoreOps.cs index eed91fe0a0..12637006e5 100644 --- a/libs/server/Storage/Session/MainStore/MainStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/MainStoreOps.cs @@ -788,20 +788,63 @@ public unsafe GarnetStatus EXPIRE(ArgSlice key, ArgSli /// Basic context for the main store /// Object context for the object store /// When true the command executed is PEXPIRE, expire by default. - /// + /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND public unsafe GarnetStatus EXPIRE(ArgSlice key, TimeSpan expiry, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext, bool milliseconds = false) where TContext : ITsavoriteContext where TObjectContext : ITsavoriteContext + { + return EXPIRE(key, DateTimeOffset.UtcNow.Ticks + expiry.Ticks, out timeoutSet, storeType, expireOption, ref context, ref objectStoreContext, milliseconds ? RespCommand.PEXPIRE : RespCommand.EXPIRE); + } + + /// + /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970). + /// + /// + /// + /// The key to set the timeout on. + /// Absolute Unix timestamp + /// True when the timeout was properly set. + /// The store to operate on. + /// Flags to use for the operation. + /// Basic context for the main store + /// Object context for the object store + /// When true, is treated as milliseconds else seconds + /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND + public unsafe GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext, bool milliseconds = false) + where TContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + { + var expiryTimestampTicks = milliseconds ? ConvertUtils.UnixTimestampInMillisecondsToTicks(expiryTimestamp) : ConvertUtils.UnixTimestampInSecondsToTicks(expiryTimestamp); + return EXPIRE(key, expiryTimestampTicks, out timeoutSet, storeType, expireOption, ref context, ref objectStoreContext, milliseconds ? RespCommand.PEXPIRE : RespCommand.EXPIRE); + } + + /// + /// Set a timeout on key using ticks. + /// + /// + /// + /// The key to set the timeout on. + /// The timestamp in ticks + /// True when the timeout was properly set. + /// The store to operate on. + /// Flags to use for the operation. + /// Basic context for the main store + /// Object context for the object store + /// Resp Command to be executed. + /// Return GarnetStatus.OK when key found, else GarnetStatus.NOTFOUND + private unsafe GarnetStatus EXPIRE(ArgSlice key, long expiryInTicks, out bool timeoutSet, StoreType storeType, ExpireOption expireOption, ref TContext context, ref TObjectContext objectStoreContext, RespCommand respCommand) + where TContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext { byte* pbCmdInput = stackalloc byte[sizeof(int) + sizeof(long) + RespInputHeader.Size + sizeof(byte)]; *(int*)pbCmdInput = sizeof(long) + RespInputHeader.Size; - ((RespInputHeader*)(pbCmdInput + sizeof(int) + sizeof(long)))->cmd = milliseconds ? RespCommand.PEXPIRE : RespCommand.EXPIRE; + ((RespInputHeader*)(pbCmdInput + sizeof(int) + sizeof(long)))->cmd = respCommand; ((RespInputHeader*)(pbCmdInput + sizeof(int) + sizeof(long)))->flags = 0; *(pbCmdInput + sizeof(int) + sizeof(long) + RespInputHeader.Size) = (byte)expireOption; ref var input = ref SpanByte.Reinterpret(pbCmdInput); - input.ExtraMetadata = DateTimeOffset.UtcNow.Ticks + expiry.Ticks; + input.ExtraMetadata = expiryInTicks; var rmwOutput = stackalloc byte[ObjectOutputHeader.Size]; var output = new SpanByteAndMemory(SpanByte.FromPinnedPointer(rmwOutput, ObjectOutputHeader.Size)); @@ -843,7 +886,7 @@ public unsafe GarnetStatus EXPIRE(ArgSlice key, TimeSp { header = new RespInputHeader { - cmd = milliseconds ? RespCommand.PEXPIRE : RespCommand.EXPIRE, + cmd = respCommand, type = GarnetObjectType.Expire, }, parseState = parseState, diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 1e00233072..48176aabc5 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -111,6 +111,7 @@ public class SupportedCommand new("EXEC", RespCommand.EXEC), new("EXISTS", RespCommand.EXISTS), new("EXPIRE", RespCommand.EXPIRE), + new("EXPIREAT", RespCommand.EXPIREAT), new("FAILOVER", RespCommand.FAILOVER), new("FLUSHALL", RespCommand.FLUSHALL), new("FLUSHDB", RespCommand.FLUSHDB), @@ -182,6 +183,7 @@ public class SupportedCommand new("MULTI", RespCommand.MULTI), new("PERSIST", RespCommand.PERSIST), new("PEXPIRE", RespCommand.PEXPIRE), + new("PEXPIREAT", RespCommand.PEXPIREAT), new("PFADD", RespCommand.PFADD), new("PFCOUNT", RespCommand.PFCOUNT), new("PFMERGE", RespCommand.PFMERGE), diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index 8c755b0074..758ad68d30 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -2604,8 +2604,6 @@ static async Task DoExistsMultiAsync(GarnetClient client) [Test] public async Task ExpireACLsAsync() { - // TODO: expire doesn't support combinations of flags (XX GT, XX LT are legal) so those will need to be tested when implemented - await CheckCommandsAsync( "EXPIRE", [DoExpireAsync, DoExpireNXAsync, DoExpireXXAsync, DoExpireGTAsync, DoExpireLTAsync] @@ -2642,6 +2640,96 @@ static async Task DoExpireLTAsync(GarnetClient client) } } + [Test] + public async Task ExpireAtACLsAsync() + { + await CheckCommandsAsync( + "EXPIREAT", + [DoExpireAsync, DoExpireNXAsync, DoExpireXXAsync, DoExpireGTAsync, DoExpireLTAsync] + ); + + + static async Task DoExpireAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("EXPIREAT", ["foo", expireTimestamp]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireNXAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("EXPIREAT", ["foo", "10", "NX"]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireXXAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("EXPIREAT", ["foo", "10", "XX"]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireGTAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("EXPIREAT", ["foo", "10", "GT"]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireLTAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("EXPIREAT", ["foo", "10", "LT"]); + ClassicAssert.AreEqual(0, val); + } + } + + [Test] + public async Task PExpireAtACLsAsync() + { + await CheckCommandsAsync( + "PEXPIREAT", + [DoExpireAsync, DoExpireNXAsync, DoExpireXXAsync, DoExpireGTAsync, DoExpireLTAsync] + ); + + + static async Task DoExpireAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeMilliseconds().ToString(); + long val = await client.ExecuteForLongResultAsync("PEXPIREAT", ["foo", expireTimestamp]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireNXAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("PEXPIREAT", ["foo", "10", "NX"]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireXXAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("PEXPIREAT", ["foo", "10", "XX"]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireGTAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("PEXPIREAT", ["foo", "10", "GT"]); + ClassicAssert.AreEqual(0, val); + } + + static async Task DoExpireLTAsync(GarnetClient client) + { + var expireTimestamp = DateTimeOffset.UtcNow.AddMinutes(1).ToUnixTimeSeconds().ToString(); + long val = await client.ExecuteForLongResultAsync("PEXPIREAT", ["foo", "10", "LT"]); + ClassicAssert.AreEqual(0, val); + } + } + [Test] public async Task FailoverACLsAsync() { diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index ac5a60fff8..3c5aba514a 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -1912,6 +1912,593 @@ public void KeyExpireBadOptionTests(string command) } } + #region ExpireAt + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithStringAndObject(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + + var actualResult = (int)db.Execute(command, "key", expireTimeUnix); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT")] + [TestCase("PEXPIREAT")] + public void KeyExpireAtWithUnknownKey(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix); + ClassicAssert.AreEqual(actualResult, 0); + } + + [Test] + [TestCase("EXPIREAT")] + [TestCase("PEXPIREAT")] + public void KeyExpireAtWithoutArgs(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + + Assert.Throws(() => db.Execute(command, key)); + } + + [Test] + [TestCase("EXPIREAT")] + [TestCase("PEXPIREAT")] + public void KeyExpireAtWithUnknownArgs(string command) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + Assert.Throws(() => db.Execute(command, key, expireTimeUnix, "YY")); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithNxOptionAndKeyHasExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(1); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(10); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "Nx"); + ClassicAssert.AreEqual(actualResult, 0); + + // Test if the existing expiry time is still the same + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithNxOptionAndKeyHasNoExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var expireTimeSpan = TimeSpan.FromMinutes(10); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "nX"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxOptionAndKeyHasExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(1); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(10); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "Xx"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxOptionAndKeyHasNoExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var expireTimeSpan = TimeSpan.FromMinutes(10); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "xX"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsFalse(actualTtl.HasValue); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithGtOptionAndExistingKeyHasSmallerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(1); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(10); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "gT"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithGtOptionAndExistingKeyHasLargerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(10); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "Gt"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithGtOptionAndExistingKeyNoExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "GT"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsFalse(actualTtl.HasValue); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxAndGtOptionAndExistingKeyHasSmallerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(1); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(10); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "xx", "GT"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxAndGtOptionAndExistingKeyHasLargerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(10); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "gt", "XX"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxAndGtOptionAndExistingKeyNoExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "Gt", "xX"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsFalse(actualTtl.HasValue); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithLtOptionAndExistingKeyHasSmallerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(1); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(10); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "lT"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithLtOptionAndExistingKeyHasLargerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(10); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "LT"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithLtOptionAndExistingKeyNoExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "LT"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxAndLtOptionAndExistingKeyHasSmallerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(1); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(10); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "LT", "XX"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, existingExpireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxAndLtOptionAndExistingKeyHasLargerExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + var existingExpireTimeSpan = TimeSpan.FromMinutes(10); + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + db.KeyExpire(key, existingExpireTimeSpan); + } + else + { + db.StringSet(key, "valueA", existingExpireTimeSpan); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "xX", "Lt"); + ClassicAssert.AreEqual(actualResult, 1); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsTrue(actualTtl.HasValue); + ClassicAssert.Greater(actualTtl.Value.TotalMilliseconds, 0); + ClassicAssert.LessOrEqual(actualTtl.Value.TotalMilliseconds, expireTimeSpan.TotalMilliseconds); + } + + [Test] + [TestCase("EXPIREAT", false)] + [TestCase("EXPIREAT", true)] + [TestCase("PEXPIREAT", false)] + [TestCase("PEXPIREAT", true)] + public void KeyExpireAtWithXxAndLtOptionAndExistingKeyNoExpire(string command, bool isObject) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + if (isObject) + { + db.SortedSetAdd(key, [new SortedSetEntry("element", 1.0)]); + } + else + { + db.StringSet(key, "valueA"); + } + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + var actualResult = (int)db.Execute(command, key, expireTimeUnix, "XX", "LT"); + ClassicAssert.AreEqual(actualResult, 0); + + var actualTtl = db.KeyTimeToLive(key); + ClassicAssert.IsFalse(actualTtl.HasValue); + } + + [Test] + [TestCase("EXPIREAT", "XX", "NX")] + [TestCase("EXPIREAT", "NX", "GT")] + [TestCase("EXPIREAT", "LT", "NX")] + [TestCase("PEXPIREAT", "XX", "NX")] + [TestCase("PEXPIREAT", "NX", "GT")] + [TestCase("PEXPIREAT", "LT", "NX")] + public void KeyExpireAtWithInvalidOptionCombination(string command, string optionA, string optionB) + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key"; + db.StringSet(key, "valueA"); + var expireTimeSpan = TimeSpan.FromMinutes(1); + var expireTimeUnix = command == "EXPIREAT" ? DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds() : DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + + Assert.Throws(() => db.Execute(command, key, expireTimeUnix, optionA, optionA)); + } + + #endregion + [Test] public async Task ReAddExpiredKey() { diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index 31eec09a7c..febf726cba 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -135,7 +135,7 @@ Note that this list is subject to change as we continue to expand our API comman | | STATS | ➖ | | **GENERIC** | [PERSIST](generic-commands.md#persist) | ➕ | | | | [PEXPIRE](generic-commands.md#pexpire) | ➕ | | -| | PEXPIREAT | ➖ | | +| | [PEXPIREAT](generic-commands.md#pexpireat) | ➕ | | | | PEXPIRETIME | ➖ | | | | [PTTL](generic-commands.md#pttl) | ➕ | | | | RANDOMKEY | ➖ | | @@ -196,7 +196,7 @@ Note that this list is subject to change as we continue to expand our API comman | | DUMP | ➖ | | | | [EXISTS](generic-commands.md#exists) | ➕ | | | | [EXPIRE](generic-commands.md#expire) | ➕ | | -| | EXPIREAT | ➖ | | +| | [EXPIREAT](generic-commands.md#expireat) | ➕ | | | | EXPIRETIME | ➖ | | | | [KEYS](generic-commands.md#keys) | ➕ | | | | [MIGRATE](generic-commands.md#migrate) | ➕ | | diff --git a/website/docs/commands/generic-commands.md b/website/docs/commands/generic-commands.md index 7b56261700..850130f84b 100644 --- a/website/docs/commands/generic-commands.md +++ b/website/docs/commands/generic-commands.md @@ -166,12 +166,41 @@ The EXPIRE command supports a set of options: * `GT` -- Set expiry only when the new expiry is greater than current one * `LT` -- Set expiry only when the new expiry is less than current one +The GT, LT and NX options are mutually exclusive. + #### Resp Reply One of the following: * Integer reply: 0 if the timeout was not set; for example, the key doesn't exist, or the operation was skipped because of the provided arguments. +* Integer reply: 1 if the timeout was set. + +--- + +### EXPIREAT + +#### Syntax + +```bash + EXPIREAT key seconds [NX | XX | GT | LT] +``` + +Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in seconds. After the timestamp, the key will automatically be deleted. + +The EXPIREAT command supports a set of options: +* `NX` -- Set expiry only when the key has no expiry +* `XX` -- Set expiry only when the key has an existing expiry +* `GT` -- Set expiry only when the new expiry is greater than current one +* `LT` -- Set expiry only when the new expiry is less than current one + +The GT, LT and NX options are mutually exclusive. + +#### Resp Reply + +One of the following: + +* Integer reply: 0 if the timeout was not set; for example, the key doesn't exist, or the operation was skipped because of the provided arguments. * Integer reply: 1 if the timeout was set. --- @@ -242,8 +271,36 @@ One of the following: * Integer reply: 0 if key does not exist or does not have an associated timeout. * Integer reply: 1 if the timeout has been removed. +--- + +### PEXPIREAT + +#### Syntax + +```bash + PEXPIREAT key seconds [NX | XX | GT | LT] +``` + +Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in milliseconds. After the timestamp, the key will automatically be deleted. + +The PEXPIREAT command supports a set of options: + +* `NX` -- Set expiry only when the key has no expiry +* `XX` -- Set expiry only when the key has an existing expiry +* `GT` -- Set expiry only when the new expiry is greater than current one +* `LT` -- Set expiry only when the new expiry is less than current one + +The GT, LT and NX options are mutually exclusive. + +#### Resp Reply + +One of the following: + +* Integer reply: 0 if the timeout was not set; for example, the key doesn't exist, or the operation was skipped because of the provided arguments. +* Integer reply: 1 if the timeout was set. --- + ### PTTL #### Syntax From 28328d028c53261b494e9b7e801bbb1a8d975506 Mon Sep 17 00:00:00 2001 From: Tejas Kulkarni Date: Sat, 5 Oct 2024 01:15:35 +0530 Subject: [PATCH 2/4] Server side transaction failure in Cluster Mode (#702) * Removing IClusterSession as it can be fetched through respserver session. It's not set and results in object reference null exception for cluster mode transactions * Version bumped. --------- Co-authored-by: Badrish Chandramouli --- .azure/pipelines/azure-pipelines-external-release.yml | 2 +- libs/host/GarnetServer.cs | 2 +- libs/server/Transaction/TransactionManager.cs | 4 ---- libs/server/Transaction/TxnKeyManager.cs | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.azure/pipelines/azure-pipelines-external-release.yml b/.azure/pipelines/azure-pipelines-external-release.yml index a223f020b0..a7de9c0fcd 100644 --- a/.azure/pipelines/azure-pipelines-external-release.yml +++ b/.azure/pipelines/azure-pipelines-external-release.yml @@ -3,7 +3,7 @@ # 1) update the name: string below (line 6) -- this is the version for the nuget package (e.g. 1.0.0) # 2) update \libs\host\GarnetServer.cs readonly string version (~line 53) -- NOTE - these two values need to be the same ###################################### -name: 1.0.29 +name: 1.0.30 trigger: branches: include: diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index fcc2333045..86c01cd888 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -52,7 +52,7 @@ public class GarnetServer : IDisposable protected StoreWrapper storeWrapper; // IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6. - readonly string version = "1.0.29"; + readonly string version = "1.0.30"; /// /// Resp protocol version diff --git a/libs/server/Transaction/TransactionManager.cs b/libs/server/Transaction/TransactionManager.cs index e35fc15b1e..9123eaf1f7 100644 --- a/libs/server/Transaction/TransactionManager.cs +++ b/libs/server/Transaction/TransactionManager.cs @@ -57,9 +57,6 @@ public sealed unsafe partial class TransactionManager // Not readonly to avoid defensive copy GarnetWatchApi garnetTxPrepareApi; - // Cluster session - IClusterSession clusterSession; - // Not readonly to avoid defensive copy LockableGarnetApi garnetTxMainApi; @@ -118,7 +115,6 @@ internal TransactionManager( this.logger = logger; this.respSession = respSession; - this.clusterSession = respSession.clusterSession; watchContainer = new WatchedKeysContainer(initialSliceBufferSize, functionsState.watchVersionMap); keyEntries = new TxnKeyEntries(initialSliceBufferSize, lockableContext, objectStoreLockableContext); diff --git a/libs/server/Transaction/TxnKeyManager.cs b/libs/server/Transaction/TxnKeyManager.cs index be90499b1d..0dec851736 100644 --- a/libs/server/Transaction/TxnKeyManager.cs +++ b/libs/server/Transaction/TxnKeyManager.cs @@ -31,7 +31,7 @@ public unsafe void VerifyKeyOwnership(ArgSlice key, LockType type) if (!clusterEnabled) return; bool readOnly = type == LockType.Shared; - if (!clusterSession.CheckSingleKeySlotVerify(key, readOnly, respSession.SessionAsking)) + if (!respSession.clusterSession.CheckSingleKeySlotVerify(key, readOnly, respSession.SessionAsking)) { this.state = TxnState.Aborted; return; From 7335d931564b2d88d82246bd607b173428b8d5c6 Mon Sep 17 00:00:00 2001 From: Yoganand Rajasekaran <60369795+yrajas@users.noreply.github.com> Date: Fri, 4 Oct 2024 13:20:51 -0700 Subject: [PATCH 3/4] [Breaking change] Simplify object store heap size configuration (#697) **Breaks the previously available startup options --obj-memory and --obj-total-memory** This change simplifies configuring object store heap size for continuous monitoring. Instead of configuring total object store size, memory constraints for object store can now be configured using the following simple settings. obj-index - Size of Object store index obj-log-memory - Size of object store log that contains references to heap objects (Previously available as --obj-memory) obj-heap-memory - Size of object store heap Previously available --obj-total-memory option is now removed as object store memory can now be controlled using the above options in a fine grained manner. Total object store memory footprint = obj-index + obj-log-memory + obj-heap-memory. No changes to how the main store size is configured. * Simplified object store memory configuration * Added additional details on GC memory to INFO MEMORY command output. This also includes memory that is held by GC not yet released back to the system. * Bug fix to account for null IGarnetObject. * Updated documentation. --- libs/host/Configuration/Options.cs | 12 +++---- libs/host/GarnetServer.cs | 14 ++++++-- libs/host/defaults.conf | 10 +++--- libs/server/Metrics/Info/GarnetInfoMetrics.cs | 32 ++++++++++++++----- libs/server/Servers/GarnetServerOptions.cs | 10 +++--- .../Functions/ObjectStore/RMWMethods.cs | 5 ++- .../Storage/SizeTracker/CacheSizeTracker.cs | 16 +++------- .../cs/src/core/Allocator/AllocatorBase.cs | 3 ++ .../src/core/Index/Common/LogSizeTracker.cs | 14 ++++---- .../src/core/Index/Tsavorite/LogAccessor.cs | 5 +++ .../cs/src/core/TsavoriteLog/TsavoriteLog.cs | 10 ++++++ test/Garnet.test/CacheSizeTrackerTests.cs | 2 +- test/Garnet.test/RespAdminCommandsTests.cs | 2 +- test/Garnet.test/TestUtils.cs | 6 ++-- website/docs/getting-started/configuration.md | 8 ++--- website/docs/getting-started/memory.md | 11 +++---- 16 files changed, 99 insertions(+), 61 deletions(-) diff --git a/libs/host/Configuration/Options.cs b/libs/host/Configuration/Options.cs index 991beb9755..1bae3a7d84 100644 --- a/libs/host/Configuration/Options.cs +++ b/libs/host/Configuration/Options.cs @@ -52,7 +52,7 @@ internal sealed class Options public string SegmentSize { get; set; } [MemorySizeValidation] - [Option('i', "index", Required = false, HelpText = "Size of hash index in bytes (rounds down to power of 2)")] + [Option('i', "index", Required = false, HelpText = "Start size of hash index in bytes (rounds down to power of 2)")] public string IndexSize { get; set; } [MemorySizeValidation(false)] @@ -64,11 +64,11 @@ internal sealed class Options public int MutablePercent { get; set; } [MemorySizeValidation(false)] - [Option("obj-total-memory", Required = false, HelpText = "Total object store log memory used including heap memory in bytes")] - public string ObjectStoreTotalMemorySize { get; set; } + [Option("obj-heap-memory", Required = false, HelpText = "Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap)")] + public string ObjectStoreHeapMemorySize { get; set; } [MemorySizeValidation] - [Option("obj-memory", Required = false, HelpText = "Object store log memory used in bytes excluding heap memory")] + [Option("obj-log-memory", Required = false, HelpText = "Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log)")] public string ObjectStoreLogMemorySize { get; set; } [MemorySizeValidation] @@ -80,7 +80,7 @@ internal sealed class Options public string ObjectStoreSegmentSize { get; set; } [MemorySizeValidation] - [Option("obj-index", Required = false, HelpText = "Size of object store hash index in bytes (rounds down to power of 2)")] + [Option("obj-index", Required = false, HelpText = "Start size of object store hash index in bytes (rounds down to power of 2)")] public string ObjectStoreIndexSize { get; set; } [MemorySizeValidation(false)] @@ -575,7 +575,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null) IndexSize = IndexSize, IndexMaxSize = IndexMaxSize, MutablePercent = MutablePercent, - ObjectStoreTotalMemorySize = ObjectStoreTotalMemorySize, + ObjectStoreHeapMemorySize = ObjectStoreHeapMemorySize, ObjectStoreLogMemorySize = ObjectStoreLogMemorySize, ObjectStorePageSize = ObjectStorePageSize, ObjectStoreSegmentSize = ObjectStoreSegmentSize, diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs index 86c01cd888..ce81c77b10 100644 --- a/libs/host/GarnetServer.cs +++ b/libs/host/GarnetServer.cs @@ -198,6 +198,14 @@ private void InitializeServer() logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled"); + if (logger != null) + { + var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0); + if (objectStore != null) + configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0); + logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit); + } + // Create Garnet TCP server if none was provided. this.server ??= new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, logger); @@ -274,7 +282,7 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana if (!opts.DisableObjects) { objKvSettings = opts.GetObjectStoreSettings(this.loggerFactory?.CreateLogger("TsavoriteKV [obj]"), - out var objTotalMemorySize); + out var objHeapMemorySize); // Run checkpoint on its own thread to control p99 objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs; @@ -296,8 +304,8 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana () => new GarnetObjectSerializer(customCommandManager)) , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions)); - if (objTotalMemorySize > 0) - objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objTotalMemorySize, + if (objHeapMemorySize > 0) + objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize, this.loggerFactory); } } diff --git a/libs/host/defaults.conf b/libs/host/defaults.conf index cb8ce793d9..27e700c966 100644 --- a/libs/host/defaults.conf +++ b/libs/host/defaults.conf @@ -18,7 +18,7 @@ /* Size of each log segment in bytes on disk (rounds down to power of 2) */ "SegmentSize" : "1g", - /* Size of hash index in bytes (rounds down to power of 2) */ + /* Start size of hash index in bytes (rounds down to power of 2) */ "IndexSize" : "128m", /* Max size of hash index in bytes (rounds down to power of 2) */ @@ -27,10 +27,10 @@ /* Percentage of log memory that is kept mutable */ "MutablePercent" : 90, - /* Total object store log memory used including heap memory in bytes */ - "ObjectStoreTotalMemorySize" : "", + /* Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap) */ + "ObjectStoreHeapMemorySize" : "", - /* Object store log memory used in bytes excluding heap memory */ + /* Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log) */ "ObjectStoreLogMemorySize" : "32m", /* Size of each object store page in bytes (rounds down to power of 2) */ @@ -39,7 +39,7 @@ /* Size of each object store log segment in bytes on disk (rounds down to power of 2) */ "ObjectStoreSegmentSize" : "32m", - /* Size of object store hash index in bytes (rounds down to power of 2) */ + /* Start size of object store hash index in bytes (rounds down to power of 2) */ "ObjectStoreIndexSize" : "16m", /* Max size of object store hash index in bytes (rounds down to power of 2) */ diff --git a/libs/server/Metrics/Info/GarnetInfoMetrics.cs b/libs/server/Metrics/Info/GarnetInfoMetrics.cs index 4af1fb53de..9ec1758cc5 100644 --- a/libs/server/Metrics/Info/GarnetInfoMetrics.cs +++ b/libs/server/Metrics/Info/GarnetInfoMetrics.cs @@ -69,19 +69,28 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper) var total_main_store_size = main_store_index_size + main_store_log_memory_size + main_store_read_cache_size; var object_store_index_size = -1L; - var object_store_log_memory_references_size = -1L; - var object_store_read_cache_size = -1L; + var object_store_log_memory_size = -1L; + var object_store_read_cache_log_memory_size = -1L; + var object_store_heap_memory_size = -1L; + var object_store_read_cache_heap_memory_size = -1L; var total_object_store_size = -1L; var disableObj = storeWrapper.serverOptions.DisableObjects; + var aof_log_memory_size = storeWrapper.appendOnlyFile?.MemorySizeBytes ?? -1; + if (!disableObj) { object_store_index_size = storeWrapper.objectStore.IndexSize * 64; - object_store_log_memory_references_size = storeWrapper.objectStore.Log.MemorySizeBytes; - object_store_read_cache_size = (storeWrapper.objectStore.ReadCache != null ? storeWrapper.objectStore.ReadCache.MemorySizeBytes : 0); - total_object_store_size = object_store_index_size + object_store_log_memory_references_size + object_store_read_cache_size; + object_store_log_memory_size = storeWrapper.objectStore.Log.MemorySizeBytes; + object_store_read_cache_log_memory_size = storeWrapper.objectStore.ReadCache?.MemorySizeBytes ?? 0; + object_store_heap_memory_size = storeWrapper.objectStoreSizeTracker?.mainLogTracker.LogHeapSizeBytes ?? 0; + object_store_read_cache_heap_memory_size = storeWrapper.objectStoreSizeTracker?.readCacheTracker?.LogHeapSizeBytes ?? 0; + total_object_store_size = object_store_index_size + object_store_log_memory_size + object_store_read_cache_log_memory_size + object_store_heap_memory_size + object_store_read_cache_heap_memory_size; } + var gcMemoryInfo = GC.GetGCMemoryInfo(); + var gcAvailableMemory = gcMemoryInfo.TotalCommittedBytes - gcMemoryInfo.HeapSizeBytes; + memoryInfo = [ new("system_page_size", Environment.SystemPageSize.ToString()), @@ -105,14 +114,21 @@ private void PopulateMemoryInfo(StoreWrapper storeWrapper) new("proc_physical_memory_size(MB)", SystemMetrics.GetPhysicalMemoryUsage(1 << 20).ToString()), new("proc_peak_physical_memory_size", SystemMetrics.GetPeakPhysicalMemoryUsage().ToString()), new("proc_peak_physical_memory_size(MB)", SystemMetrics.GetPeakPhysicalMemoryUsage(1 << 20).ToString()), + new("gc_committed_bytes", gcMemoryInfo.TotalCommittedBytes.ToString()), + new("gc_heap_bytes", gcMemoryInfo.HeapSizeBytes.ToString()), + new("gc_managed_memory_bytes_excluding_heap", gcAvailableMemory.ToString()), + new("gc_fragmented_bytes", gcMemoryInfo.FragmentedBytes.ToString()), new("main_store_index_size", main_store_index_size.ToString()), new("main_store_log_memory_size", main_store_log_memory_size.ToString()), new("main_store_read_cache_size", main_store_read_cache_size.ToString()), new("total_main_store_size", total_main_store_size.ToString()), new("object_store_index_size", object_store_index_size.ToString()), - new("object_store_log_memory_references_size", object_store_log_memory_references_size.ToString()), - new("object_store_read_cache_size", object_store_read_cache_size.ToString()), - new("total_object_store_size", total_object_store_size.ToString()) + new("object_store_log_memory_size", object_store_log_memory_size.ToString()), + new("object_store_read_cache_log_memory_size", object_store_read_cache_log_memory_size.ToString()), + new("object_store_heap_memory_size", object_store_heap_memory_size.ToString()), + new("object_store_read_cache_heap_memory_size", object_store_read_cache_heap_memory_size.ToString()), + new("total_object_store_size", total_object_store_size.ToString()), + new("aof_memory_size", aof_log_memory_size.ToString()) ]; } diff --git a/libs/server/Servers/GarnetServerOptions.cs b/libs/server/Servers/GarnetServerOptions.cs index 6dd347ab02..1d1f938902 100644 --- a/libs/server/Servers/GarnetServerOptions.cs +++ b/libs/server/Servers/GarnetServerOptions.cs @@ -22,9 +22,9 @@ public class GarnetServerOptions : ServerOptions public bool DisableObjects = false; /// - /// Total memory size limit of object store including heap memory of objects. + /// Heap memory size limit of object store. /// - public string ObjectStoreTotalMemorySize = ""; + public string ObjectStoreHeapMemorySize = ""; /// /// Object store log memory used in bytes excluding heap memory. @@ -528,7 +528,7 @@ public static int MemorySizeBits(string memorySize, string storePageSize, out in /// /// Get KVSettings for the object store log /// - public KVSettings GetObjectStoreSettings(ILogger logger, out long objTotalMemorySize) + public KVSettings GetObjectStoreSettings(ILogger logger, out long objHeapMemorySize) { if (ObjectStoreMutablePercent is < 10 or > 95) throw new Exception("ObjectStoreMutablePercent must be between 10 and 95"); @@ -576,8 +576,8 @@ public KVSettings GetObjectStoreSettings(ILogger logger, } logger?.LogInformation("[Object Store] Using log mutable percentage of {ObjectStoreMutablePercent}%", ObjectStoreMutablePercent); - objTotalMemorySize = ParseSize(ObjectStoreTotalMemorySize); - logger?.LogInformation("[Object Store] Total memory size including heap objects is {totalMemorySize}", (objTotalMemorySize > 0 ? PrettySize(objTotalMemorySize) : "unlimited")); + objHeapMemorySize = ParseSize(ObjectStoreHeapMemorySize); + logger?.LogInformation("[Object Store] Total memory size including heap objects is {totalMemorySize}", (objHeapMemorySize > 0 ? PrettySize(objHeapMemorySize) : "unlimited")); if (EnableStorageTier) { diff --git a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs index 0bb36fe293..866b90c028 100644 --- a/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/RMWMethods.cs @@ -171,6 +171,7 @@ public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetOb { // We're performing the object update here (and not in CopyUpdater) so that we are guaranteed that // the record was CASed into the hash chain before it gets modified + var oldValueSize = oldValue.Size; oldValue.CopyUpdate(ref oldValue, ref value, rmwInfo.RecordInfo.IsInNewVersion); functionsState.watchVersionMap.IncrementVersion(rmwInfo.KeyHash); @@ -220,7 +221,9 @@ public bool PostCopyUpdater(ref byte[] key, ref ObjectInput input, ref IGarnetOb } } - functionsState.objectStoreSizeTracker?.AddTrackedSize(MemoryUtils.CalculateKeyValueSize(key, value)); + // If oldValue has been set to null, subtract it's size from the tracked heap size + var sizeAdjustment = oldValue == null ? value.Size - oldValueSize : value.Size; + functionsState.objectStoreSizeTracker?.AddTrackedSize(sizeAdjustment); if (functionsState.appendOnlyFile != null) WriteLogRMW(ref key, ref input, rmwInfo.Version, rmwInfo.SessionID); diff --git a/libs/server/Storage/SizeTracker/CacheSizeTracker.cs b/libs/server/Storage/SizeTracker/CacheSizeTracker.cs index b319ef59eb..21b6bca3f2 100644 --- a/libs/server/Storage/SizeTracker/CacheSizeTracker.cs +++ b/libs/server/Storage/SizeTracker/CacheSizeTracker.cs @@ -22,13 +22,11 @@ public class CacheSizeTracker { internal readonly LogSizeTracker mainLogTracker; internal readonly LogSizeTracker readCacheTracker; - internal long targetSize; + public long TargetSize; private const int deltaFraction = 10; // 10% of target size private TsavoriteKV store; - internal long IndexSizeBytes => store.IndexSize * 64 + store.OverflowBucketCount * 64; - internal bool Stopped => mainLogTracker.Stopped && (readCacheTracker == null || readCacheTracker.Stopped); /// Helps calculate size of a record including heap memory in Object store. @@ -44,7 +42,7 @@ public readonly long CalculateRecordSize(RecordInfo recordInfo, byte[] key, IGar { long size = Utility.RoundUp(key.Length, IntPtr.Size) + MemoryUtils.ByteArrayOverhead; - if (!recordInfo.Tombstone) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter) + if (!recordInfo.Tombstone && value != null) // ignore deleted values being evicted (they are accounted for by ConcurrentDeleter) size += value.Size; return size; @@ -64,6 +62,7 @@ public CacheSizeTracker(TsavoriteKV 0); this.store = store; + this.TargetSize = targetSize; var logSizeCalculator = new LogSizeCalculator(); var (mainLogTargetSizeBytes, readCacheTargetSizeBytes) = CalculateLogTargetSizeBytes(targetSize); @@ -96,13 +95,8 @@ public void Start(CancellationToken token) /// Target size public (long mainLogSizeBytes, long readCacheSizeBytes) CalculateLogTargetSizeBytes(long newTargetSize) { - long residual = newTargetSize - IndexSizeBytes; - - if (residual <= 0) - throw new TsavoriteException($"Target size {newTargetSize} must be larger than index size {IndexSizeBytes}"); - - var mainLogSizeBytes = this.store.ReadCache == null ? residual : residual / 2; - var readCacheSizeBytes = this.store.ReadCache == null ? 0 : residual / 2; + var mainLogSizeBytes = this.store.ReadCache == null ? newTargetSize : newTargetSize / 2; + var readCacheSizeBytes = this.store.ReadCache == null ? 0 : newTargetSize / 2; return (mainLogSizeBytes, readCacheSizeBytes); } diff --git a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs index 5fc6cb5cf0..2d9380a7dd 100644 --- a/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs +++ b/libs/storage/Tsavorite/cs/src/core/Allocator/AllocatorBase.cs @@ -714,6 +714,9 @@ protected void Initialize(long firstValidAddress) /// Minimum number of empty pages in circular buffer to be maintained to account for non-power-of-two size public int MinEmptyPageCount; + /// Maximum memory size in bytes + public long MaxMemorySizeBytes => (BufferSize - MinEmptyPageCount) * (long)PageSize; + /// How many pages do we leave empty in the in-memory buffer (between 0 and BufferSize-1) public int EmptyPageCount { diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs index 0695ab3b1c..f1994158db 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Common/LogSizeTracker.cs @@ -197,11 +197,11 @@ async Task ResizerTask(CancellationToken token) /// private void ResizeIfNeeded(CancellationToken token) { - // Include memory size from the log (logAccessor.MemorySizeBytes) + heap size (logSize.Total) to check utilization - if (logSize.Total + logAccessor.MemorySizeBytes > highTargetSize) + // Monitor the heap size + if (logSize.Total > highTargetSize) { - logger?.LogDebug("Heap size {totalLogSize} + log {MemorySizeBytes} > target {highTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, logAccessor.MemorySizeBytes, highTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount); - while (logSize.Total + logAccessor.MemorySizeBytes > highTargetSize && + logger?.LogDebug("Heap size {totalLogSize} > target {highTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, highTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount); + while (logSize.Total > highTargetSize && logAccessor.EmptyPageCount < logAccessor.MaxEmptyPageCount) { token.ThrowIfCancellationRequested(); @@ -216,10 +216,10 @@ private void ResizeIfNeeded(CancellationToken token) logger?.LogDebug("Increasing empty page count to {EmptyPageCount}", logAccessor.EmptyPageCount); } } - else if (logSize.Total + logAccessor.MemorySizeBytes < lowTargetSize) + else if (logSize.Total < lowTargetSize) { - logger?.LogDebug("Heap size {totalLogSize} + log {MemorySizeBytes} < target {lowTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, logAccessor.MemorySizeBytes, lowTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount); - while (logSize.Total + logAccessor.MemorySizeBytes < lowTargetSize && + logger?.LogDebug("Heap size {totalLogSize} < target {lowTargetSize}. Alloc: {AllocatedPageCount} EPC: {EmptyPageCount}", logSize.Total, lowTargetSize, logAccessor.AllocatedPageCount, logAccessor.EmptyPageCount); + while (logSize.Total < lowTargetSize && logAccessor.EmptyPageCount > logAccessor.MinEmptyPageCount) { token.ThrowIfCancellationRequested(); diff --git a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs index 087de8917e..a12ee914dd 100644 --- a/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs +++ b/libs/storage/Tsavorite/cs/src/core/Index/Tsavorite/LogAccessor.cs @@ -107,6 +107,11 @@ public void SetEmptyPageCount(int pageCount, bool wait = false) /// public long MemorySizeBytes => ((long)(allocatorBase.AllocatedPageCount + allocator.OverflowPageCount)) << allocatorBase.LogPageSizeBits; + /// + /// Maximum memory size in bytes + /// + public long MaxMemorySizeBytes => allocatorBase.MaxMemorySizeBytes; + /// /// Number of pages allocated /// diff --git a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs index 05cbc22678..50d4ced698 100644 --- a/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs +++ b/libs/storage/Tsavorite/cs/src/core/TsavoriteLog/TsavoriteLog.cs @@ -157,6 +157,16 @@ public sealed class TsavoriteLog : IDisposable /// readonly bool AutoCommit; + /// + /// Maximum memory size in bytes + /// + public long MaxMemorySizeBytes => allocator.MaxMemorySizeBytes; + + /// + /// Actual memory used by log + /// + public long MemorySizeBytes => ((long)(allocator.AllocatedPageCount + allocator.OverflowPageCount)) << allocator.LogPageSizeBits; + /// /// Create new log instance /// diff --git a/test/Garnet.test/CacheSizeTrackerTests.cs b/test/Garnet.test/CacheSizeTrackerTests.cs index b0c857e14f..577e5dfa27 100644 --- a/test/Garnet.test/CacheSizeTrackerTests.cs +++ b/test/Garnet.test/CacheSizeTrackerTests.cs @@ -25,7 +25,7 @@ public class CacheSizeTrackerTests public void Setup() { TestUtils.DeleteDirectory(TestUtils.MethodTestDir, wait: true); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, MemorySize: "2k", PageSize: "512", lowMemory: true, objectStoreIndexSize: "1k", objectStoreTotalMemorySize: "8k"); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, MemorySize: "2k", PageSize: "512", lowMemory: true, objectStoreIndexSize: "1k", objectStoreHeapMemorySize: "5k"); server.Start(); objStore = server.Provider.StoreWrapper.objectStore; cacheSizeTracker = server.Provider.StoreWrapper.objectStoreSizeTracker; diff --git a/test/Garnet.test/RespAdminCommandsTests.cs b/test/Garnet.test/RespAdminCommandsTests.cs index 2e7284a09a..19963ba9b3 100644 --- a/test/Garnet.test/RespAdminCommandsTests.cs +++ b/test/Garnet.test/RespAdminCommandsTests.cs @@ -335,7 +335,7 @@ public void SeSaveRecoverMultipleObjectsTest(int memorySize, int recoveryMemoryS } server.Dispose(false); - server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, MemorySize: sizeToString(recoveryMemorySize), PageSize: sizeToString(pageSize), objectStoreTotalMemorySize: "64k"); + server = TestUtils.CreateGarnetServer(TestUtils.MethodTestDir, tryRecover: true, lowMemory: true, MemorySize: sizeToString(recoveryMemorySize), PageSize: sizeToString(pageSize), objectStoreHeapMemorySize: "64k"); server.Start(); ClassicAssert.LessOrEqual(server.Provider.StoreWrapper.objectStore.MaxAllocatedPageCount, (recoveryMemorySize / pageSize) + 1); diff --git a/test/Garnet.test/TestUtils.cs b/test/Garnet.test/TestUtils.cs index b4260057a4..18b0c5a8fc 100644 --- a/test/Garnet.test/TestUtils.cs +++ b/test/Garnet.test/TestUtils.cs @@ -167,7 +167,7 @@ public static GarnetServer CreateGarnetServer( string defaultPassword = null, bool useAcl = false, // NOTE: Temporary until ACL is enforced as default string aclFile = null, - string objectStoreTotalMemorySize = default, + string objectStoreHeapMemorySize = default, string objectStoreIndexSize = "16k", string objectStoreIndexMaxSize = default, string indexSize = "1m", @@ -258,8 +258,8 @@ public static GarnetServer CreateGarnetServer( LoadModuleCS = loadModulePaths }; - if (!string.IsNullOrEmpty(objectStoreTotalMemorySize)) - opts.ObjectStoreTotalMemorySize = objectStoreTotalMemorySize; + if (!string.IsNullOrEmpty(objectStoreHeapMemorySize)) + opts.ObjectStoreHeapMemorySize = objectStoreHeapMemorySize; if (indexMaxSize != default) opts.IndexMaxSize = indexMaxSize; if (objectStoreIndexMaxSize != default) opts.ObjectStoreIndexMaxSize = objectStoreIndexMaxSize; diff --git a/website/docs/getting-started/configuration.md b/website/docs/getting-started/configuration.md index b1eae64365..04c7836491 100644 --- a/website/docs/getting-started/configuration.md +++ b/website/docs/getting-started/configuration.md @@ -74,14 +74,14 @@ For all available command line settings, run `GarnetServer.exe -h` or `GarnetSer | **MemorySize** | ```-m```
```--memory``` | ```string``` | Memory size | Total log memory used in bytes (rounds down to power of 2) | | **PageSize** | ```-p```
```--page``` | ```string``` | Memory size | Size of each page in bytes (rounds down to power of 2) | | **SegmentSize** | ```-s```
```--segment``` | ```string``` | Memory size | Size of each log segment in bytes on disk (rounds down to power of 2) | -| **IndexSize** | ```-i```
```--index``` | ```string``` | Memory size | Size of hash index in bytes (rounds down to power of 2) | +| **IndexSize** | ```-i```
```--index``` | ```string``` | Memory size | Start size of hash index in bytes (rounds down to power of 2) | | **IndexMaxSize** | ```--index-max-size``` | ```string``` | Memory size | Max size of hash index in bytes (rounds down to power of 2) | | **MutablePercent** | ```--mutable-percent``` | ```int``` | | Percentage of log memory that is kept mutable | -| **ObjectStoreTotalMemorySize** | ```--obj-total-memory``` | ```string``` | Memory size | Total object store log memory used including heap memory in bytes | -| **ObjectStoreLogMemorySize** | ```--obj-memory``` | ```string``` | Memory size | Object store log memory used in bytes excluding heap memory | +| **ObjectStoreHeapMemorySize** | ```--obj-heap-memory``` | ```string``` | Memory size | Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap) | +| **ObjectStoreLogMemorySize** | ```--obj-log-memory``` | ```string``` | Memory size | Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log) | | **ObjectStorePageSize** | ```--obj-page``` | ```string``` | Memory size | Size of each object store page in bytes (rounds down to power of 2) | | **ObjectStoreSegmentSize** | ```--obj-segment``` | ```string``` | Memory size | Size of each object store log segment in bytes on disk (rounds down to power of 2) | -| **ObjectStoreIndexSize** | ```--obj-index``` | ```string``` | Memory size | Size of object store hash index in bytes (rounds down to power of 2) | +| **ObjectStoreIndexSize** | ```--obj-index``` | ```string``` | Memory size | Start size of object store hash index in bytes (rounds down to power of 2) | | **ObjectStoreIndexMaxSize** | ```--obj-index-max-size``` | ```string``` | Memory size | Max size of object store hash index in bytes (rounds down to power of 2) | | **ObjectStoreMutablePercent** | ```--obj-mutable-percent``` | ```int``` | | Percentage of object store log memory that is kept mutable | | **EnableStorageTier** | ```--storage-tier``` | ```bool``` | | Enable tiering of records (hybrid log) to storage, to support a larger-than-memory store. Use --logdir to specify storage directory. | diff --git a/website/docs/getting-started/memory.md b/website/docs/getting-started/memory.md index 38d2c98681..49e4fa7c42 100644 --- a/website/docs/getting-started/memory.md +++ b/website/docs/getting-started/memory.md @@ -78,7 +78,7 @@ However, the log memory is handled differently, as described below. In case of the object store, the hybrid log holds _references_ to keys and values (which are objects), rather than the actual keys and values themselves. The memory occupied by the object store log is configured using -`ObjectStoreLogMemorySize` (`--obj-memory`). However, this parameter only controls the number of records +`ObjectStoreLogMemorySize` (`--obj-log-memory`). However, this parameter only controls the number of records in the object store, where each record consists of: * An 8-byte header, called `RecordInfo`, which holds metadata and the logical address of the previous entry in a record chain. * An 8-byte reference to the key object, which is a byte array on heap (byte[]) @@ -90,13 +90,12 @@ setting `ObjectStoreLogMemorySize` to S merely implies that you can hold at most memory. This means, of course, that we need to track the total memory using a different mechanism. For this, Garnet -exposes a configuration called `ObjectStoreTotalMemorySize` (`--obj-total-memory`) which represents total object -store log memory used, including the hybrid log and the heap memory in bytes. You can use this parameter +exposes a configuration called `ObjectStoreHeapMemorySize` (`--obj-heap-memory`) which represents the heap memory +used by key byte arrays and the `IGarnetObject` instances in bytes. You can use this parameter in combination with the `--obj-log-memory` to control the total memory used by the object store. To summarize, the total space occupied by the object store is the sum of: * Object store index size (and overflow buckets), as before -* `ObjectStoreTotalMemorySize` - -with `ObjectStoreLogMemorySize` used to control the maximum _number_ of records in memory. +* `ObjectStoreLogMemorySize` (`--obj-log-memory`) which controls the maximum _number_ of records in memory. +* `ObjectStoreHeapMemorySize` (`--obj-heap-memory`) which controls the total heap size occupied by the objects. From 41d77ba232eb152a1d6e16d20fb359911750ec5d Mon Sep 17 00:00:00 2001 From: Vijay Nirmal Date: Sat, 5 Oct 2024 03:10:43 +0530 Subject: [PATCH 4/4] [Compatibility] Adding EXPIRETIME and PEXPIRETIME command (#664) * Adding EXPIRETIME and EXPIRETIME command * Review comment fix for RespInputHeader * Review comment fix, added switch --------- Co-authored-by: Tal Zaccai --- libs/common/ConvertUtils.cs | 20 +++ libs/server/API/GarnetApi.cs | 12 ++ libs/server/API/GarnetWatchApi.cs | 18 ++ libs/server/API/IGarnetApi.cs | 22 +++ libs/server/Objects/Types/GarnetObjectType.cs | 10 ++ libs/server/Resp/KeyAdminCommands.cs | 36 ++++ libs/server/Resp/Parser/RespCommand.cs | 10 ++ libs/server/Resp/RespCommandsInfo.json | 58 ++++++ libs/server/Resp/RespServerSession.cs | 2 + .../Functions/MainStore/PrivateMethods.cs | 11 ++ .../Functions/ObjectStore/ReadMethods.cs | 47 +++-- .../Storage/Session/MainStore/MainStoreOps.cs | 65 +++++++ .../CommandInfoUpdater/SupportedCommand.cs | 2 + test/Garnet.test/Resp/ACL/RespCommandTests.cs | 30 ++++ test/Garnet.test/RespTests.cs | 167 ++++++++++++++++++ website/docs/commands/api-compatibility.md | 4 +- website/docs/commands/generic-commands.md | 40 +++++ 17 files changed, 535 insertions(+), 19 deletions(-) diff --git a/libs/common/ConvertUtils.cs b/libs/common/ConvertUtils.cs index 89ef9ec763..140d97ed36 100644 --- a/libs/common/ConvertUtils.cs +++ b/libs/common/ConvertUtils.cs @@ -70,5 +70,25 @@ public static long UnixTimestampInMillisecondsToTicks(long unixTimestamp) { return unixTimestamp * TimeSpan.TicksPerMillisecond + _unixEpochTicks; } + + /// + /// Convert ticks to Unix time in seconds. + /// + /// The ticks to convert. + /// The Unix time in seconds. + public static long UnixTimeInSecondsFromTicks(long ticks) + { + return ticks > 0 ? (ticks - _unixEpochTicks) / TimeSpan.TicksPerSecond : -1; + } + + /// + /// Convert ticks to Unix time in milliseconds. + /// + /// The ticks to convert. + /// The Unix time in milliseconds. + public static long UnixTimeInMillisecondsFromTicks(long ticks) + { + return ticks > 0 ? (ticks - _unixEpochTicks) / TimeSpan.TicksPerMillisecond : -1; + } } } \ No newline at end of file diff --git a/libs/server/API/GarnetApi.cs b/libs/server/API/GarnetApi.cs index da6f90165c..ac364ad74e 100644 --- a/libs/server/API/GarnetApi.cs +++ b/libs/server/API/GarnetApi.cs @@ -93,6 +93,18 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM #endregion + #region EXPIRETIME + + /// + public GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) + => storageSession.EXPIRETIME(ref key, storeType, ref output, ref context, ref objectContext); + + /// + public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) + => storageSession.EXPIRETIME(ref key, storeType, ref output, ref context, ref objectContext, milliseconds: true); + + #endregion + #region SET /// public GarnetStatus SET(ref SpanByte key, ref SpanByte value) diff --git a/libs/server/API/GarnetWatchApi.cs b/libs/server/API/GarnetWatchApi.cs index bbae63343a..8a4e6044b2 100644 --- a/libs/server/API/GarnetWatchApi.cs +++ b/libs/server/API/GarnetWatchApi.cs @@ -77,6 +77,24 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM #endregion + #region EXPIRETIME + + /// + public GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) + { + garnetApi.WATCH(new ArgSlice(ref key), storeType); + return garnetApi.EXPIRETIME(ref key, storeType, ref output); + } + + /// + public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output) + { + garnetApi.WATCH(new ArgSlice(ref key), storeType); + return garnetApi.PEXPIRETIME(ref key, storeType, ref output); + } + + #endregion + #region SortedSet Methods /// diff --git a/libs/server/API/IGarnetApi.cs b/libs/server/API/IGarnetApi.cs index 5b5f2997c8..53af70fdb2 100644 --- a/libs/server/API/IGarnetApi.cs +++ b/libs/server/API/IGarnetApi.cs @@ -1085,6 +1085,28 @@ public interface IGarnetReadApi #endregion + #region EXPIRETIME + + /// + /// Returns the absolute Unix timestamp (since January 1, 1970) in seconds at which the given key will expire. + /// + /// The key to get the expiration time for. + /// The type of store to retrieve the key from. + /// The output containing the expiration time. + /// The status of the operation. + GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output); + + /// + /// Returns the absolute Unix timestamp (since January 1, 1970) in milliseconds at which the given key will expire. + /// + /// The key to get the expiration time for. + /// The type of store to retrieve the key from. + /// The output containing the expiration time. + /// The status of the operation. + GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output); + + #endregion + #region SortedSet Methods /// diff --git a/libs/server/Objects/Types/GarnetObjectType.cs b/libs/server/Objects/Types/GarnetObjectType.cs index a5d7f3e596..0657ab059d 100644 --- a/libs/server/Objects/Types/GarnetObjectType.cs +++ b/libs/server/Objects/Types/GarnetObjectType.cs @@ -29,6 +29,16 @@ public enum GarnetObjectType : byte /// Set, + /// + /// Special type indicating EXPIRETIME command + /// + Expiretime = 0xf9, + + /// + /// Special type indicating PEXPIRETIME command + /// + PExpiretime = 0xfa, + /// /// Special type indicating PERSIST command /// diff --git a/libs/server/Resp/KeyAdminCommands.cs b/libs/server/Resp/KeyAdminCommands.cs index 029e2ca16b..4d2aaf6ddc 100644 --- a/libs/server/Resp/KeyAdminCommands.cs +++ b/libs/server/Resp/KeyAdminCommands.cs @@ -404,5 +404,41 @@ private bool NetworkTTL(RespCommand command, ref TGarnetApi storageA } return true; } + + /// + /// Get the absolute Unix timestamp at which the given key will expire. + /// + /// + /// either if the call is for EXPIRETIME or PEXPIRETIME command + /// + /// Returns the absolute Unix timestamp (since January 1, 1970) in seconds or milliseconds at which the given key will expire. + private bool NetworkEXPIRETIME(RespCommand command, ref TGarnetApi storageApi) + where TGarnetApi : IGarnetApi + { + if (parseState.Count != 1) + { + return AbortWithWrongNumberOfArguments(nameof(RespCommand.EXPIRETIME)); + } + + var sbKey = parseState.GetArgSliceByRef(0).SpanByte; + var o = new SpanByteAndMemory(dcurr, (int)(dend - dcurr)); + var status = command == RespCommand.EXPIRETIME ? + storageApi.EXPIRETIME(ref sbKey, StoreType.All, ref o) : + storageApi.PEXPIRETIME(ref sbKey, StoreType.All, ref o); + + if (status == GarnetStatus.OK) + { + if (!o.IsSpanByte) + SendAndReset(o.Memory, o.Length); + else + dcurr += o.Length; + } + else + { + while (!RespWriteUtils.WriteDirect(CmdStrings.RESP_RETURN_VAL_N2, ref dcurr, dend)) + SendAndReset(); + } + return true; + } } } \ No newline at end of file diff --git a/libs/server/Resp/Parser/RespCommand.cs b/libs/server/Resp/Parser/RespCommand.cs index efd8cf3df6..ce316987da 100644 --- a/libs/server/Resp/Parser/RespCommand.cs +++ b/libs/server/Resp/Parser/RespCommand.cs @@ -25,6 +25,7 @@ public enum RespCommand : byte COSCAN, DBSIZE, EXISTS, + EXPIRETIME, GEODIST, GEOHASH, GEOPOS, @@ -49,6 +50,7 @@ public enum RespCommand : byte LRANGE, MEMORY_USAGE, MGET, + PEXPIRETIME, PFCOUNT, PTTL, SCAN, @@ -1329,6 +1331,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SDIFFSTORE; } + else if (*(ulong*)(ptr + 1) == MemoryMarshal.Read("10\r\nEXPI"u8) && *(uint*)(ptr + 9) == MemoryMarshal.Read("RETIME\r\n"u8)) + { + return RespCommand.EXPIRETIME; + } break; case 11: @@ -1356,6 +1362,10 @@ private RespCommand FastParseArrayCommand(ref int count, ref ReadOnlySpan { return RespCommand.SINTERSTORE; } + else if (*(ulong*)(ptr + 2) == MemoryMarshal.Read("1\r\nPEXPI"u8) && *(uint*)(ptr + 10) == MemoryMarshal.Read("RETIME\r\n"u8)) + { + return RespCommand.PEXPIRETIME; + } break; case 12: diff --git a/libs/server/Resp/RespCommandsInfo.json b/libs/server/Resp/RespCommandsInfo.json index b6c5265760..11527e027b 100644 --- a/libs/server/Resp/RespCommandsInfo.json +++ b/libs/server/Resp/RespCommandsInfo.json @@ -1623,6 +1623,35 @@ ], "SubCommands": null }, + { + "Command": "EXPIRETIME", + "Name": "EXPIRETIME", + "IsInternal": false, + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, KeySpace, Read", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Notes": null, + "Flags": "RO, Access" + } + ], + "SubCommands": null + }, { "Command": "FAILOVER", "Name": "FAILOVER", @@ -3341,6 +3370,35 @@ ], "SubCommands": null }, + { + "Command": "PEXPIRETIME", + "Name": "PEXPIRETIME", + "IsInternal": false, + "Arity": 2, + "Flags": "Fast, ReadOnly", + "FirstKey": 1, + "LastKey": 1, + "Step": 1, + "AclCategories": "Fast, KeySpace, Read", + "Tips": null, + "KeySpecifications": [ + { + "BeginSearch": { + "TypeDiscriminator": "BeginSearchIndex", + "Index": 1 + }, + "FindKeys": { + "TypeDiscriminator": "FindKeysRange", + "LastKey": 0, + "KeyStep": 1, + "Limit": 0 + }, + "Notes": null, + "Flags": "RO, Access" + } + ], + "SubCommands": null + }, { "Command": "PFADD", "Name": "PFADD", diff --git a/libs/server/Resp/RespServerSession.cs b/libs/server/Resp/RespServerSession.cs index 848a0edb3d..2933508380 100644 --- a/libs/server/Resp/RespServerSession.cs +++ b/libs/server/Resp/RespServerSession.cs @@ -516,6 +516,8 @@ private bool ProcessBasicCommands(RespCommand cmd, ref TGarnetApi st RespCommand.EXISTS => NetworkEXISTS(ref storageApi), RespCommand.EXPIRE => NetworkEXPIRE(RespCommand.EXPIRE, ref storageApi), RespCommand.PEXPIRE => NetworkEXPIRE(RespCommand.PEXPIRE, ref storageApi), + RespCommand.EXPIRETIME => NetworkEXPIRETIME(RespCommand.EXPIRETIME, ref storageApi), + RespCommand.PEXPIRETIME => NetworkEXPIRETIME(RespCommand.PEXPIRETIME, ref storageApi), RespCommand.PERSIST => NetworkPERSIST(ref storageApi), RespCommand.GETRANGE => NetworkGetRange(ref storageApi), RespCommand.TTL => NetworkTTL(RespCommand.TTL, ref storageApi), diff --git a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs index 193a125f43..5eb0835968 100644 --- a/libs/server/Storage/Functions/MainStore/PrivateMethods.cs +++ b/libs/server/Storage/Functions/MainStore/PrivateMethods.cs @@ -217,6 +217,17 @@ void CopyRespToWithInput(ref SpanByte input, ref SpanByte value, ref SpanByteAnd (start, end) = NormalizeRange(start, end, len); CopyRespTo(ref value, ref dst, start, end); return; + + case RespCommand.EXPIRETIME: + var expireTime = ConvertUtils.UnixTimeInSecondsFromTicks(value.MetadataSize > 0 ? value.ExtraMetadata : -1); + CopyRespNumber(expireTime, ref dst); + return; + + case RespCommand.PEXPIRETIME: + var pexpireTime = ConvertUtils.UnixTimeInMillisecondsFromTicks(value.MetadataSize > 0 ? value.ExtraMetadata : -1); + CopyRespNumber(pexpireTime, ref dst); + return; + default: throw new GarnetException("Unsupported operation on input"); } diff --git a/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs b/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs index 2cc20b0dc0..b360f7849a 100644 --- a/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs +++ b/libs/server/Storage/Functions/ObjectStore/ReadMethods.cs @@ -25,27 +25,40 @@ public bool SingleReader(ref byte[] key, ref ObjectInput input, ref IGarnetObjec if (input.header.type != 0) { - if (input.header.type == GarnetObjectType.Ttl || input.header.type == GarnetObjectType.PTtl) // TTL command + switch (input.header.type) { - var ttlValue = input.header.type == GarnetObjectType.Ttl ? - ConvertUtils.SecondsFromDiffUtcNowTicks(value.Expiration > 0 ? value.Expiration : -1) : - ConvertUtils.MillisecondsFromDiffUtcNowTicks(value.Expiration > 0 ? value.Expiration : -1); - CopyRespNumber(ttlValue, ref dst.spanByteAndMemory); - return true; - } + case GarnetObjectType.Ttl: + var ttlValue = ConvertUtils.SecondsFromDiffUtcNowTicks(value.Expiration > 0 ? value.Expiration : -1); + CopyRespNumber(ttlValue, ref dst.spanByteAndMemory); + return true; + case GarnetObjectType.PTtl: + ttlValue = ConvertUtils.MillisecondsFromDiffUtcNowTicks(value.Expiration > 0 ? value.Expiration : -1); + CopyRespNumber(ttlValue, ref dst.spanByteAndMemory); + return true; + + case GarnetObjectType.Expiretime: + var expireTime = ConvertUtils.UnixTimeInSecondsFromTicks(value.Expiration > 0 ? value.Expiration : -1); + CopyRespNumber(expireTime, ref dst.spanByteAndMemory); + return true; + case GarnetObjectType.PExpiretime: + expireTime = ConvertUtils.UnixTimeInMillisecondsFromTicks(value.Expiration > 0 ? value.Expiration : -1); + CopyRespNumber(expireTime, ref dst.spanByteAndMemory); + return true; - if ((byte)input.header.type < CustomCommandManager.StartOffset) - return value.Operate(ref input, ref dst.spanByteAndMemory, out _, out _); + default: + if ((byte)input.header.type < CustomCommandManager.StartOffset) + return value.Operate(ref input, ref dst.spanByteAndMemory, out _, out _); - if (IncorrectObjectType(ref input, value, ref dst.spanByteAndMemory)) - return true; + if (IncorrectObjectType(ref input, value, ref dst.spanByteAndMemory)) + return true; - (IMemoryOwner Memory, int Length) outp = (dst.spanByteAndMemory.Memory, 0); - var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); - var result = customObjectCommand.Reader(key, ref input, value, ref outp, ref readInfo); - dst.spanByteAndMemory.Memory = outp.Memory; - dst.spanByteAndMemory.Length = outp.Length; - return result; + (IMemoryOwner Memory, int Length) outp = (dst.spanByteAndMemory.Memory, 0); + var customObjectCommand = GetCustomObjectCommand(ref input, input.header.type); + var result = customObjectCommand.Reader(key, ref input, value, ref outp, ref readInfo); + dst.spanByteAndMemory.Memory = outp.Memory; + dst.spanByteAndMemory.Length = outp.Length; + return result; + } } dst.garnetObject = value; diff --git a/libs/server/Storage/Session/MainStore/MainStoreOps.cs b/libs/server/Storage/Session/MainStore/MainStoreOps.cs index 12637006e5..344c4a72b4 100644 --- a/libs/server/Storage/Session/MainStore/MainStoreOps.cs +++ b/libs/server/Storage/Session/MainStore/MainStoreOps.cs @@ -302,6 +302,71 @@ public unsafe GarnetStatus TTL(ref SpanByte key, Store return GarnetStatus.NOTFOUND; } + /// + /// Get the absolute Unix timestamp at which the given key will expire. + /// + /// + /// + /// The key to get the Unix timestamp. + /// The store to operate on + /// Span to allocate the output of the operation + /// Basic Context of the store + /// Object Context of the store + /// when true the command to execute is PEXPIRETIME. + /// Returns the absolute Unix timestamp (since January 1, 1970) in seconds or milliseconds at which the given key will expire. + public unsafe GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output, ref TContext context, ref TObjectContext objectContext, bool milliseconds = false) + where TContext : ITsavoriteContext + where TObjectContext : ITsavoriteContext + { + int inputSize = sizeof(int) + RespInputHeader.Size; + byte* pbCmdInput = stackalloc byte[inputSize]; + + byte* pcurr = pbCmdInput; + *(int*)pcurr = inputSize - sizeof(int); + pcurr += sizeof(int); + (*(RespInputHeader*)pcurr).cmd = milliseconds ? RespCommand.PEXPIRETIME : RespCommand.EXPIRETIME; + (*(RespInputHeader*)pcurr).flags = 0; + + if (storeType == StoreType.Main || storeType == StoreType.All) + { + var status = context.Read(ref key, ref Unsafe.AsRef(pbCmdInput), ref output); + + if (status.IsPending) + { + StartPendingMetrics(); + CompletePendingForSession(ref status, ref output, ref context); + StopPendingMetrics(); + } + + if (status.Found) return GarnetStatus.OK; + } + + if ((storeType == StoreType.Object || storeType == StoreType.All) && !objectStoreBasicContext.IsNull) + { + var objInput = new ObjectInput + { + header = new RespInputHeader + { + type = milliseconds ? GarnetObjectType.PExpiretime : GarnetObjectType.Expiretime, + }, + }; + + var keyBA = key.ToByteArray(); + var objO = new GarnetObjectStoreOutput { spanByteAndMemory = output }; + var status = objectContext.Read(ref keyBA, ref objInput, ref objO); + + if (status.IsPending) + CompletePendingForObjectStoreSession(ref status, ref objO, ref objectContext); + + if (status.Found) + { + output = objO.spanByteAndMemory; + return GarnetStatus.OK; + } + } + return GarnetStatus.NOTFOUND; + } + public GarnetStatus SET(ref SpanByte key, ref SpanByte value, ref TContext context) where TContext : ITsavoriteContext { diff --git a/playground/CommandInfoUpdater/SupportedCommand.cs b/playground/CommandInfoUpdater/SupportedCommand.cs index 48176aabc5..6cd0d47eeb 100644 --- a/playground/CommandInfoUpdater/SupportedCommand.cs +++ b/playground/CommandInfoUpdater/SupportedCommand.cs @@ -112,6 +112,7 @@ public class SupportedCommand new("EXISTS", RespCommand.EXISTS), new("EXPIRE", RespCommand.EXPIRE), new("EXPIREAT", RespCommand.EXPIREAT), + new("EXPIRETIME", RespCommand.EXPIRETIME), new("FAILOVER", RespCommand.FAILOVER), new("FLUSHALL", RespCommand.FLUSHALL), new("FLUSHDB", RespCommand.FLUSHDB), @@ -184,6 +185,7 @@ public class SupportedCommand new("PERSIST", RespCommand.PERSIST), new("PEXPIRE", RespCommand.PEXPIRE), new("PEXPIREAT", RespCommand.PEXPIREAT), + new("PEXPIRETIME", RespCommand.PEXPIRETIME), new("PFADD", RespCommand.PFADD), new("PFCOUNT", RespCommand.PFCOUNT), new("PFMERGE", RespCommand.PFMERGE), diff --git a/test/Garnet.test/Resp/ACL/RespCommandTests.cs b/test/Garnet.test/Resp/ACL/RespCommandTests.cs index 758ad68d30..0c7c5cfdb8 100644 --- a/test/Garnet.test/Resp/ACL/RespCommandTests.cs +++ b/test/Garnet.test/Resp/ACL/RespCommandTests.cs @@ -4411,6 +4411,36 @@ static async Task DoRegisterCSAsync(GarnetClient client) } } + [Test] + public async Task ExpireTimeACLsAsync() + { + await CheckCommandsAsync( + "EXPIRETIME", + [DoExpireTimeAsync] + ); + + static async Task DoExpireTimeAsync(GarnetClient client) + { + var val = await client.ExecuteForLongResultAsync("EXPIRETIME", ["foo"]); + ClassicAssert.AreEqual(-2, val); + } + } + + [Test] + public async Task PExpireTimeACLsAsync() + { + await CheckCommandsAsync( + "PEXPIRETIME", + [DoPExpireTimeAsync] + ); + + static async Task DoPExpireTimeAsync(GarnetClient client) + { + var val = await client.ExecuteForLongResultAsync("PEXPIRETIME", ["foo"]); + ClassicAssert.AreEqual(-2, val); + } + } + [Test] public async Task RenameACLsAsync() { diff --git a/test/Garnet.test/RespTests.cs b/test/Garnet.test/RespTests.cs index 3c5aba514a..3e2e9c6243 100644 --- a/test/Garnet.test/RespTests.cs +++ b/test/Garnet.test/RespTests.cs @@ -1142,6 +1142,173 @@ public void MultipleExistsKeysAndObjects() ClassicAssert.AreEqual(3, exists); } + #region Expiretime + + [Test] + public void ExpiretimeWithStingValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + string key = "key1"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + db.StringSet(key, "test1", expireTimeSpan); + + var actualExpireTime = (long)db.Execute("EXPIRETIME", key); + + ClassicAssert.GreaterOrEqual(actualExpireTime, DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + var expireExpireTime = DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds(); + ClassicAssert.LessOrEqual(actualExpireTime, expireExpireTime); + } + + [Test] + public void ExpiretimeWithUnknownKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var expireTime = (long)db.Execute("EXPIRETIME", "keyZ"); + + ClassicAssert.AreEqual(-2, expireTime); + } + + [Test] + public void ExpiretimeWithNoKeyExpiration() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + string key = "key1"; + db.StringSet(key, "test1"); + + var expireTime = (long)db.Execute("EXPIRETIME", key); + + ClassicAssert.AreEqual(-1, expireTime); + } + + [Test] + public void ExpiretimeWithInvalidNumberOfArgs() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var exception = Assert.Throws(() => db.Execute("EXPIRETIME")); + Assert.That(exception.Message, Does.StartWith("ERR wrong number of arguments")); + } + + [Test] + public void ExpiretimeWithObjectValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + var origList = new RedisValue[] { "a", "b", "c", "d" }; + var count = db.ListRightPush(key, origList); + var expirySet = db.KeyExpire(key, expireTimeSpan); + + var actualExpireTime = (long)db.Execute("EXPIRETIME", key); + + ClassicAssert.GreaterOrEqual(actualExpireTime, DateTimeOffset.UtcNow.ToUnixTimeSeconds()); + var expireExpireTime = DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeSeconds(); + ClassicAssert.LessOrEqual(actualExpireTime, expireExpireTime); + } + + [Test] + public void ExpiretimeWithNoKeyExpirationForObjectValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + var origList = new RedisValue[] { "a", "b", "c", "d" }; + var count = db.ListRightPush(key, origList); + + var expireTime = (long)db.Execute("EXPIRETIME", key); + + ClassicAssert.AreEqual(-1, expireTime); + } + + [Test] + public void PExpiretimeWithStingValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + string key = "key1"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + db.StringSet(key, "test1", expireTimeSpan); + + var actualExpireTime = (long)db.Execute("PEXPIRETIME", key); + + ClassicAssert.GreaterOrEqual(actualExpireTime, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + var expireExpireTime = DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + ClassicAssert.LessOrEqual(actualExpireTime, expireExpireTime); + } + + [Test] + public void PExpiretimeWithUnknownKey() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var expireTime = (long)db.Execute("PEXPIRETIME", "keyZ"); + + ClassicAssert.AreEqual(-2, expireTime); + } + + [Test] + public void PExpiretimeWithNoKeyExpiration() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + string key = "key1"; + db.StringSet(key, "test1"); + + var expireTime = (long)db.Execute("PEXPIRETIME", key); + + ClassicAssert.AreEqual(-1, expireTime); + } + + [Test] + public void PExpiretimeWithInvalidNumberOfArgs() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + + var exception = Assert.Throws(() => db.Execute("PEXPIRETIME")); + Assert.That(exception.Message, Does.StartWith("ERR wrong number of arguments")); + } + + [Test] + public void PExpiretimeWithObjectValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + var expireTimeSpan = TimeSpan.FromMinutes(1); + var origList = new RedisValue[] { "a", "b", "c", "d" }; + var count = db.ListRightPush(key, origList); + var expirySet = db.KeyExpire(key, expireTimeSpan); + + var actualExpireTime = (long)db.Execute("PEXPIRETIME", key); + + ClassicAssert.GreaterOrEqual(actualExpireTime, DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + var expireExpireTime = DateTimeOffset.UtcNow.Add(expireTimeSpan).ToUnixTimeMilliseconds(); + ClassicAssert.LessOrEqual(actualExpireTime, expireExpireTime); + } + + [Test] + public void PExpiretimeWithNoKeyExpirationForObjectValue() + { + using var redis = ConnectionMultiplexer.Connect(TestUtils.GetConfig()); + var db = redis.GetDatabase(0); + var key = "key1"; + var origList = new RedisValue[] { "a", "b", "c", "d" }; + var count = db.ListRightPush(key, origList); + + var expireTime = (long)db.Execute("PEXPIRETIME", key); + + ClassicAssert.AreEqual(-1, expireTime); + } + + #endregion [Test] public void SingleRename() diff --git a/website/docs/commands/api-compatibility.md b/website/docs/commands/api-compatibility.md index febf726cba..dfecea3a36 100644 --- a/website/docs/commands/api-compatibility.md +++ b/website/docs/commands/api-compatibility.md @@ -136,7 +136,7 @@ Note that this list is subject to change as we continue to expand our API comman | **GENERIC** | [PERSIST](generic-commands.md#persist) | ➕ | | | | [PEXPIRE](generic-commands.md#pexpire) | ➕ | | | | [PEXPIREAT](generic-commands.md#pexpireat) | ➕ | | -| | PEXPIRETIME | ➖ | | +| | [PEXPIRETIME](generic-commands.md#pexpiretime) | ➕ | | | | [PTTL](generic-commands.md#pttl) | ➕ | | | | RANDOMKEY | ➖ | | | | [RENAME](generic-commands.md#rename) | ➕ | | @@ -197,7 +197,7 @@ Note that this list is subject to change as we continue to expand our API comman | | [EXISTS](generic-commands.md#exists) | ➕ | | | | [EXPIRE](generic-commands.md#expire) | ➕ | | | | [EXPIREAT](generic-commands.md#expireat) | ➕ | | -| | EXPIRETIME | ➖ | | +| | [EXPIRETIME](generic-commands.md#expiretime) | ➕ | | | | [KEYS](generic-commands.md#keys) | ➕ | | | | [MIGRATE](generic-commands.md#migrate) | ➕ | | | | MOVE | ➖ | | diff --git a/website/docs/commands/generic-commands.md b/website/docs/commands/generic-commands.md index 850130f84b..59ebf9b933 100644 --- a/website/docs/commands/generic-commands.md +++ b/website/docs/commands/generic-commands.md @@ -205,6 +205,26 @@ One of the following: --- +### EXPIRETIME + +#### Syntax + +```bash + EXPIRETIME key +``` + +Returns the absolute Unix timestamp (since January 1, 1970) in seconds at which the given key will expire. + +#### Resp Reply + +One of the following: + +* Integer reply: Expiration Unix timestamp in milliseconds. +* Integer reply: -1 if the key exists but has no associated expiration time. +* Integer reply: -2 if the key does not exist. + +--- + ### KEYS #### Syntax @@ -273,6 +293,26 @@ One of the following: --- +### PEXPIRETIME + +#### Syntax + +```bash + PEXPIRETIME key +``` + +Returns the absolute Unix timestamp (since January 1, 1970) in milliseconds at which the given key will expire. + +#### Resp Reply + +One of the following: + +* Integer reply: Expiration Unix timestamp in milliseconds. +* Integer reply: -1 if the key exists but has no associated expiration time. +* Integer reply: -2 if the key does not exist. + +--- + ### PEXPIREAT #### Syntax