Merge branch 'main' into new/novalue-option-for-hscan

.azure/pipelines/azure-pipelines-external-release.yml

@@ -3,7 +3,7 @@
 #      1) update the name: string below (line 6)     -- this is the version for the nuget package (e.g. 1.0.0)
 #      2) update \libs\host\GarnetServer.cs readonly string version  (~line 53)   -- NOTE - these two values need to be the same 
 ###################################### 
-name: 1.0.29
+name: 1.0.30
 trigger:
   branches:
     include:

libs/common/ConvertUtils.cs

@@ -2,7 +2,7 @@
 // Licensed under the MIT license.
 
 using System;
-using System.Diagnostics;
+using System.Runtime.CompilerServices;
 
 namespace Garnet.common
 {
@@ -11,6 +11,11 @@ namespace Garnet.common
     /// </summary>
     public static class ConvertUtils
     {
+        /// <summary>
+        /// Contains the number of ticks representing 1970/1/1. Value is equal to new DateTime(1970, 1, 1).Ticks
+        /// </summary>
+        private static readonly long _unixEpochTicks = DateTimeOffset.UnixEpoch.Ticks;
+
         /// <summary>
         /// Convert diff ticks - utcNow.ticks to seconds.
         /// </summary>
@@ -43,5 +48,47 @@ public static long MillisecondsFromDiffUtcNowTicks(long ticks)
             }
             return milliseconds;
         }
+
+        /// <summary>
+        /// Converts a Unix timestamp in seconds to ticks.
+        /// </summary>
+        /// <param name="unixTimestamp">The Unix timestamp in seconds.</param>
+        /// <returns>The equivalent number of ticks.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static long UnixTimestampInSecondsToTicks(long unixTimestamp)
+        {
+            return unixTimestamp * TimeSpan.TicksPerSecond + _unixEpochTicks;
+        }
+
+        /// <summary>
+        /// Converts a Unix timestamp in milliseconds to ticks.
+        /// </summary>
+        /// <param name="unixTimestamp">The Unix timestamp in milliseconds.</param>
+        /// <returns>The equivalent number of ticks.</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static long UnixTimestampInMillisecondsToTicks(long unixTimestamp)
+        {
+            return unixTimestamp * TimeSpan.TicksPerMillisecond + _unixEpochTicks;
+        }
+
+        /// <summary>
+        /// Convert ticks to Unix time in seconds.
+        /// </summary>
+        /// <param name="ticks">The ticks to convert.</param>
+        /// <returns>The Unix time in seconds.</returns>
+        public static long UnixTimeInSecondsFromTicks(long ticks)
+        {
+            return ticks > 0 ? (ticks - _unixEpochTicks) / TimeSpan.TicksPerSecond : -1;
+        }
+
+        /// <summary>
+        /// Convert ticks to Unix time in milliseconds.
+        /// </summary>
+        /// <param name="ticks">The ticks to convert.</param>
+        /// <returns>The Unix time in milliseconds.</returns>
+        public static long UnixTimeInMillisecondsFromTicks(long ticks)
+        {
+            return ticks > 0 ? (ticks - _unixEpochTicks) / TimeSpan.TicksPerMillisecond : -1;
+        }
     }
 }

libs/host/Configuration/Options.cs

@@ -52,7 +52,7 @@ internal sealed class Options
         public string SegmentSize { get; set; }
 
         [MemorySizeValidation]
-        [Option('i', "index", Required = false, HelpText = "Size of hash index in bytes (rounds down to power of 2)")]
+        [Option('i', "index", Required = false, HelpText = "Start size of hash index in bytes (rounds down to power of 2)")]
         public string IndexSize { get; set; }
 
         [MemorySizeValidation(false)]
@@ -64,11 +64,11 @@ internal sealed class Options
         public int MutablePercent { get; set; }
 
         [MemorySizeValidation(false)]
-        [Option("obj-total-memory", Required = false, HelpText = "Total object store log memory used including heap memory in bytes")]
-        public string ObjectStoreTotalMemorySize { get; set; }
+        [Option("obj-heap-memory", Required = false, HelpText = "Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap)")]
+        public string ObjectStoreHeapMemorySize { get; set; }
 
         [MemorySizeValidation]
-        [Option("obj-memory", Required = false, HelpText = "Object store log memory used in bytes excluding heap memory")]
+        [Option("obj-log-memory", Required = false, HelpText = "Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log)")]
         public string ObjectStoreLogMemorySize { get; set; }
 
         [MemorySizeValidation]
@@ -80,7 +80,7 @@ internal sealed class Options
         public string ObjectStoreSegmentSize { get; set; }
 
         [MemorySizeValidation]
-        [Option("obj-index", Required = false, HelpText = "Size of object store hash index in bytes (rounds down to power of 2)")]
+        [Option("obj-index", Required = false, HelpText = "Start size of object store hash index in bytes (rounds down to power of 2)")]
         public string ObjectStoreIndexSize { get; set; }
 
         [MemorySizeValidation(false)]
@@ -575,7 +575,7 @@ public GarnetServerOptions GetServerOptions(ILogger logger = null)
                 IndexSize = IndexSize,
                 IndexMaxSize = IndexMaxSize,
                 MutablePercent = MutablePercent,
-                ObjectStoreTotalMemorySize = ObjectStoreTotalMemorySize,
+                ObjectStoreHeapMemorySize = ObjectStoreHeapMemorySize,
                 ObjectStoreLogMemorySize = ObjectStoreLogMemorySize,
                 ObjectStorePageSize = ObjectStorePageSize,
                 ObjectStoreSegmentSize = ObjectStoreSegmentSize,

libs/host/GarnetServer.cs

@@ -52,7 +52,7 @@ public class GarnetServer : IDisposable
         protected StoreWrapper storeWrapper;
 
         // IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6.
-        readonly string version = "1.0.29";
+        readonly string version = "1.0.30";
 
         /// <summary>
         /// Resp protocol version
@@ -198,6 +198,14 @@ private void InitializeServer()
 
             logger?.LogTrace("TLS is {tlsEnabled}", opts.TlsOptions == null ? "disabled" : "enabled");
 
+            if (logger != null)
+            {
+                var configMemoryLimit = (store.IndexSize * 64) + store.Log.MaxMemorySizeBytes + (store.ReadCache?.MaxMemorySizeBytes ?? 0) + (appendOnlyFile?.MaxMemorySizeBytes ?? 0);
+                if (objectStore != null)
+                    configMemoryLimit += objectStore.IndexSize * 64 + objectStore.Log.MaxMemorySizeBytes + (objectStore.ReadCache?.MaxMemorySizeBytes ?? 0) + (objectStoreSizeTracker?.TargetSize ?? 0);
+                logger.LogInformation("Total configured memory limit: {configMemoryLimit}", configMemoryLimit);
+            }
+
             // Create Garnet TCP server if none was provided.
             this.server ??= new GarnetServerTcp(opts.Address, opts.Port, 0, opts.TlsOptions, opts.NetworkSendThrottleMax, logger);
 
@@ -274,7 +282,7 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana
             if (!opts.DisableObjects)
             {
                 objKvSettings = opts.GetObjectStoreSettings(this.loggerFactory?.CreateLogger("TsavoriteKV  [obj]"),
-                    out var objTotalMemorySize);
+                    out var objHeapMemorySize);
 
                 // Run checkpoint on its own thread to control p99
                 objKvSettings.ThrottleCheckpointFlushDelayMs = opts.CheckpointThrottleFlushDelayMs;
@@ -296,8 +304,8 @@ private void CreateObjectStore(IClusterFactory clusterFactory, CustomCommandMana
                         () => new GarnetObjectSerializer(customCommandManager))
                     , (allocatorSettings, storeFunctions) => new(allocatorSettings, storeFunctions));
 
-                if (objTotalMemorySize > 0)
-                    objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objTotalMemorySize,
+                if (objHeapMemorySize > 0)
+                    objectStoreSizeTracker = new CacheSizeTracker(objectStore, objKvSettings, objHeapMemorySize,
                         this.loggerFactory);
             }
         }

libs/host/defaults.conf

@@ -18,7 +18,7 @@
 	/* Size of each log segment in bytes on disk (rounds down to power of 2) */
 	"SegmentSize" : "1g",
 
-	/* Size of hash index in bytes (rounds down to power of 2) */
+	/* Start size of hash index in bytes (rounds down to power of 2) */
 	"IndexSize" : "128m",
 
 	/* Max size of hash index in bytes (rounds down to power of 2) */
@@ -27,10 +27,10 @@
 	/* Percentage of log memory that is kept mutable */
 	"MutablePercent" : 90,
 
-	/* Total object store log memory used including heap memory in bytes */
-	"ObjectStoreTotalMemorySize" : "",
+	/* Object store heap memory size in bytes (Sum of size taken up by all object instances in the heap) */
+	"ObjectStoreHeapMemorySize" : "",
 
-	/* Object store log memory used in bytes excluding heap memory */
+	/* Object store log memory used in bytes (Size of only the log with references to heap objects, excludes size of heap memory consumed by the objects themselves referred to from the log) */
 	"ObjectStoreLogMemorySize" : "32m",
 
 	/* Size of each object store page in bytes (rounds down to power of 2) */
@@ -39,7 +39,7 @@
 	/* Size of each object store log segment in bytes on disk (rounds down to power of 2) */
 	"ObjectStoreSegmentSize" : "32m",
 
-	/* Size of object store hash index in bytes (rounds down to power of 2) */
+	/* Start size of object store hash index in bytes (rounds down to power of 2) */
 	"ObjectStoreIndexSize" : "16m",
 
 	/* Max size of object store hash index in bytes (rounds down to power of 2) */

libs/server/API/GarnetApi.cs

@@ -93,6 +93,18 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM
 
         #endregion
 
+        #region EXPIRETIME
+
+        /// <inheritdoc />
+        public GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output)
+            => storageSession.EXPIRETIME(ref key, storeType, ref output, ref context, ref objectContext);
+
+        /// <inheritdoc />
+        public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output)
+            => storageSession.EXPIRETIME(ref key, storeType, ref output, ref context, ref objectContext, milliseconds: true);
+
+        #endregion
+
         #region SET
         /// <inheritdoc />
         public GarnetStatus SET(ref SpanByte key, ref SpanByte value)
@@ -181,6 +193,18 @@ public GarnetStatus PEXPIRE(ArgSlice key, TimeSpan expiry, out bool timeoutSet,
 
         #endregion
 
+        #region EXPIREAT
+
+        /// <inheritdoc />
+        public GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None)
+            => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectContext);
+
+        /// <inheritdoc />
+        public GarnetStatus PEXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None)
+             => storageSession.EXPIREAT(key, expiryTimestamp, out timeoutSet, storeType, expireOption, ref context, ref objectContext, milliseconds: true);
+
+        #endregion
+
         #region PERSIST
         /// <inheritdoc />
         public unsafe GarnetStatus PERSIST(ArgSlice key, StoreType storeType = StoreType.All)

libs/server/API/GarnetWatchApi.cs

@@ -77,6 +77,24 @@ public GarnetStatus PTTL(ref SpanByte key, StoreType storeType, ref SpanByteAndM
 
         #endregion
 
+        #region EXPIRETIME
+
+        /// <inheritdoc />
+        public GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output)
+        {
+            garnetApi.WATCH(new ArgSlice(ref key), storeType);
+            return garnetApi.EXPIRETIME(ref key, storeType, ref output);
+        }
+
+        /// <inheritdoc />
+        public GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output)
+        {
+            garnetApi.WATCH(new ArgSlice(ref key), storeType);
+            return garnetApi.PEXPIRETIME(ref key, storeType, ref output);
+        }
+
+        #endregion
+
         #region SortedSet Methods
 
         /// <inheritdoc />

libs/server/API/IGarnetApi.cs

@@ -176,6 +176,32 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi
 
         #endregion
 
+        #region EXPIREAT
+
+        /// <summary>
+        /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in seconds
+        /// </summary>
+        /// <param name="key">Key</param>
+        /// <param name="expiryTimestamp">Absolute Unix timestamp in seconds</param>
+        /// <param name="timeoutSet">Whether timeout was set by the call</param>
+        /// <param name="storeType">Store type: main, object, or both</param>
+        /// <param name="expireOption">Expire option</param>
+        /// <returns></returns>
+        GarnetStatus EXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None);
+
+        /// <summary>
+        /// Set a timeout on key using absolute Unix timestamp (seconds since January 1, 1970) in milliseconds
+        /// </summary>
+        /// <param name="key">Key</param>
+        /// <param name="expiryTimestamp">Absolute Unix timestamp in milliseconds</param>
+        /// <param name="timeoutSet">Whether timeout was set by the call</param>
+        /// <param name="storeType">Store type: main, object, or both</param>
+        /// <param name="expireOption">Expire option</param>
+        /// <returns></returns>
+        GarnetStatus PEXPIREAT(ArgSlice key, long expiryTimestamp, out bool timeoutSet, StoreType storeType = StoreType.All, ExpireOption expireOption = ExpireOption.None);
+
+        #endregion
+
         #region PERSIST
         /// <summary>
         /// PERSIST
@@ -1059,6 +1085,28 @@ public interface IGarnetReadApi
 
         #endregion
 
+        #region EXPIRETIME
+
+        /// <summary>
+        /// Returns the absolute Unix timestamp (since January 1, 1970) in seconds at which the given key will expire.
+        /// </summary>
+        /// <param name="key">The key to get the expiration time for.</param>
+        /// <param name="storeType">The type of store to retrieve the key from.</param>
+        /// <param name="output">The output containing the expiration time.</param>
+        /// <returns>The status of the operation.</returns>
+        GarnetStatus EXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output);
+
+        /// <summary>
+        /// Returns the absolute Unix timestamp (since January 1, 1970) in milliseconds at which the given key will expire.
+        /// </summary>
+        /// <param name="key">The key to get the expiration time for.</param>
+        /// <param name="storeType">The type of store to retrieve the key from.</param>
+        /// <param name="output">The output containing the expiration time.</param>
+        /// <returns>The status of the operation.</returns>
+        GarnetStatus PEXPIRETIME(ref SpanByte key, StoreType storeType, ref SpanByteAndMemory output);
+
+        #endregion
+
         #region SortedSet Methods
 
         /// <summary>

libs/server/ExpireOption.cs

@@ -1,32 +1,43 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT license.
 
+using System;
+
 namespace Garnet.server
 {
     /// <summary>
     /// Expire option
     /// </summary>
+    [Flags]
     public enum ExpireOption : byte
     {
         /// <summary>
         /// None
         /// </summary>
-        None,
+        None = 0,
         /// <summary>
         /// Set expiry only when the key has no expiry
         /// </summary>
-        NX,
+        NX = 1 << 0,
         /// <summary>
         /// Set expiry only when the key has an existing expiry 
         /// </summary>
-        XX,
+        XX = 1 << 1,
         /// <summary>
         /// Set expiry only when the new expiry is greater than current one
         /// </summary>
-        GT,
+        GT = 1 << 2,
         /// <summary>
         /// Set expiry only when the new expiry is less than current one
         /// </summary>
-        LT
+        LT = 1 << 3,
+        /// <summary>
+        /// Set expiry only when the key has an existing expiry and the new expiry is greater than current one
+        /// </summary>
+        XXGT = XX | GT,
+        /// <summary>
+        /// Set expiry only when the key has an existing expiry and the new expiry is less than current one
+        /// </summary>
+        XXLT = XX | LT,
     }
 }