Migration Misc Fixes (#552)

vazois · web-flow · commit 27257740ac46 · 2024-08-03T11:32:10.000-07:00
* separate error message when adding keySlice to migration tracker

* safely add and remove sessions in migration task store

* fix slotsrange option

* safely deal with multiple dispose calls to MigrateSession

* fix bug when iterating stores

* fix stores window scan tracking when copyOption is not enabled

* remove logger messages

* release version 1.0.18

* add slotsrange data test

* add back some log messages

* nit; change info to trace message
diff --git a/.azure/pipelines/azure-pipelines-external-release.yml b/.azure/pipelines/azure-pipelines-external-release.yml
@@ -3,7 +3,7 @@
 #      1) update the name: string below (line 6)     -- this is the version for the nuget package (e.g. 1.0.0)
 #      2) update \libs\host\GarnetServer.cs readonly string version  (~line 45)   -- NOTE - these two values need to be the same 
 ###################################### 
-name: 1.0.17
+name: 1.0.18
 trigger:
   branches:
     include:
diff --git a/libs/cluster/CmdStrings.cs b/libs/cluster/CmdStrings.cs
@@ -56,6 +56,7 @@ static class CmdStrings
         public static ReadOnlySpan<byte> RESP_ERR_GENERIC_MIGRATE_TO_MYSELF => "ERR Can't MIGRATE to myself"u8;
         public static ReadOnlySpan<byte> RESP_ERR_GENERIC_INCOMPLETESLOTSRANGE => "ERR incomplete slotrange"u8;
         public static ReadOnlySpan<byte> RESP_ERR_GENERIC_SLOTNOTMIGRATING => "ERR slot state not set to MIGRATING state"u8;
+        public static ReadOnlySpan<byte> RESP_ERR_GENERIC_FAILEDTOADDKEY => "ERR Failed to add key for migration tracking"u8;
         public static ReadOnlySpan<byte> RESP_ERR_GENERIC_PARSING => "ERR Parsing error"u8;
 
         /// <summary>
diff --git a/libs/cluster/Server/ClusterManager.cs b/libs/cluster/Server/ClusterManager.cs
@@ -123,11 +123,7 @@ public void Start()
         public void FlushConfig()
         {
             lock (this)
-            {
-                logger?.LogTrace("Start FlushConfig {path}", clusterConfigDevice.FileName);
                 ClusterUtils.WriteInto(clusterConfigDevice, pool, 0, currentConfig.ToByteArray(), logger: logger);
-                logger?.LogTrace("End FlushConfig {path}", clusterConfigDevice.FileName);
-            }
         }
 
         /// <summary>
diff --git a/libs/cluster/Server/ClusterManagerSlotState.cs b/libs/cluster/Server/ClusterManagerSlotState.cs
@@ -48,7 +48,7 @@ public bool TryAddSlots(HashSet<int> slots, out int slotAssigned)
                     break;
             }
             FlushConfig();
-            logger?.LogTrace("ADD SLOTS {slots}", GetRange(slots.ToArray()));
+            logger?.LogTrace("AddSlots {slots}", GetRange(slots.ToArray()));
             return true;
         }
 
@@ -78,7 +78,7 @@ public bool TryRemoveSlots(HashSet<int> slots, out int notLocalSlot)
                     break;
             }
             FlushConfig();
-            logger?.LogTrace("REMOVE SLOTS {slots}", string.Join(",", slots));
+            logger?.LogTrace("RemoveSlots {slots}", GetRange(slots.ToArray()));
             return true;
         }
 
@@ -143,8 +143,7 @@ public bool TryPrepareSlotForMigration(int slot, string nodeid, out ReadOnlySpan
                     break;
             }
             FlushConfig();
-
-            logger?.LogInformation("MIGRATE {slot} TO {currentConfig.GetWorkerAddressFromNodeId(nodeid)}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
+            logger?.LogTrace("SetSlot MIGRATING {slot} TO {nodeId}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
             return true;
         }
 
@@ -211,8 +210,7 @@ public bool TryPrepareSlotsForMigration(HashSet<int> slots, string nodeid, out R
                     break;
             }
             FlushConfig();
-
-            logger?.LogInformation("MIGRATE {slot} TO {migrating node}", string.Join(' ', slots), currentConfig.GetWorkerAddressFromNodeId(nodeid));
+            logger?.LogTrace("SetSlotsRange MIGRATING {slot} TO {nodeId}", GetRange(slots.ToArray()), currentConfig.GetWorkerAddressFromNodeId(nodeid));
             return true;
         }
 
@@ -266,8 +264,7 @@ public bool TryPrepareSlotForImport(int slot, string nodeid, out ReadOnlySpan<by
                     break;
             }
             FlushConfig();
-
-            logger?.LogInformation("IMPORT {slot} FROM {currentConfig.GetWorkerAddressFromNodeId(nodeid)}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
+            logger?.LogTrace("SetSlot IMPORTING {slot} TO {nodeId}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
             return true;
         }
 
@@ -329,8 +326,8 @@ public bool TryPrepareSlotsForImport(HashSet<int> slots, string nodeid, out Read
                 if (Interlocked.CompareExchange(ref currentConfig, newConfig, current) == current)
                     break;
             }
-
-            logger?.LogInformation("IMPORT {slot} FROM {importingNode}", string.Join(' ', slots), currentConfig.GetWorkerAddressFromNodeId(nodeid));
+            FlushConfig();
+            logger?.LogTrace("SetSlotsRange IMPORTING {slot} TO {nodeId}", GetRange(slots.ToArray()), currentConfig.GetWorkerAddressFromNodeId(nodeid));
             return true;
         }
 
@@ -364,7 +361,7 @@ public bool TryPrepareSlotForOwnershipChange(int slot, string nodeid, out ReadOn
                         break;
                 }
                 FlushConfig();
-                logger?.LogInformation("SLOT {slot} IMPORTED TO {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
+                logger?.LogTrace("SLOT {slot} MIGRATED TO {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
                 return true;
             }
             else if (current.GetState((ushort)slot) is SlotState.IMPORTING)
@@ -385,7 +382,7 @@ public bool TryPrepareSlotForOwnershipChange(int slot, string nodeid, out ReadOn
                         break;
                 }
                 FlushConfig();
-                logger?.LogInformation("SLOT {slot} IMPORTED FROM {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
+                logger?.LogTrace("SLOT {slot} IMPORTED FROM {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));
                 return true;
             }
             return true;
@@ -418,7 +415,7 @@ public bool TryPrepareSlotsForOwnershipChange(HashSet<int> slots, string nodeid,
             }
 
             FlushConfig();
-            logger?.LogInformation("SLOT {slot} IMPORTED TO {endpoint}", slots, currentConfig.GetWorkerAddressFromNodeId(nodeid));
+            logger?.LogTrace("Slots {slot} IMPORTED TO {endpoint}", GetRange(slots.ToArray()), currentConfig.GetWorkerAddressFromNodeId(nodeid));
             return true;
         }
 
diff --git a/libs/cluster/Server/Migration/MigrateSession.cs b/libs/cluster/Server/Migration/MigrateSession.cs
@@ -45,6 +45,7 @@ internal sealed unsafe partial class MigrateSession : IDisposable
         readonly List<(int, int)> _slotRanges;
         readonly Dictionary<ArgSlice, KeyMigrationStatus> _keys;
         SingleWriterMultiReaderLock _keyDictLock;
+        SingleWriterMultiReaderLock _disposed;
 
         readonly HashSet<int> _sslots;
         readonly CancellationTokenSource _cts = new();
@@ -222,6 +223,7 @@ internal MigrateSession(
         /// </summary>
         public void Dispose()
         {
+            if (!_disposed.TryWriteLock()) return;
             _cts?.Cancel();
             _cts?.Dispose();
             _gcs.Dispose();
diff --git a/libs/cluster/Server/Migration/MigrateSessionSlots.cs b/libs/cluster/Server/Migration/MigrateSessionSlots.cs
@@ -22,8 +22,7 @@ public bool MigrateSlotsDriver()
                 while (true)
                 {
                     // Iterate main store
-                    if (!localServerSession.BasicGarnetApi.IterateMainStore(ref mainStoreGetKeysInSlots, storeTailAddress))
-                        return false;
+                    _ = localServerSession.BasicGarnetApi.IterateMainStore(ref mainStoreGetKeysInSlots, storeTailAddress);
 
                     // If did not acquire any keys stop scanning
                     if (_keys.IsNullOrEmpty())
@@ -50,8 +49,7 @@ public bool MigrateSlotsDriver()
                 while (true)
                 {
                     // Iterate object store
-                    if (!localServerSession.BasicGarnetApi.IterateObjectStore(ref objectStoreGetKeysInSlots, objectStoreTailAddress))
-                        return false;
+                    _ = localServerSession.BasicGarnetApi.IterateObjectStore(ref objectStoreGetKeysInSlots, objectStoreTailAddress);
 
                     // If did not acquire any keys stop scanning
                     if (_keys.IsNullOrEmpty())
diff --git a/libs/cluster/Server/Migration/MigrateSessionTaskStore.cs b/libs/cluster/Server/Migration/MigrateSessionTaskStore.cs
@@ -51,8 +51,13 @@ public int GetNumSessions()
                 _lock.ReadLock();
                 if (_disposed) return 0;
 
+                HashSet<MigrateSession> ss = [];
                 for (var i = 0; i < sessions.Length; i++)
-                    count += sessions[i] != null ? 1 : 0;
+                {
+                    if (sessions[i] != null)
+                        _ = ss.Add(sessions[i]);
+                }
+                count = ss.Count;
             }
             finally
             {
@@ -101,6 +106,7 @@ public bool TryAddMigrateSession(
                 _lock.WriteLock();
                 if (_disposed) return false;
 
+                // First iterate and check if corresponding slot is associated to another active migrate session
                 foreach (var slot in mSession.GetSlots)
                 {
                     if (sessions[slot] != null)
@@ -109,8 +115,12 @@ public bool TryAddMigrateSession(
                         success = false;
                         return false;
                     }
-                    sessions[slot] = mSession;
                 }
+
+                // If reached this point all slots to be migrated are not associated with any other session
+                // so we can mark them as being associated with this newly added session
+                foreach (var slot in mSession.GetSlots)
+                    sessions[slot] = mSession;
             }
             catch (Exception ex)
             {
@@ -129,6 +139,11 @@ public bool TryAddMigrateSession(
             return success;
         }
 
+        /// <summary>
+        /// Remove only the provided session instance
+        /// </summary>
+        /// <param name="mSession"></param>
+        /// <returns></returns>
         public bool TryRemove(MigrateSession mSession)
         {
             try
@@ -156,30 +171,27 @@ public bool TryRemove(MigrateSession mSession)
             }
         }
 
+        /// <summary>
+        /// Remove all sessions associated with the provided targetNodeId
+        /// </summary>
+        /// <param name="targetNodeId"></param>
+        /// <returns></returns>
         public bool TryRemove(string targetNodeId)
         {
             try
             {
                 _lock.WriteLock();
                 if (_disposed) return false;
-                HashSet<MigrateSession> mSessions = null;
                 for (var i = 0; i < sessions.Length; i++)
                 {
                     var s = sessions[i];
                     if (s != null && s.GetTargetNodeId.Equals(targetNodeId, StringComparison.Ordinal))
                     {
                         sessions[i] = null;
-                        mSessions ??= [];
-                        _ = mSessions.Add(s);
+                        s.Dispose();
                     }
                 }
 
-                if (mSessions != null)
-                {
-                    foreach (var session in mSessions)
-                        session.Dispose();
-                }
-
                 return true;
             }
             catch (Exception ex)
diff --git a/libs/cluster/Server/Migration/MigrationKeyIterationFunctions.cs b/libs/cluster/Server/Migration/MigrationKeyIterationFunctions.cs
@@ -14,64 +14,6 @@ internal sealed unsafe partial class MigrateSession : IDisposable
     {
         internal sealed class MigrationKeyIterationFunctions
         {
-            internal struct MainStoreMigrateSlots : IScanIteratorFunctions<SpanByte, SpanByte>
-            {
-                readonly MigrateSession session;
-                readonly HashSet<int> slots;
-
-                internal MainStoreMigrateSlots(MigrateSession session, HashSet<int> slots)
-                {
-                    this.session = session;
-                    this.slots = slots;
-                }
-
-                public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult)
-                {
-                    cursorRecordResult = CursorRecordResult.Accept; // default; not used here
-                    var s = HashSlotUtils.HashSlot(ref key);
-
-                    if (slots.Contains(s) && !ClusterSession.Expired(ref value) && !session.WriteOrSendMainStoreKeyValuePair(ref key, ref value))
-                        return false;
-                    return true;
-                }
-                public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult)
-                    => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult);
-                public bool OnStart(long beginAddress, long endAddress) => true;
-                public void OnStop(bool completed, long numberOfRecords) { }
-                public void OnException(Exception exception, long numberOfRecords) { }
-            }
-
-            internal struct ObjectStoreMigrateSlots : IScanIteratorFunctions<byte[], IGarnetObject>
-            {
-                readonly MigrateSession session;
-                readonly HashSet<int> slots;
-
-                internal ObjectStoreMigrateSlots(MigrateSession session, HashSet<int> slots)
-                {
-                    this.session = session;
-                    this.slots = slots;
-                }
-
-                public bool SingleReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult)
-                {
-                    cursorRecordResult = CursorRecordResult.Accept; // default; not used here
-                    var slot = HashSlotUtils.HashSlot(key);
-
-                    if (slots.Contains(slot) && !ClusterSession.Expired(ref value))
-                    {
-                        byte[] objectData = GarnetObjectSerializer.Serialize(value);
-                        if (!session.WriteOrSendObjectStoreKeyValuePair(key, objectData, value.Expiration))
-                            return false;
-                    }
-                    return true;
-                }
-                public bool ConcurrentReader(ref byte[] key, ref IGarnetObject value, RecordMetadata recordMetadata, long numberOfRecords, out CursorRecordResult cursorRecordResult)
-                    => SingleReader(ref key, ref value, recordMetadata, numberOfRecords, out cursorRecordResult);
-                public bool OnStart(long beginAddress, long endAddress) => true;
-                public void OnStop(bool completed, long numberOfRecords) { }
-                public void OnException(Exception exception, long numberOfRecords) { }
-            }
-
             internal unsafe struct MainStoreGetKeysInSlots : IScanIteratorFunctions<SpanByte, SpanByte>
             {
                 MigrationScanIterator iterator;
@@ -148,7 +90,6 @@ public void OnStop(bool completed, long numberOfRecords) { }
                 public void OnException(Exception exception, long numberOfRecords) { }
             }
 
-
             internal struct MigrationScanIterator
             {
                 readonly MigrateSession session;
@@ -198,8 +139,9 @@ public void AdvanceIterator()
                 /// <returns></returns>
                 public bool Consume(ref Span<byte> key)
                 {
-                    // Check if key is within the current processing window
-                    if (currentOffset < offset)
+                    // Check if key is within the current processing window only if _copyOption is set
+                    // in order to skip keys that have been send over to target node but not deleted locally
+                    if (session._copyOption && currentOffset < offset)
                     {
                         currentOffset++;
                         return true;
diff --git a/libs/cluster/Session/MigrateCommand.cs b/libs/cluster/Session/MigrateCommand.cs
@@ -30,6 +30,7 @@ internal enum MigrateCmdParseState : byte
             SLOTOUTOFRANGE,
             NOTMIGRATING,
             MULTI_TRANSFER_OPTION,
+            FAILEDTOADDKEY
         }
 
         private bool HandleCommandParsingErrors(MigrateCmdParseState mpState, string targetAddress, int targetPort, int slotMultiRef)
@@ -48,6 +49,7 @@ private bool HandleCommandParsingErrors(MigrateCmdParseState mpState, string tar
                 MigrateCmdParseState.INCOMPLETESLOTSRANGE => CmdStrings.RESP_ERR_GENERIC_INCOMPLETESLOTSRANGE,
                 MigrateCmdParseState.SLOTOUTOFRANGE => Encoding.ASCII.GetBytes($"ERR Slot {slotMultiRef} out of range."),
                 MigrateCmdParseState.NOTMIGRATING => CmdStrings.RESP_ERR_GENERIC_SLOTNOTMIGRATING,
+                MigrateCmdParseState.FAILEDTOADDKEY => CmdStrings.RESP_ERR_GENERIC_FAILEDTOADDKEY,
                 _ => CmdStrings.RESP_ERR_GENERIC_PARSING,
             };
             while (!RespWriteUtils.WriteError(errorMessage, ref dcurr, dend))
@@ -181,9 +183,12 @@ private bool TryMIGRATE(int count, byte* ptr)
 
                         // Add pointer of current parsed key
                         if (!keys.TryAdd(new ArgSlice(keyPtr, ksize), KeyMigrationStatus.QUEUED))
+                        {
                             logger?.LogWarning($"Failed to add {{key}}", Encoding.ASCII.GetString(keyPtr, ksize));
-                        else
-                            _ = slots.Add(slot);
+                            pstate = MigrateCmdParseState.FAILEDTOADDKEY;
+                            continue;
+                        }
+                        _ = slots.Add(slot);
                     }
                 }
                 else if (option.Equals("SLOTS", StringComparison.OrdinalIgnoreCase))
@@ -228,6 +233,9 @@ private bool TryMIGRATE(int count, byte* ptr)
                 }
                 else if (option.Equals("SLOTSRANGE", StringComparison.OrdinalIgnoreCase))
                 {
+                    if (transferOption == TransferOption.KEYS)
+                        pstate = MigrateCmdParseState.MULTI_TRANSFER_OPTION;
+                    transferOption = TransferOption.SLOTS;
                     slots = [];
                     if (args == 0 || (args & 0x1) > 0)
                     {
diff --git a/libs/host/GarnetServer.cs b/libs/host/GarnetServer.cs
@@ -50,7 +50,7 @@ public class GarnetServer : IDisposable
         protected StoreWrapper storeWrapper;
 
         // IMPORTANT: Keep the version in sync with .azure\pipelines\azure-pipelines-external-release.yml line ~6.
-        readonly string version = "1.0.17";
+        readonly string version = "1.0.18";
 
         /// <summary>
         /// Resp protocol version
diff --git a/test/Garnet.test.cluster/ClusterMigrateTests.cs b/test/Garnet.test.cluster/ClusterMigrateTests.cs
diff --git a/test/Garnet.test.cluster/ClusterTestUtils.cs b/test/Garnet.test.cluster/ClusterTestUtils.cs

Original file line number	Diff line number	Diff line change
`@@ -123,11 +123,7 @@ public void Start()`
`123`	`123`	`public void FlushConfig()`
`124`	`124`	`{`
`125`	`125`	`lock (this)`
`126`		`- {`
`127`		`- logger?.LogTrace("Start FlushConfig {path}", clusterConfigDevice.FileName);`
`128`	`126`	`ClusterUtils.WriteInto(clusterConfigDevice, pool, 0, currentConfig.ToByteArray(), logger: logger);`
`129`		`- logger?.LogTrace("End FlushConfig {path}", clusterConfigDevice.FileName);`
`130`		`- }`
`131`	`127`	`}`
`132`	`128`
`133`	`129`	`/// <summary>`
Original file line number	Diff line number	Diff line change
`@@ -48,7 +48,7 @@ public bool TryAddSlots(HashSet<int> slots, out int slotAssigned)`
`48`	`48`	`break;`
`49`	`49`	`}`
`50`	`50`	`FlushConfig();`
`51`		`- logger?.LogTrace("ADD SLOTS {slots}", GetRange(slots.ToArray()));`
	`51`	`+ logger?.LogTrace("AddSlots {slots}", GetRange(slots.ToArray()));`
`52`	`52`	`return true;`
`53`	`53`	`}`
`54`	`54`
`@@ -78,7 +78,7 @@ public bool TryRemoveSlots(HashSet<int> slots, out int notLocalSlot)`
`78`	`78`	`break;`
`79`	`79`	`}`
`80`	`80`	`FlushConfig();`
`81`		`- logger?.LogTrace("REMOVE SLOTS {slots}", string.Join(",", slots));`
	`81`	`+ logger?.LogTrace("RemoveSlots {slots}", GetRange(slots.ToArray()));`
`82`	`82`	`return true;`
`83`	`83`	`}`
`84`	`84`
`@@ -143,8 +143,7 @@ public bool TryPrepareSlotForMigration(int slot, string nodeid, out ReadOnlySpan`
`143`	`143`	`break;`
`144`	`144`	`}`
`145`	`145`	`FlushConfig();`
`146`		`-`
`147`		`- logger?.LogInformation("MIGRATE {slot} TO {currentConfig.GetWorkerAddressFromNodeId(nodeid)}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`146`	`+ logger?.LogTrace("SetSlot MIGRATING {slot} TO {nodeId}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`148`	`147`	`return true;`
`149`	`148`	`}`
`150`	`149`
`@@ -211,8 +210,7 @@ public bool TryPrepareSlotsForMigration(HashSet<int> slots, string nodeid, out R`
`211`	`210`	`break;`
`212`	`211`	`}`
`213`	`212`	`FlushConfig();`
`214`		`-`
`215`		`- logger?.LogInformation("MIGRATE {slot} TO {migrating node}", string.Join(' ', slots), currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`213`	`+ logger?.LogTrace("SetSlotsRange MIGRATING {slot} TO {nodeId}", GetRange(slots.ToArray()), currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`216`	`214`	`return true;`
`217`	`215`	`}`
`218`	`216`
`@@ -266,8 +264,7 @@ public bool TryPrepareSlotForImport(int slot, string nodeid, out ReadOnlySpan<by`
`266`	`264`	`break;`
`267`	`265`	`}`
`268`	`266`	`FlushConfig();`
`269`		`-`
`270`		`- logger?.LogInformation("IMPORT {slot} FROM {currentConfig.GetWorkerAddressFromNodeId(nodeid)}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`267`	`+ logger?.LogTrace("SetSlot IMPORTING {slot} TO {nodeId}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`271`	`268`	`return true;`
`272`	`269`	`}`
`273`	`270`
`@@ -329,8 +326,8 @@ public bool TryPrepareSlotsForImport(HashSet<int> slots, string nodeid, out Read`
`329`	`326`	`if (Interlocked.CompareExchange(ref currentConfig, newConfig, current) == current)`
`330`	`327`	`break;`
`331`	`328`	`}`
`332`		`-`
`333`		`- logger?.LogInformation("IMPORT {slot} FROM {importingNode}", string.Join(' ', slots), currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`329`	`+ FlushConfig();`
	`330`	`+ logger?.LogTrace("SetSlotsRange IMPORTING {slot} TO {nodeId}", GetRange(slots.ToArray()), currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`334`	`331`	`return true;`
`335`	`332`	`}`
`336`	`333`
`@@ -364,7 +361,7 @@ public bool TryPrepareSlotForOwnershipChange(int slot, string nodeid, out ReadOn`
`364`	`361`	`break;`
`365`	`362`	`}`
`366`	`363`	`FlushConfig();`
`367`		`- logger?.LogInformation("SLOT {slot} IMPORTED TO {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`364`	`+ logger?.LogTrace("SLOT {slot} MIGRATED TO {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`368`	`365`	`return true;`
`369`	`366`	`}`
`370`	`367`	`else if (current.GetState((ushort)slot) is SlotState.IMPORTING)`
`@@ -385,7 +382,7 @@ public bool TryPrepareSlotForOwnershipChange(int slot, string nodeid, out ReadOn`
`385`	`382`	`break;`
`386`	`383`	`}`
`387`	`384`	`FlushConfig();`
`388`		`- logger?.LogInformation("SLOT {slot} IMPORTED FROM {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`385`	`+ logger?.LogTrace("SLOT {slot} IMPORTED FROM {nodeid}", slot, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`389`	`386`	`return true;`
`390`	`387`	`}`
`391`	`388`	`return true;`
`@@ -418,7 +415,7 @@ public bool TryPrepareSlotsForOwnershipChange(HashSet<int> slots, string nodeid,`
`418`	`415`	`}`
`419`	`416`
`420`	`417`	`FlushConfig();`
`421`		`- logger?.LogInformation("SLOT {slot} IMPORTED TO {endpoint}", slots, currentConfig.GetWorkerAddressFromNodeId(nodeid));`
	`418`	`+ logger?.LogTrace("Slots {slot} IMPORTED TO {endpoint}", GetRange(slots.ToArray()), currentConfig.GetWorkerAddressFromNodeId(nodeid));`
`422`	`419`	`return true;`
`423`	`420`	`}`
`424`	`421`
Original file line number	Diff line number	Diff line change
`@@ -51,8 +51,13 @@ public int GetNumSessions()`
`51`	`51`	`_lock.ReadLock();`
`52`	`52`	`if (_disposed) return 0;`
`53`	`53`
	`54`	`+ HashSet<MigrateSession> ss = [];`
`54`	`55`	`for (var i = 0; i < sessions.Length; i++)`
`55`		`- count += sessions[i] != null ? 1 : 0;`
	`56`	`+ {`
	`57`	`+ if (sessions[i] != null)`
	`58`	`+ _ = ss.Add(sessions[i]);`
	`59`	`+ }`
	`60`	`+ count = ss.Count;`
`56`	`61`	`}`
`57`	`62`	`finally`
`58`	`63`	`{`
`@@ -101,6 +106,7 @@ public bool TryAddMigrateSession(`
`101`	`106`	`_lock.WriteLock();`
`102`	`107`	`if (_disposed) return false;`
`103`	`108`
	`109`	`+ // First iterate and check if corresponding slot is associated to another active migrate session`
`104`	`110`	`foreach (var slot in mSession.GetSlots)`
`105`	`111`	`{`
`106`	`112`	`if (sessions[slot] != null)`
`@@ -109,8 +115,12 @@ public bool TryAddMigrateSession(`
`109`	`115`	`success = false;`
`110`	`116`	`return false;`
`111`	`117`	`}`
`112`		`- sessions[slot] = mSession;`
`113`	`118`	`}`
	`119`	`+`
	`120`	`+ // If reached this point all slots to be migrated are not associated with any other session`
	`121`	`+ // so we can mark them as being associated with this newly added session`
	`122`	`+ foreach (var slot in mSession.GetSlots)`
	`123`	`+ sessions[slot] = mSession;`
`114`	`124`	`}`
`115`	`125`	`catch (Exception ex)`
`116`	`126`	`{`
`@@ -129,6 +139,11 @@ public bool TryAddMigrateSession(`
`129`	`139`	`return success;`
`130`	`140`	`}`
`131`	`141`
	`142`	`+ /// <summary>`
	`143`	`+ /// Remove only the provided session instance`
	`144`	`+ /// </summary>`
	`145`	`+ /// <param name="mSession"></param>`
	`146`	`+ /// <returns></returns>`
`132`	`147`	`public bool TryRemove(MigrateSession mSession)`
`133`	`148`	`{`
`134`	`149`	`try`
`@@ -156,30 +171,27 @@ public bool TryRemove(MigrateSession mSession)`
`156`	`171`	`}`
`157`	`172`	`}`
`158`	`173`
	`174`	`+ /// <summary>`
	`175`	`+ /// Remove all sessions associated with the provided targetNodeId`
	`176`	`+ /// </summary>`
	`177`	`+ /// <param name="targetNodeId"></param>`
	`178`	`+ /// <returns></returns>`
`159`	`179`	`public bool TryRemove(string targetNodeId)`
`160`	`180`	`{`
`161`	`181`	`try`
`162`	`182`	`{`
`163`	`183`	`_lock.WriteLock();`
`164`	`184`	`if (_disposed) return false;`
`165`		`- HashSet<MigrateSession> mSessions = null;`
`166`	`185`	`for (var i = 0; i < sessions.Length; i++)`
`167`	`186`	`{`
`168`	`187`	`var s = sessions[i];`
`169`	`188`	`if (s != null && s.GetTargetNodeId.Equals(targetNodeId, StringComparison.Ordinal))`
`170`	`189`	`{`
`171`	`190`	`sessions[i] = null;`
`172`		`- mSessions ??= [];`
`173`		`- _ = mSessions.Add(s);`
	`191`	`+ s.Dispose();`
`174`	`192`	`}`
`175`	`193`	`}`
`176`	`194`
`177`		`- if (mSessions != null)`
`178`		`- {`
`179`		`- foreach (var session in mSessions)`
`180`		`- session.Dispose();`
`181`		`- }`
`182`		`-`
`183`	`195`	`return true;`
`184`	`196`	`}`
`185`	`197`	`catch (Exception ex)`