fix: networkmanager cannot be started within OnClientStopped or OnServerStopped callbacks

.yamato/project.metafile

@@ -12,7 +12,7 @@
     # smaller_flavor --> An override for flavor that determines the VM size/resources for lighter weight jobs that can run on a smaller vm
     # larger_flavor --> An override for flavor that determines the VM size/resources for heavier weight jobs that can need a bigger vm
     # standalone --> Specifies the build target platform (e.g., StandaloneLinux64, Android, IOS)
-    # model --> Defines specific hardware model requirements (e.g., rtx2080, iPhone model 13). Notice that trunk currently (19.08.2025) has 13.0 as minimal iOS version which devices below this are not supporting
+    # model --> Defines specific hardware model requirements (e.g., iPhone model 13). Notice that trunk currently (19.08.2025) has 13.0 as minimal iOS version which devices below this are not supporting
     # base --> Indicates the base operating system for build operations (e.g., win, mac)
     # architecture --> Specifies the target CPU architecture (e.g., armv7, arm64)
 
@@ -50,15 +50,13 @@ test_platforms:
           smaller_flavor: b1.medium
           larger_flavor: b1.xlarge
           standalone: StandaloneLinux64
-          model: rtx2080
         - name: win
           type: Unity::VM
           image: package-ci/win10:v4
           flavor: b1.large
           smaller_flavor: b1.medium
           larger_flavor: b1.xlarge
           standalone: StandaloneWindows64
-          model: rtx2080
         - name: mac
           type: Unity::VM::osx
           image: package-ci/macos-13-arm64:v4 # ARM64 to support M1 model (below)

com.unity.netcode.gameobjects/CHANGELOG.md

@@ -41,6 +41,7 @@ Additional documentation and release notes are available at [Multiplayer Documen
 
 ### Fixed
 
+- Fixed issue where starting the NetworkManager within `OnClientStopped` or `OnServerStopped` resulted in a broken `NetworkManager` state. (#3908)
 - Fixed issue where an attachable could log an error upon being de-spawned during shutdown. (#3895)
 - NestedNetworkVariables initialized with no value no longer throw an error. (#3891)
 - Fixed `NetworkShow` behavior when it is called twice. (#3867)

com.unity.netcode.gameobjects/Documentation~/components/core/networkmanager.md

@@ -169,8 +169,6 @@ Subscribe to the `NetworkManager.OnClientDisconnectCallback` event to receive no
 - On the client-side, the client identifier parameter is the identifier assigned to the client.
   - _The exception to this is when a client is disconnected before its connection is approved._
 
-You can also use the `NetworkManager.OnServerStopped` and `NetworkManager.OnClientStopped` callbacks to get local notifications when the server or client stops respectively.
-
 ### Connection notification manager example
 
 Below is one example of how you can provide client connect and disconnect notifications to any type of NetworkBehaviour or MonoBehaviour derived component.
@@ -256,3 +254,32 @@ public class ConnectionNotificationManager : MonoBehaviour
     }
 }
 ```
+
+## Additional NetworkManager notifications
+
+### Instantiation and destroying
+
+There are times when it could be useful to be notified when a NetworkManager has been instantiated or is about to be destroyed. There are two static NetworkManager events you can use for this:
+
+- NetworkManager.OnInstantiated: This is invoked when a NetworkManager is instantiated.
+- NetworkManager.OnDestroying: This is invoked when a NetworkManager is about to be destroyed.
+
+### When a NetworkManager is stopped
+
+You will almost always want to know when a NetworkManager has finished shutting down ("stopped"). This can be useful to know when it is safe to transition back to a main menu scene or you might want to perform other similar types of tasks. However, knowing when a NetworkManager has been stopped does not help if you want to save off any state that might exist on spawned objects because at that point everything will have been de-spawned and destroyed. If you run into this scenario then you can use the following event notification:
+
+- [NetworkManager.OnPreShutdown](https://docs.unity3d.com/Packages/com.unity.netcode.gameobjects@2.10/api/Unity.Netcode.NetworkManager.html#Unity_Netcode_NetworkManager_OnPreShutdown): This is invoked prior to finalizing the NetworkManager shutdown process. Any remaining spawned objects will still be instantiated and spawned when this event is invoked.
+
+Similar to the started events, there are two stopped events you can subscribe to that lets you know the NetworkManager is completely shutdown ("stopped"):
+
+- [NetworkManager.OnClientStopped](https://docs.unity3d.com/Packages/com.unity.netcode.gameobjects@2.10/api/Unity.Netcode.NetworkManager.html#Unity_Netcode_NetworkManager_OnClientStopped): This is invoked on a Host or client when NetworkManager has completely shutdown and is ready to be restarted.
+- [NetworkManager.OnServerStopped](https://docs.unity3d.com/Packages/com.unity.netcode.gameobjects@2.10/api/Unity.Netcode.NetworkManager.html#Unity_Netcode_NetworkManager_OnServerStopped): This is invoked on a Host or Server when NetworkManager has completely shutdown and is ready to be restarted.
+
+Since a host is both a client and a server, the event invocation order is:
+
+- OnClientStopped
+- OnServerStopped
+  - _Only if the NetworkManager instance is not restarted during `OnClientStopped`_.
+
+> [!NOTE]
+> If you start the NetworkManager during `NetworkManager.OnClientStopped`, then upon the NetworkManager having restarted successfully it will skip the invocation of `OnServerStopped` since it is no longer shutdown.

com.unity.netcode.gameobjects/Runtime/Core/NetworkManager.cs

@@ -1674,19 +1674,8 @@ internal void ShutdownInternal()
                 ConnectionManager.InvokeOnClientDisconnectCallback(LocalClientId);
             }
 
-            if (ConnectionManager.LocalClient.IsClient)
-            {
-                // If we were a client, we want to know if we were a host
-                // client or not. (why we pass in "IsServer")
-                OnClientStopped?.Invoke(ConnectionManager.LocalClient.IsServer);
-            }
-
-            if (ConnectionManager.LocalClient.IsServer)
-            {
-                // If we were a server, we want to know if we were a host
-                // or not. (why we pass in "IsClient")
-                OnServerStopped?.Invoke(ConnectionManager.LocalClient.IsClient);
-            }
+            // Save off the last local client settings
+            var localClient = ConnectionManager.LocalClient;
 
             // In the event shutdown is invoked within OnClientStopped or OnServerStopped, set it to false again
             m_ShuttingDown = false;
@@ -1706,6 +1695,21 @@ internal void ShutdownInternal()
             // can unsubscribe from tick updates and such.
             NetworkTimeSystem?.Shutdown();
             NetworkTickSystem = null;
+
+
+            if (localClient.IsClient)
+            {
+                // If we were a client, we want to know if we were a host
+                // client or not. (why we pass in "IsServer")
+                OnClientStopped?.Invoke(localClient.IsServer);
+            }
+
+            if (localClient.IsServer)
+            {
+                // If we were a server, we want to know if we were a host
+                // or not. (why we pass in "IsClient")
+                OnServerStopped?.Invoke(localClient.IsClient);
+            }
         }
 
         // Ensures that the NetworkManager is cleaned up before OnDestroy is run on NetworkObjects and NetworkBehaviours when quitting the application.

com.unity.netcode.gameobjects/Runtime/Spawning/NetworkSpawnManager.cs

@@ -1269,7 +1269,7 @@ internal unsafe void UpdateNetworkObjectSceneChanges()
             foreach (var entry in NetworkObjectsToSynchronizeSceneChanges)
             {
                 // If it fails the first update then don't add for updates
-                if (!entry.Value.UpdateForSceneChanges())
+                if (entry.Value != null && !entry.Value.UpdateForSceneChanges())
                 {
                     CleanUpDisposedObjects.Push(entry.Key);
                 }

com.unity.netcode.gameobjects/Tests/Runtime/NetworkManagerStartStopTests.cs

@@ -0,0 +1,191 @@
+using System.Collections;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using NUnit.Framework;
+using Unity.Netcode.TestHelpers.Runtime;
+using UnityEngine.TestTools;
+
+namespace Unity.Netcode.RuntimeTests
+{
+    [TestFixture(NetworkTopologyTypes.ClientServer)]
+    internal class NetworkManagerStartStopTests : NetcodeIntegrationTest
+    {
+        private const int k_NumberOfSessions = 5;
+        protected override int NumberOfClients => 2;
+        private OnClientStoppedHandler m_StoppedHandler;
+        private int m_ExpectedNumberOfClients = 0;
+        public NetworkManagerStartStopTests(NetworkTopologyTypes networkTopologyType) : base(networkTopologyType, HostOrServer.Host) { }
+
+        /// <summary>
+        /// This test will not work with the CMB service since it requires the service
+        /// to remain active after all clients have disconnected.
+        /// </summary>
+        protected override bool UseCMBService()
+        {
+            return false;
+        }
+
+        private void ShutdownIfListening()
+        {
+            var networkManager = m_StoppedHandler.NetworkManager;
+            if (networkManager.IsListening)
+            {
+                m_StoppedHandler.NetworkManager.Shutdown();
+            }
+        }
+
+        private bool NetworkManagerCompletedSessionCount(StringBuilder errorLog)
+        {
+            // Once the session count is decremented to zero the condition has been met.
+            if (m_StoppedHandler.SessionCount != 0)
+            {
+                // If we are a host, then only shutdown once all clients have reconnected
+                if (m_StoppedHandler.IsSessionAuthority && m_StoppedHandler.NetworkManager.ConnectedClientsIds.Count != m_ExpectedNumberOfClients)
+                {
+                    errorLog.Append($"[{m_StoppedHandler.NetworkManager.name}] Waiting for {m_ExpectedNumberOfClients} clients to connect but there are only {m_StoppedHandler.NetworkManager.ConnectedClientsIds.Count} connected!");
+                    return false;
+                }
+                ShutdownIfListening();
+                errorLog.Append($"[{m_StoppedHandler.NetworkManager.name}] Still has a session count of {m_StoppedHandler.SessionCount}!");
+            }
+            return errorLog.Length == 0;
+        }
+
+        [UnityTest]
+        public IEnumerator StartFromWithinOnClientStopped()
+        {
+            var authority = GetAuthorityNetworkManager();
+            m_ExpectedNumberOfClients = authority.ConnectedClientsIds.Count;
+
+            // Validate a client can disconnect and immediately reconnect from within OnClientStopped
+            m_StoppedHandler = new OnClientStoppedHandler(k_NumberOfSessions, GetNonAuthorityNetworkManager());
+            ShutdownIfListening();
+            yield return WaitForConditionOrTimeOut(NetworkManagerCompletedSessionCount);
+            AssertOnTimeout($"Not all {nameof(NetworkManager)} instances finished their sessions!");
+
+            // Validate a host can disconnect and immediately reconnect from within OnClientStopped
+            m_StoppedHandler = new OnHostStoppedHandler(k_NumberOfSessions, authority, m_NetworkManagers.ToList());
+            ShutdownIfListening();
+            yield return WaitForConditionOrTimeOut(NetworkManagerCompletedSessionCount);
+            AssertOnTimeout($"Not all {nameof(NetworkManager)} instances finished their sessions!");
+
+            // Verify OnServerStopped is not invoked if NetworkManager is started again within OnClientStopped (it should not invoke if it is listening).
+            Assert.False((m_StoppedHandler as OnHostStoppedHandler).OnServerStoppedInvoked, $"{nameof(NetworkManager.OnServerStopped)} was invoked when it should not have been invoked!");
+        }
+    }
+
+    internal class OnHostStoppedHandler : OnClientStoppedHandler
+    {
+        public bool OnServerStoppedInvoked = false;
+
+        private List<NetworkManager> m_Clients = new List<NetworkManager>();
+
+        private Networking.Transport.NetworkEndpoint m_Endpoint;
+
+        protected override void OnClientStopped(bool wasHost)
+        {
+            m_Endpoint.Port++;
+            var unityTransport = (Transports.UTP.UnityTransport)NetworkManager.NetworkConfig.NetworkTransport;
+            unityTransport.SetConnectionData(m_Endpoint);
+            // Make sure all clients are shutdown or shutting down
+            foreach (var networkManager in m_Clients)
+            {
+                if (networkManager.IsListening && !networkManager.ShutdownInProgress)
+                {
+                    networkManager.Shutdown();
+                }
+            }
+
+            base.OnClientStopped(wasHost);
+            if (SessionCount != 0)
+            {
+                NetworkManager.StartCoroutine(StartClients());
+            }
+
+        }
+
+        private IEnumerator StartClients()
+        {
+            var nextPhase = false;
+            var timeout = UnityEngine.Time.realtimeSinceStartup + 5.0f;
+            while (!nextPhase)
+            {
+                if (!nextPhase && timeout < UnityEngine.Time.realtimeSinceStartup)
+                {
+                    Assert.Fail($"Timed out waiting for all {nameof(NetworkManager)} instances to shutdown!");
+                    yield break;
+                }
+
+                nextPhase = true;
+                foreach (var networkManager in m_Clients)
+                {
+                    if (networkManager.ShutdownInProgress || networkManager.IsListening)
+                    {
+                        nextPhase = false;
+                    }
+                }
+                yield return null;
+            }
+
+            // Now, start all of the clients and have them connect again
+            foreach (var networkManager in m_Clients)
+            {
+                var unityTransport = (Transports.UTP.UnityTransport)networkManager.NetworkConfig.NetworkTransport;
+                unityTransport.SetConnectionData(m_Endpoint);
+                networkManager.StartClient();
+            }
+        }
+
+        public OnHostStoppedHandler(int numberOfSessions, NetworkManager authority, List<NetworkManager> networkManagers) : base(numberOfSessions, authority)
+        {
+            m_Endpoint = ((Transports.UTP.UnityTransport)authority.NetworkConfig.NetworkTransport).GetLocalEndpoint();
+            networkManagers.Remove(authority);
+            m_Clients = networkManagers;
+            authority.OnServerStopped += OnServerStopped;
+        }
+
+        private void OnServerStopped(bool wasHost)
+        {
+            OnServerStoppedInvoked = SessionCount != 0;
+        }
+    }
+
+    internal class OnClientStoppedHandler
+    {
+        public NetworkManager NetworkManager { get; private set; }
+
+        public int SessionCount { get; private set; }
+        public bool IsSessionAuthority { get; private set; }
+
+        protected virtual void OnClientStopped(bool wasHost)
+        {
+            SessionCount--;
+            if (SessionCount <= 0)
+            {
+                NetworkManager.OnClientStopped -= OnClientStopped;
+                return;
+            }
+
+            if (wasHost)
+            {
+                NetworkManager.StartHost();
+            }
+            else
+            {
+                NetworkManager.StartClient();
+            }
+        }
+
+        public OnClientStoppedHandler(int sessionCount, NetworkManager networkManager)
+        {
+            NetworkManager = networkManager;
+            NetworkManager.OnClientStopped += OnClientStopped;
+            SessionCount = sessionCount;
+            IsSessionAuthority = networkManager.IsServer || networkManager.LocalClient.IsSessionOwner;
+        }
+
+        public OnClientStoppedHandler() { }
+
+    }
+}