Make connection state more robust to backgrounding tabs (#34622)

And also add some extra metadata around connection state.

I think it's possible for a tab to be backgrounded and have the timeouts (both the interval for checking connection state + the timeout for reconnecting the WS) to be delayed.

To avoid problems here, I'm requiring two consecutive disconnected connection states within `CONNECTION_STATE_CHECK_INTERVAL_MS * 2` of each other.

I also realized I wanted to know a little bit more information here, like if the client ever connected successfully, whether a client reconnected successfully later after a disconnect, and also `connectionCount` (which can be a little misleading, but can be a potential indicator for a bad connection)

GitOrigin-RevId: 29aa1cf348714aadd2e339a4bff01be6d9534115

Author: sshader

npm-packages/convex/src/browser/sync/client.ts

@@ -109,6 +109,22 @@ export type ConnectionState = {
   hasInflightRequests: boolean;
   isWebSocketConnected: boolean;
   timeOfOldestInflightRequest: Date | null;
+  /**
+   * True if the client has ever opened a WebSocket to the "ready" state.
+   */
+  hasEverConnected: boolean;
+  /**
+   * The number of times this client has connected to the Convex backend.
+   *
+   * A number of things can cause the client to reconnect -- server errors,
+   * bad internet, auth expiring. But this number being high is an indication
+   * that the client is having trouble keeping a stable connection.
+   */
+  connectionCount: number;
+  /**
+   * The number of times this client has tried (and failed) to connect to the Convex backend.
+   */
+  connectionRetries: number;
 };
 
 /**
@@ -672,9 +688,13 @@ export class BaseConvexClient {
    * @returns The {@link ConnectionState} with the Convex backend.
    */
   connectionState(): ConnectionState {
+    const wsConnectionState = this.webSocketManager.connectionState();
     return {
       hasInflightRequests: this.requestManager.hasInflightRequests(),
-      isWebSocketConnected: this.webSocketManager.socketState() === "ready",
+      isWebSocketConnected: wsConnectionState.isConnected,
+      hasEverConnected: wsConnectionState.hasEverConnected,
+      connectionCount: wsConnectionState.connectionCount,
+      connectionRetries: wsConnectionState.connectionRetries,
       timeOfOldestInflightRequest:
         this.requestManager.timeOfOldestInflightRequest(),
     };

npm-packages/convex/src/browser/sync/web_socket_manager.ts

@@ -101,6 +101,7 @@ export class WebSocketManager {
   private socket: Socket;
 
   private connectionCount: number;
+  private _hasEverConnected: boolean = false;
   private lastCloseReason: string | null;
 
   /** Upon HTTPS/WSS failure, the first jittered backoff duration, in ms. */
@@ -197,6 +198,7 @@ export class WebSocketManager {
       };
       this.resetServerInactivityTimeout();
       if (this.socket.paused === "no") {
+        this._hasEverConnected = true;
         this.onOpen({
           connectionCount: this.connectionCount,
           lastCloseReason: this.lastCloseReason,
@@ -498,6 +500,20 @@ export class WebSocketManager {
     this.connect();
   }
 
+  connectionState(): {
+    isConnected: boolean;
+    hasEverConnected: boolean;
+    connectionCount: number;
+    connectionRetries: number;
+  } {
+    return {
+      isConnected: this.socket.state === "ready",
+      hasEverConnected: this._hasEverConnected,
+      connectionCount: this.connectionCount,
+      connectionRetries: this.retries,
+    };
+  }
+
   private _logVerbose(message: string) {
     this.logger.logVerbose(message);
   }

npm-packages/dashboard-common/src/lib/deploymentContext.tsx

@@ -1,8 +1,9 @@
 import { ConvexProvider, ConvexReactClient } from "convex/react";
-import { ConvexHttpClient } from "convex/browser";
+import { ConnectionState, ConvexHttpClient } from "convex/browser";
 import {
   createContext,
   ReactNode,
+  useCallback,
   useContext,
   useEffect,
   useLayoutEffect,
@@ -24,6 +25,12 @@ export type DeploymentInfo = (
     }
   | { ok: false; errorCode: string; errorMessage: string }
 ) & {
+  addBreadcrumb: (breadcrumb: {
+    message?: string;
+    data?: {
+      [key: string]: any;
+    };
+  }) => void;
   captureMessage: (msg: string) => void;
   captureException: (e: any) => void;
   reportHttpError: (
@@ -338,36 +345,61 @@ export function WaitForDeploymentApi({
   );
 }
 
+const CONNECTION_STATE_CHECK_INTERVAL_MS = 2500;
+
 function DeploymentWithConnectionState({
   deployment,
   children,
 }: {
   deployment: ConnectedDeployment;
   children: ReactNode;
 }) {
-  const { isSelfHosted, captureMessage } = useContext(DeploymentInfoContext);
+  const { isSelfHosted, captureMessage, addBreadcrumb } = useContext(
+    DeploymentInfoContext,
+  );
   const { client, deploymentUrl, deploymentName } = deployment;
-  const [connectionState, setConnectionState] = useState<
-    "Connected" | "Disconnected" | "LocalDeploymentMismatch" | null
-  >(null);
-  const [isDisconnected, setIsDisconnected] = useState(false);
-  useEffect(() => {
-    const checkConnection = setInterval(async () => {
-      if (connectionState === "LocalDeploymentMismatch") {
-        // Connection status doesn't matter since we're connected to the wrong deployment
-        return;
-      }
+  const [lastObservedConnectionState, setLastObservedConnectionState] =
+    useState<
+      | {
+          state: ConnectionState;
+          time: Date;
+        }
+      | "LocalDeploymentMismatch"
+      | null
+    >(null);
+  const [isDisconnected, setIsDisconnected] = useState<boolean | null>(null);
 
-      // Check WS connection status -- if we're disconnected twice in a row, treat
-      // the deployment as disconnected.
-      const nextConnectionState = client.connectionState();
+  const handleConnectionStateChange = useCallback(
+    async (
+      state: ConnectionState,
+      previousState: {
+        time: Date;
+        state: ConnectionState;
+      } | null,
+    ): Promise<
+      "Unknown" | "Disconnected" | "Connected" | "LocalDeploymentMismatch"
+    > => {
+      if (previousState === null) {
+        return "Unknown";
+      }
       if (
-        nextConnectionState.isWebSocketConnected === false &&
-        connectionState === "Disconnected"
+        previousState.time.getTime() <
+        Date.now() - CONNECTION_STATE_CHECK_INTERVAL_MS * 2
       ) {
-        setIsDisconnected(true);
+        // If the previous state was observed a while ago, consider it stale (maybe the tab
+        // got backgrounded).
+        return "Unknown";
       }
-      if (nextConnectionState.isWebSocketConnected === true) {
+
+      if (state.isWebSocketConnected === false) {
+        if (previousState.state.isWebSocketConnected === false) {
+          // we've been in state `Disconnected` twice in a row, consider the deployment
+          // to be disconnected.
+          return "Disconnected";
+        }
+        return "Unknown";
+      }
+      if (state.isWebSocketConnected === true) {
         // If this is a local deployment, check that the instance name matches what we expect.
         if (deploymentName.startsWith("local-")) {
           let instanceNameResp: Response | null = null;
@@ -381,30 +413,96 @@ function DeploymentWithConnectionState({
           if (instanceNameResp !== null && instanceNameResp.ok) {
             const instanceName = await instanceNameResp.text();
             if (instanceName !== deploymentName) {
-              setConnectionState("LocalDeploymentMismatch");
-              setIsDisconnected(true);
-              return;
+              return "LocalDeploymentMismatch";
             }
           }
         }
-        setIsDisconnected(false);
+        return "Connected";
+      }
+      return "Unknown";
+    },
+    [deploymentName, deploymentUrl],
+  );
+
+  useEffect(() => {
+    // Poll `.connectionState()` every 5 seconds. If we're disconnected twice in a row,
+    // consider the deployment to be disconnected.
+    const checkConnection = setInterval(async () => {
+      if (lastObservedConnectionState === "LocalDeploymentMismatch") {
+        // Connection status doesn't matter since we're connected to the wrong deployment
+        return;
       }
-      setConnectionState(
-        nextConnectionState.isWebSocketConnected ? "Connected" : "Disconnected",
+      // Check WS connection status -- if we're disconnected twice in a row, treat
+      // the deployment as disconnected.
+      const nextConnectionState = client.connectionState();
+      const isLocalDeployment = deploymentName.startsWith("local-");
+      const result = await handleConnectionStateChange(
+        nextConnectionState,
+        lastObservedConnectionState,
       );
-    }, 2500);
+      setLastObservedConnectionState({
+        state: nextConnectionState,
+        time: new Date(),
+      });
+      switch (result) {
+        case "Disconnected":
+          // If this is first time transitioning to disconnected, log to sentry that we've disconnected
+          if (isDisconnected !== true) {
+            if (!isLocalDeployment) {
+              addBreadcrumb({
+                message: `Cloud deployment disconnected: ${deploymentName}`,
+                data: {
+                  hasEverConnected: nextConnectionState.hasEverConnected,
+                  connectionCount: nextConnectionState.connectionCount,
+                  connectionRetries: nextConnectionState.connectionRetries,
+                },
+              });
+              // Log to sentry including the instance name when we seem to be unable to connect to a cloud deployment
+              captureMessage(`Cloud deployment is disconnected`);
+            }
+          }
+          setIsDisconnected(true);
+          break;
+        case "LocalDeploymentMismatch":
+          setLastObservedConnectionState("LocalDeploymentMismatch");
+          break;
+        case "Unknown":
+          setIsDisconnected(null);
+          break;
+        case "Connected":
+          // If transitioning from disconnected to connected, log to sentry that we've reconnected
+          if (isDisconnected === true) {
+            if (!isLocalDeployment) {
+              addBreadcrumb({
+                message: `Cloud deployment reconnected: ${deploymentName}`,
+              });
+              // Log to sentry including the instance name when we seem to be unable to connect to a cloud deployment
+              captureMessage(`Cloud deployment has reconnected`);
+            }
+          }
+          setIsDisconnected(false);
+          break;
+        default: {
+          const _exhaustiveCheck: never = result;
+          throw new Error(`Unknown connection state: ${result}`);
+        }
+      }
+    }, CONNECTION_STATE_CHECK_INTERVAL_MS);
     return () => clearInterval(checkConnection);
-  });
-  useEffect(() => {
-    if (isDisconnected && !deploymentName.startsWith("local-")) {
-      // Log to sentry including the instance name when we seem to be unable to connect to a cloud deployment
-      captureMessage(`Cloud deployment is disconnected: ${deploymentName}`);
-    }
-  }, [isDisconnected, deploymentName, captureMessage]);
+  }, [
+    lastObservedConnectionState,
+    deploymentName,
+    deploymentUrl,
+    client,
+    addBreadcrumb,
+    captureMessage,
+    handleConnectionStateChange,
+    isDisconnected,
+  ]);
   const value = useMemo(
     () => ({
       deployment,
-      isDisconnected,
+      isDisconnected: isDisconnected === true,
     }),
     [deployment, isDisconnected],
   );

npm-packages/dashboard-common/src/lib/mockDeploymentInfo.tsx

@@ -5,6 +5,7 @@ export const mockDeploymentInfo: DeploymentInfo = {
   reportHttpError: () => {},
   captureException: () => {},
   captureMessage: () => {},
+  addBreadcrumb: () => {},
   deploymentUrl: process.env.NEXT_PUBLIC_DEPLOYMENT_URL!,
   adminKey: process.env.NEXT_PUBLIC_ADMIN_KEY!,
   useCurrentTeam: () => ({

npm-packages/dashboard-self-hosted/src/pages/_app.tsx

@@ -99,6 +99,7 @@ export default App;
 
 const deploymentInfo: Omit<DeploymentInfo, "deploymentUrl" | "adminKey"> = {
   ok: true,
+  addBreadcrumb: console.error,
   captureMessage: console.error,
   captureException: console.error,
   reportHttpError: (

npm-packages/dashboard/src/providers/DeploymentInfoProvider.tsx

@@ -3,6 +3,7 @@ import { useRouter } from "next/router";
 import {
   captureException,
   captureMessage,
+  addBreadcrumb,
   ErrorBoundary,
 } from "@sentry/nextjs";
 import { reportHttpError } from "hooks/fetching";
@@ -59,6 +60,7 @@ export function DeploymentInfoProvider({
       );
       setDeploymentInfo({
         ...info,
+        addBreadcrumb,
         captureMessage,
         captureException,
         reportHttpError,