Implement Reconnection Logic in ApiServerContext

- Added state management for connection issues, including reconnect progress and attempts, to enhance the robustness of the API server connection.
- Introduced functions to handle reconnect scheduling, countdowns, and teardown of socket connections, improving error handling during disconnections.
- Updated the connectToServer function to manage socket connections more effectively, ensuring a smoother user experience during connectivity issues.
- Enhanced the disconnect event handling to provide better feedback and state management when the API server is disconnected.

src/components/Dashboard/context/ApiServerContext.jsx

@@ -8,7 +8,7 @@ import {
   useCallback
 } from 'react'
 import io from 'socket.io-client'
-import { message, Modal, Space, Button, Typography } from 'antd'
+import { message, Modal, Space, Button, Typography, Flex } from 'antd'
 import PropTypes from 'prop-types'
 import { AuthContext } from './AuthContext'
 import { useLocation, useNavigate } from 'react-router-dom'
@@ -20,12 +20,14 @@ import LockIcon from '../../Icons/LockIcon'
 import config from '../../../config'
 import loglevel from 'loglevel'
 import { getModelByName } from '../../../database/ObjectModels'
+import ProgressDisplay from '../common/ProgressDisplay'
 const logger = loglevel.getLogger('ApiServerContext')
 logger.setLevel(config.logLevel)
 
 const { Text } = Typography
 
 const SPOTLIGHT_CACHE_TTL_MS = 10_000
+const RECONNECT_DELAYS_MS = [3000, 5000, 10000, 15000]
 
 const spotlightCache = new Map()
 const runningSpotlightFetches = new Map()
@@ -134,6 +136,22 @@ const ApiServerProvider = ({ children }) => {
   const subscribedActivityServerSubscriptionsRef = useRef(new Set())
   const notificationListenersRef = useRef(new Set())
   const completedLaunchSessionsRef = useRef(new Set())
+  const [connectionIssue, setConnectionIssue] = useState(false)
+  const [reconnectProgress, setReconnectProgress] = useState(0)
+  const [reconnectRemainingMs, setReconnectRemainingMs] = useState(
+    RECONNECT_DELAYS_MS[0]
+  )
+  const [isReconnecting, setIsReconnecting] = useState(false)
+  const reconnectAttemptRef = useRef(0)
+  const reconnectTimerRef = useRef(null)
+  const reconnectProgressTimerRef = useRef(null)
+  const reconnectScheduledRef = useRef(false)
+  const isReconnectingRef = useRef(false)
+  const hasConnectedOnceRef = useRef(false)
+  const connectToServerRef = useRef(null)
+  const scheduleReconnectRef = useRef(null)
+  const attemptReconnectRef = useRef(null)
+  const teardownSocketRef = useRef(null)
 
   const handleActivityUpdate = useCallback(async (data) => {
     logger.debug('Notifying activity update:', data)
@@ -299,15 +317,138 @@ const ApiServerProvider = ({ children }) => {
     [getUserSettings, messageApi]
   )
 
+  const clearReconnectTimers = useCallback(() => {
+    if (reconnectTimerRef.current) {
+      clearTimeout(reconnectTimerRef.current)
+      reconnectTimerRef.current = null
+    }
+    if (reconnectProgressTimerRef.current) {
+      clearInterval(reconnectProgressTimerRef.current)
+      reconnectProgressTimerRef.current = null
+    }
+  }, [])
+
+  const resetReconnectState = useCallback(() => {
+    clearReconnectTimers()
+    reconnectAttemptRef.current = 0
+    reconnectScheduledRef.current = false
+    isReconnectingRef.current = false
+    setIsReconnecting(false)
+    setReconnectProgress(0)
+    setReconnectRemainingMs(RECONNECT_DELAYS_MS[0])
+  }, [clearReconnectTimers])
+
+  const startReconnectCountdown = useCallback(
+    (delayMs) => {
+      clearReconnectTimers()
+      const startedAt = Date.now()
+      setReconnectRemainingMs(delayMs)
+      setReconnectProgress(0)
+      setIsReconnecting(false)
+
+      reconnectProgressTimerRef.current = setInterval(() => {
+        const elapsed = Date.now() - startedAt
+        const remaining = Math.max(0, delayMs - elapsed)
+        setReconnectRemainingMs(remaining)
+        setReconnectProgress(Math.min(100, (elapsed / delayMs) * 100))
+      }, 100)
+
+      reconnectTimerRef.current = setTimeout(() => {
+        attemptReconnectRef.current?.()
+      }, delayMs)
+    },
+    [clearReconnectTimers]
+  )
+
+  const markConnectionLost = useCallback(() => {
+    isReconnectingRef.current = false
+    setIsReconnecting(false)
+    setConnecting(false)
+    setConnected(false)
+    setConnectionIssue(true)
+    setUserSettings(createEmptyUserSettings())
+    setUserSettingsLoaded(false)
+    clearSubscriptions()
+    scheduleReconnectRef.current?.()
+  }, [clearSubscriptions])
+
+  const scheduleReconnect = useCallback(() => {
+    if (!token || authenticated != true) {
+      return
+    }
+    if (isReconnectingRef.current || reconnectScheduledRef.current) {
+      return
+    }
+
+    reconnectScheduledRef.current = true
+    setConnectionIssue(true)
+    const delayIndex = Math.min(
+      reconnectAttemptRef.current,
+      RECONNECT_DELAYS_MS.length - 1
+    )
+    startReconnectCountdown(RECONNECT_DELAYS_MS[delayIndex])
+  }, [token, authenticated, startReconnectCountdown])
+
+  const attemptReconnect = useCallback(() => {
+    if (!token || authenticated != true) {
+      return
+    }
+    if (isReconnectingRef.current) {
+      return
+    }
+
+    clearReconnectTimers()
+    reconnectScheduledRef.current = false
+    isReconnectingRef.current = true
+    reconnectAttemptRef.current += 1
+    setIsReconnecting(true)
+    setReconnectProgress(100)
+    setConnecting(true)
+    setConnectionIssue(true)
+
+    if (socketRef.current) {
+      if (!socketRef.current.connected) {
+        socketRef.current.connect()
+      }
+    } else {
+      connectToServerRef.current?.()
+    }
+  }, [token, authenticated, clearReconnectTimers])
+
+  const teardownSocket = useCallback(() => {
+    resetReconnectState()
+    setConnectionIssue(false)
+    setConnecting(false)
+    const socket = socketRef.current
+    socketRef.current = null
+    socket?.disconnect()
+  }, [resetReconnectState])
+
+  scheduleReconnectRef.current = scheduleReconnect
+  attemptReconnectRef.current = attemptReconnect
+  teardownSocketRef.current = teardownSocket
+
   const connectToServer = useCallback(() => {
     if (token && authenticated == true) {
+      if (socketRef.current) {
+        if (!socketRef.current.connected) {
+          socketRef.current.connect()
+        }
+        return
+      }
+
       logger.debug('Token is available, connecting to api server...')
 
       const newSocket = io(config.apiServerUrl, {
-        reconnectionAttempts: 3,
+        reconnection: false,
+        reconnectionAttempts: 0,
+        forceNew: true,
         timeout: 3000,
         auth: { type: 'user' }
       })
+      if (typeof newSocket.io?.reconnection === 'function') {
+        newSocket.io.reconnection(false)
+      }
 
       setConnecting(true)
 
@@ -317,6 +458,13 @@ const ApiServerProvider = ({ children }) => {
           if (result?.valid !== true) {
             setConnecting(false)
             setError('Api Server authentication failed')
+            if (hasConnectedOnceRef.current) {
+              markConnectionLost()
+              newSocket.disconnect()
+              return
+            }
+            resetReconnectState()
+            setConnectionIssue(false)
             newSocket.disconnect()
             return
           }
@@ -327,9 +475,12 @@ const ApiServerProvider = ({ children }) => {
           if (!newSocket.connected) {
             return
           }
+          hasConnectedOnceRef.current = true
           setConnecting(false)
           setConnected(true)
           setError(null)
+          setConnectionIssue(false)
+          resetReconnectState()
           resubscribeActivityListeners(newSocket)
         })
       })
@@ -359,23 +510,29 @@ const ApiServerProvider = ({ children }) => {
         })
       })
 
-      newSocket.on('disconnect', () => {
-        logger.debug('Api Server disconnected')
+      newSocket.on('disconnect', (reason) => {
+        if (socketRef.current !== newSocket) {
+          return
+        }
+
+        logger.debug('Api Server disconnected', reason)
         setError('Api Server disconnected')
-        clearSubscriptions()
-        setConnected(false)
-        setUserSettings(createEmptyUserSettings())
-        setUserSettingsLoaded(false)
+
+        if (reason === 'io client disconnect') {
+          return
+        }
+
+        markConnectionLost()
       })
 
       newSocket.on('connect_error', (err) => {
+        if (socketRef.current !== newSocket) {
+          return
+        }
+
         logger.error('Api Server connection error:', err)
-        messageApi.error('Api Server connection error: ' + err.message)
         setError('Api Server connection error')
-        clearSubscriptions()
-        setConnected(false)
-        setUserSettings(createEmptyUserSettings())
-        setUserSettingsLoaded(false)
+        markConnectionLost()
       })
 
       newSocket.on('error', (err) => {
@@ -388,33 +545,42 @@ const ApiServerProvider = ({ children }) => {
   }, [
     token,
     authenticated,
-    messageApi,
     handleActivityUpdate,
     clearSubscriptions,
     getUserSettings,
-    resubscribeActivityListeners
+    resubscribeActivityListeners,
+    resetReconnectState,
+    markConnectionLost
   ])
 
+  connectToServerRef.current = connectToServer
+
   useEffect(() => {
     if (token && authenticated == true) {
-      connectToServer()
-    } else if (!token && socketRef.current) {
-      logger.debug('Token not available, disconnecting api server...')
-      socketRef.current.disconnect()
-      socketRef.current = null
-      setUserSettings(createEmptyUserSettings())
-      setUserSettingsLoaded(false)
+      connectToServerRef.current?.()
+      return
     }
 
-    // Clean up function
+    logger.debug('Token not available, disconnecting api server...')
+    teardownSocketRef.current?.()
+    setUserSettings(createEmptyUserSettings())
+    setUserSettingsLoaded(false)
+  }, [token, authenticated])
+
+  useEffect(() => {
     return () => {
-      if (socketRef.current) {
-        logger.debug('Cleaning up api server connection...')
-        socketRef.current.disconnect()
-        socketRef.current = null
-      }
+      teardownSocketRef.current?.()
     }
-  }, [token, authenticated, connectToServer])
+  }, [])
+
+  useEffect(() => {
+    if (!token || authenticated != true || !connectionIssue) {
+      return
+    }
+    if (!reconnectScheduledRef.current && !isReconnectingRef.current) {
+      scheduleReconnect()
+    }
+  }, [token, authenticated, connectionIssue, scheduleReconnect])
 
   useEffect(() => {
     const handlePageVisible = () => {
@@ -429,7 +595,7 @@ const ApiServerProvider = ({ children }) => {
 
       if (!socket.connected) {
         logger.debug('Page visible with disconnected socket, reconnecting...')
-        socket.connect()
+        attemptReconnect()
         return
       }
 
@@ -448,10 +614,7 @@ const ApiServerProvider = ({ children }) => {
       logger.debug('Page restored from bfcache, reconnecting socket...')
       subscribedActivityServerSubscriptionsRef.current.clear()
       setConnected(false)
-      if (socketRef.current) {
-        socketRef.current.disconnect()
-        socketRef.current = null
-      }
+      teardownSocket()
       connectToServer()
     }
 
@@ -462,7 +625,14 @@ const ApiServerProvider = ({ children }) => {
       document.removeEventListener('visibilitychange', handlePageVisible)
       window.removeEventListener('pageshow', handlePageShow)
     }
-  }, [token, authenticated, connectToServer, resubscribeActivityListeners])
+  }, [
+    token,
+    authenticated,
+    connectToServer,
+    resubscribeActivityListeners,
+    attemptReconnect,
+    teardownSocket
+  ])
 
   const setObjectActivity = (id, type, mode) => {
     logger.debug('Setting activity for', id, mode)
@@ -1951,9 +2121,15 @@ const ApiServerProvider = ({ children }) => {
 
       const tryFetchRenderedFile = async () => {
         try {
-          const result = await fetchTemplateDownload(id, content, object, type, {
-            renderRequestId
-          })
+          const result = await fetchTemplateDownload(
+            id,
+            content,
+            object,
+            type,
+            {
+              renderRequestId
+            }
+          )
           finish(result)
           return true
         } catch (err) {
@@ -2460,6 +2636,11 @@ const ApiServerProvider = ({ children }) => {
     }
   }
 
+  const reconnectSecondsRemaining = Math.max(
+    1,
+    Math.ceil(reconnectRemainingMs / 1000)
+  )
+
   // Sanitize a string so it is safe to use as a filename on most file systems
   const formatFileName = (name) => {
     if (!name || typeof name !== 'string') {
@@ -2553,6 +2734,45 @@ const ApiServerProvider = ({ children }) => {
     >
       {contextHolder}
       {children}
+      <Modal
+        title={
+          <Space size={'middle'}>
+            <ExclamationOctagonIcon />
+            Connection Lost
+          </Space>
+        }
+        open={Boolean(token) && authenticated == true && connectionIssue}
+        style={{ maxWidth: 480 }}
+        zIndex={3000}
+        closable={false}
+        centered
+        maskClosable={false}
+        getContainer={() => document.body}
+        footer={[
+          <Button
+            key='reconnect'
+            loading={isReconnecting}
+            onClick={() => attemptReconnect()}
+          >
+            Reconnect
+          </Button>
+        ]}
+      >
+        <Flex vertical gap='middle'>
+          <Text>
+            {isReconnecting
+              ? 'Reconnecting to the API server...'
+              : `Lost connection to the API server. Reconnecting in ${reconnectSecondsRemaining} second${
+                  reconnectSecondsRemaining === 1 ? '' : 's'
+                }...`}
+          </Text>
+          <ProgressDisplay
+            percent={isReconnecting ? 0 : 100 - reconnectProgress}
+            showInfo={false}
+            status={'exception'}
+          />
+        </Flex>
+      </Modal>
       <Modal
         title={
           <Space size={'middle'}>