fix: videorenderer didnt call update subscriptions on remote reconnect

packages/react-native-sdk/src/components/Participant/ParticipantView/VideoRenderer/TrackSubscriber.tsx

@@ -0,0 +1,125 @@
+import { forwardRef, useImperativeHandle, useEffect, useMemo } from 'react';
+import { LayoutChangeEvent } from 'react-native';
+import {
+  Call,
+  CallingState,
+  hasScreenShare,
+  hasVideo,
+  SfuModels,
+  type VideoTrackType,
+  DebounceType,
+} from '@stream-io/video-client';
+import {
+  BehaviorSubject,
+  combineLatest,
+  distinctUntilChanged,
+  distinctUntilKeyChanged,
+  map,
+  takeWhile,
+} from 'rxjs';
+export type TrackSubscriberHandle = {
+  onLayoutUpdate: (event: LayoutChangeEvent) => void;
+};
+
+type TrackSubscriberProps = {
+  participantSessionId: string;
+  call: Call;
+  trackType: VideoTrackType;
+  isVisible: boolean;
+};
+
+/**
+ * This component is used to subscribe to the video + audio track of the participant in the following cases:
+ * 1. When the participant starts publishing the video track
+ * 2. When the participant changes the video track dimensions
+ * 3. When the participant becomes visible
+ * 4. On joined callingState, this handles reconnection
+
+ * This component is used to unsubscribe to video track and subscribe only to the audio track of the participant (by passing undefined dimensions) in the following cases:
+ * 1. When the participant stops publishing the video track
+ * 2. When the participant becomes invisible
+*/
+const TrackSubscriber = forwardRef<TrackSubscriberHandle, TrackSubscriberProps>(
+  (props, ref) => {
+    const { call, participantSessionId, trackType, isVisible } = props;
+    const dimensions$ = useMemo(() => {
+      return new BehaviorSubject<SfuModels.VideoDimension | undefined>(
+        undefined,
+      );
+    }, []);
+
+    useEffect(() => {
+      const requestTrackWithDimensions = (
+        debounceType: DebounceType,
+        dimension: SfuModels.VideoDimension | undefined,
+      ) => {
+        if (dimension && (dimension.width === 0 || dimension.height === 0)) {
+          // ignore 0x0 dimensions. this can happen when the video element
+          // is not visible (e.g., has display: none).
+          // we treat this as "unsubscription" as we don't want to keep
+          // consuming bandwidth for a video that is not visible on the screen.
+          dimension = undefined;
+        }
+        call.state.updateParticipantTracks(trackType, {
+          [participantSessionId]: { dimension },
+        });
+        call.dynascaleManager.applyTrackSubscriptions(debounceType);
+      };
+      const isPublishingTrack$ = call.state.participants$.pipe(
+        map((ps) => ps.find((p) => p.sessionId === participantSessionId)),
+        takeWhile((p) => !!p),
+        distinctUntilChanged(),
+        distinctUntilKeyChanged('publishedTracks'),
+        map((p) =>
+          trackType === 'videoTrack' ? hasVideo(p) : hasScreenShare(p),
+        ),
+        distinctUntilChanged(),
+      );
+      const isJoinedState$ = call.state.callingState$.pipe(
+        map((callingState) => callingState === CallingState.JOINED),
+      );
+
+      const subscription = combineLatest([
+        dimensions$,
+        isPublishingTrack$,
+        isJoinedState$,
+      ]).subscribe(([dimension, isPublishing, isJoined]) => {
+        if (isJoined && isPublishing && !isVisible) {
+          // the participant is publishing and we are not visible, so we unsubscribe from the video track
+          requestTrackWithDimensions(DebounceType.FAST, undefined);
+        } else if (isJoined && isPublishing && isVisible && dimension) {
+          // the participant is publishing and we are visible and have valid dimensions, so we subscribe to the video track
+          requestTrackWithDimensions(DebounceType.IMMEDIATE, dimension);
+        } else if (isJoined && !isPublishing) {
+          // the participant stopped publishing a track, so we unsubscribe from the video track
+          requestTrackWithDimensions(DebounceType.FAST, undefined);
+        }
+        // if isPublishing but no dimension yet, we wait for dimensions
+      });
+
+      return () => {
+        subscription.unsubscribe();
+      };
+    }, [call, participantSessionId, trackType, isVisible, dimensions$]);
+
+    useImperativeHandle(
+      ref,
+      () => ({
+        onLayoutUpdate: (event) => {
+          const dimension = {
+            width: Math.trunc(event.nativeEvent.layout.width),
+            height: Math.trunc(event.nativeEvent.layout.height),
+          };
+          dimensions$.next(dimension);
+        },
+      }),
+      [dimensions$],
+    );
+
+    return null;
+  },
+);
+
+TrackSubscriber.displayName = 'TrackSubscriber';
+
+export default TrackSubscriber;

packages/react-native-sdk/src/components/Participant/ParticipantView/VideoRenderer.tsx → packages/react-native-sdk/src/components/Participant/ParticipantView/VideoRenderer/index.tsx

@@ -2,21 +2,20 @@ import React, { useEffect, useRef } from 'react';
 import { Platform, StyleSheet, View } from 'react-native';
 import type { MediaStream } from '@stream-io/react-native-webrtc';
 import { RTCView } from '@stream-io/react-native-webrtc';
-import type { ParticipantViewProps } from './ParticipantView';
+import type { ParticipantViewProps } from '../ParticipantView';
 import {
-  CallingState,
   hasPausedTrack,
   hasScreenShare,
   hasVideo,
-  SfuModels,
   type VideoTrackType,
   VisibilityState,
 } from '@stream-io/video-client';
 import { useCall, useCallStateHooks } from '@stream-io/video-react-bindings';
-import { ParticipantVideoFallback as DefaultParticipantVideoFallback } from './ParticipantVideoFallback';
-import { useTheme } from '../../../contexts/ThemeContext';
-import { useTrackDimensions } from '../../../hooks/useTrackDimensions';
-import { useScreenshotIosContext } from '../../../contexts/internal/ScreenshotIosContext';
+import { ParticipantVideoFallback as DefaultParticipantVideoFallback } from '../ParticipantVideoFallback';
+import { useTheme } from '../../../../contexts/ThemeContext';
+import { useTrackDimensions } from '../../../../hooks/useTrackDimensions';
+import { useScreenshotIosContext } from '../../../../contexts/internal/ScreenshotIosContext';
+import TrackSubscriber, { TrackSubscriberHandle } from './TrackSubscriber';
 
 const DEFAULT_VIEWPORT_VISIBILITY_STATE: Record<
   VideoTrackType,
@@ -56,16 +55,9 @@ export const VideoRenderer = ({
     theme: { videoRenderer },
   } = useTheme();
   const call = useCall();
-  const { useCallCallingState, useCameraState, useIncomingVideoSettings } =
-    useCallStateHooks();
+  const { useCameraState, useIncomingVideoSettings } = useCallStateHooks();
+  const trackSubscriberRef = useRef<TrackSubscriberHandle>(null);
   const { isParticipantVideoEnabled } = useIncomingVideoSettings();
-  const callingState = useCallCallingState();
-  const pendingVideoLayoutRef = useRef<SfuModels.VideoDimension | undefined>(
-    undefined,
-  );
-  const subscribedVideoLayoutRef = useRef<SfuModels.VideoDimension | undefined>(
-    undefined,
-  );
   const { direction } = useCameraState();
   const viewRef = useRef(null);
   const {
@@ -91,7 +83,6 @@ export const VideoRenderer = ({
     ? hasScreenShare(participant)
     : hasVideo(participant);
 
-  const hasJoinedCall = callingState === CallingState.JOINED;
   const videoStreamToRender = (isScreenSharing
     ? screenShareStream
     : videoStream) as unknown as MediaStream | undefined;
@@ -179,11 +170,6 @@ export const VideoRenderer = ({
           },
         }));
       }
-      if (subscribedVideoLayoutRef.current) {
-        // when video is enabled again, we want to use the last subscribed dimension to resubscribe
-        pendingVideoLayoutRef.current = subscribedVideoLayoutRef.current;
-        subscribedVideoLayoutRef.current = undefined;
-      }
     }
   }, [
     sessionId,
@@ -194,101 +180,26 @@ export const VideoRenderer = ({
     isLocalParticipant,
   ]);
 
-  useEffect(() => {
-    if (!hasJoinedCall && subscribedVideoLayoutRef.current) {
-      // when call is joined again, we want to use the last subscribed dimension to resubscribe
-      pendingVideoLayoutRef.current = subscribedVideoLayoutRef.current;
-      subscribedVideoLayoutRef.current = undefined;
-    }
-  }, [hasJoinedCall]);
-
-  /**
-   * This effect updates the subscription either
-   * 1. when video tracks are published and was unpublished before
-   * 2. when the view's visibility changes
-   * 3. when call was rejoined
-   */
-  useEffect(() => {
-    if (!call || isLocalParticipant) {
-      return;
-    }
-    // NOTE: We only want to update the subscription if the pendingVideoLayoutRef is set
-    const updateIsNeeded = pendingVideoLayoutRef.current;
-
-    if (!updateIsNeeded || !isPublishingVideoTrack || !hasJoinedCall) {
-      return;
-    }
-
-    // NOTE: When the view is not visible, we want to subscribe to audio only.
-    // We unsubscribe their video by setting the dimension to undefined
-    const dimension = isVisible ? pendingVideoLayoutRef.current : undefined;
-    call.state.updateParticipantTracks(trackType, {
-      [sessionId]: { dimension },
-    });
-    call.dynascaleManager.applyTrackSubscriptions();
-
-    if (dimension) {
-      subscribedVideoLayoutRef.current = pendingVideoLayoutRef.current;
-      pendingVideoLayoutRef.current = undefined;
-    }
-  }, [
-    call,
-    isPublishingVideoTrack,
-    trackType,
-    isVisible,
-    sessionId,
-    hasJoinedCall,
-    isLocalParticipant,
-  ]);
-
-  useEffect(() => {
-    return () => {
-      subscribedVideoLayoutRef.current = undefined;
-      pendingVideoLayoutRef.current = undefined;
-    };
-  }, [trackType, sessionId]);
-
   const onLayout: React.ComponentProps<typeof RTCView>['onLayout'] = (
     event,
   ) => {
-    if (!call || isLocalParticipant) {
-      return;
-    }
-    const dimension = {
-      width: Math.trunc(event.nativeEvent.layout.width),
-      height: Math.trunc(event.nativeEvent.layout.height),
-    };
-
-    // NOTE: If the participant hasn't published a video track yet,
-    // or the view is not viewable, we store the dimensions and handle it
-    // when the track is published or the video is enabled.
-    if (!isPublishingVideoTrack || !isVisible || !hasJoinedCall) {
-      pendingVideoLayoutRef.current = dimension;
-      return;
-    }
-
-    // NOTE: We don't want to update the subscription if the dimension hasn't changed
-    if (
-      subscribedVideoLayoutRef.current?.width === dimension.width &&
-      subscribedVideoLayoutRef.current?.height === dimension.height
-    ) {
-      return;
-    }
-    call.state.updateParticipantTracks(trackType, {
-      [sessionId]: {
-        dimension,
-      },
-    });
-    call.dynascaleManager.applyTrackSubscriptions();
-    subscribedVideoLayoutRef.current = dimension;
-    pendingVideoLayoutRef.current = undefined;
+    trackSubscriberRef.current?.onLayoutUpdate(event);
   };
 
   return (
     <View
       onLayout={onLayout}
       style={[styles.container, videoRenderer.container]}
     >
+      {call && !isLocalParticipant && (
+        <TrackSubscriber
+          ref={trackSubscriberRef}
+          call={call}
+          participantSessionId={sessionId}
+          trackType={trackType}
+          isVisible={isVisible}
+        />
+      )}
       {canShowVideo &&
       videoStreamToRender &&
       (objectFit || isVideoDimensionsValid) ? (