diff --git a/packages/react-sdk/rollup.config.mjs b/packages/react-sdk/rollup.config.mjs index 0dbf04a5bc..6d2998f134 100644 --- a/packages/react-sdk/rollup.config.mjs +++ b/packages/react-sdk/rollup.config.mjs @@ -11,6 +11,9 @@ const chunkFileNames = (chunkInfo) => { if (chunkInfo.name.includes('CallStatsLatencyChart')) { return 'latency-chart-[hash].[format].js'; } + if (chunkInfo.name.includes('BackgroundFilters')) { + return 'background-filters-[hash].[format].js'; + } return '[name]-[hash].[format].js'; }; diff --git a/packages/react-sdk/src/components/BackgroundFilters/BackgroundFilters.tsx b/packages/react-sdk/src/components/BackgroundFilters/BackgroundFilters.tsx index 05e0a3c602..1124f8c915 100644 --- a/packages/react-sdk/src/components/BackgroundFilters/BackgroundFilters.tsx +++ b/packages/react-sdk/src/components/BackgroundFilters/BackgroundFilters.tsx @@ -1,8 +1,7 @@ import { - createContext, + Context, PropsWithChildren, useCallback, - useContext, useEffect, useMemo, useRef, @@ -13,19 +12,23 @@ import { useCall, useCallStateHooks } from '@stream-io/video-react-bindings'; import { Call, disposeOfMediaStream } from '@stream-io/video-client'; import { BackgroundBlurLevel, - BackgroundFilter, createRenderer, - isPlatformSupported, isMediaPipePlatformSupported, - loadTFLite, + isPlatformSupported, loadMediaPipe, - PlatformSupportFlags, - VirtualBackground, + loadTFLite, + PerformanceStats, Renderer, TFLite, - PerformanceStats, + VirtualBackground, } from '@stream-io/video-filters-web'; import clsx from 'clsx'; +import type { + BackgroundFiltersPerformance, + BackgroundFiltersProps, + BackgroundFiltersContextValue, + PerformanceDegradationReason, +} from './types'; /** * Constants for FPS warning calculation. @@ -39,32 +42,6 @@ const DEFAULT_FPS = 30; const DEVIATION_LIMIT = 0.5; const OUTLIER_PERSISTENCE = 5; -/** - * Configuration for performance metric thresholds. - */ -export type BackgroundFiltersPerformanceThresholds = { - /** - * The lower FPS threshold for triggering a performance warning. - * When the EMA FPS falls below this value, a warning is shown. - * @default 23 - */ - fpsWarningThresholdLower?: number; - - /** - * The upper FPS threshold for clearing a performance warning. - * When the EMA FPS rises above this value, the warning is cleared. - * @default 25 - */ - fpsWarningThresholdUpper?: number; - - /** - * The default FPS value used as the initial value for the EMA (Exponential Moving Average) - * calculation and when stats are unavailable or when resetting the filter. - * @default 30 - */ - defaultFps?: number; -}; - /** * Represents the available background filter processing engines. */ @@ -74,160 +51,6 @@ enum FilterEngine { NONE, } -/** - * Represents the possible reasons for background filter performance degradation. - */ -export enum PerformanceDegradationReason { - FRAME_DROP = 'frame-drop', - CPU_THROTTLING = 'cpu-throttling', -} - -export type BackgroundFiltersProps = PlatformSupportFlags & { - /** - * A list of URLs to use as background images. - */ - backgroundImages?: string[]; - - /** - * The background filter to apply to the video (by default). - * @default undefined no filter applied - */ - backgroundFilter?: BackgroundFilter; - - /** - * The URL of the image to use as the background (by default). - */ - backgroundImage?: string; - - /** - * The level of blur to apply to the background (by default). - * @default 'high'. - */ - backgroundBlurLevel?: BackgroundBlurLevel; - - /** - * The base path for the TensorFlow Lite files. - * @default 'https://unpkg.com/@stream-io/video-filters-web/mediapipe'. - */ - basePath?: string; - - /** - * The path to the TensorFlow Lite WebAssembly file. - * - * Override this prop to use a custom path to the TensorFlow Lite WebAssembly file - * (e.g., if you choose to host it yourself). - */ - tfFilePath?: string; - - /** - * The path to the MediaPipe model file. - * Override this prop to use a custom path to the MediaPipe model file - * (e.g., if you choose to host it yourself). - */ - modelFilePath?: string; - - /** - * When true, the filter uses the legacy TensorFlow-based segmentation model. - * When false, it uses the default MediaPipe Tasks Vision model. - * - * Only enable this if you need to mimic the behavior of older SDK versions. - */ - useLegacyFilter?: boolean; - - /** - * When a started filter encounters an error, this callback will be executed. - * The default behavior (not overridable) is unregistering a failed filter. - * Use this callback to display UI error message, disable the corresponding stream, - * or to try registering the filter again. - */ - onError?: (error: any) => void; - - /** - * Configuration for performance metric thresholds. - * Use this to customize when performance warnings are triggered. - */ - performanceThresholds?: BackgroundFiltersPerformanceThresholds; -}; - -/** - * Performance degradation information for background filters. - * - * Performance is calculated using an Exponential Moving Average (EMA) of FPS values - * to smooth out quick spikes and provide stable performance warnings. - */ -export type BackgroundFiltersPerformance = { - /** - * Whether performance is currently degraded. - */ - degraded: boolean; - /** - * Reasons for performance degradation. - */ - reason?: Array; -}; - -export type BackgroundFiltersAPI = { - /** - * Whether the current platform supports the background filters. - */ - isSupported: boolean; - - /** - * Indicates whether the background filters engine is loaded and ready. - */ - isReady: boolean; - - /** - * Performance information for background filters. - */ - performance: BackgroundFiltersPerformance; - - /** - * Disables all background filters applied to the video. - */ - disableBackgroundFilter: () => void; - - /** - * Applies a background blur filter to the video. - * - * @param blurLevel the level of blur to apply to the background. - */ - applyBackgroundBlurFilter: (blurLevel: BackgroundBlurLevel) => void; - - /** - * Applies a background image filter to the video. - * - * @param imageUrl the URL of the image to use as the background. - */ - applyBackgroundImageFilter: (imageUrl: string) => void; -}; - -/** - * The context value for the background filters context. - */ -export type BackgroundFiltersContextValue = BackgroundFiltersProps & - BackgroundFiltersAPI; - -/** - * The context for the background filters. - */ -const BackgroundFiltersContext = createContext< - BackgroundFiltersContextValue | undefined ->(undefined); - -/** - * A hook to access the background filters context API. - */ -export const useBackgroundFilters = () => { - const context = useContext(BackgroundFiltersContext); - if (!context) { - throw new Error( - 'useBackgroundFilters must be used within a BackgroundFiltersProvider', - ); - } - return context; -}; - /** * Determines which filter engine is available. * MEDIA_PIPE is the default unless legacy filters are requested or MediaPipe is unsupported. @@ -239,12 +62,11 @@ const determineEngine = async ( forceSafariSupport: boolean | undefined, forceMobileSupport: boolean | undefined, ): Promise => { - const isTfPlatformSupported = await isPlatformSupported({ - forceSafariSupport, - forceMobileSupport, - }); - if (useLegacyFilter) { + const isTfPlatformSupported = await isPlatformSupported({ + forceSafariSupport, + forceMobileSupport, + }); return isTfPlatformSupported ? FilterEngine.TF : FilterEngine.NONE; } @@ -263,9 +85,15 @@ const determineEngine = async ( * in your project before using this component. */ export const BackgroundFiltersProvider = ( - props: PropsWithChildren, + props: PropsWithChildren & { + // for code splitting. Prevents circular dependency issues where + // this Context needs to be present in the main chunk, but also + // imported by the background filters chunk. + ContextProvider: Context; + }, ) => { const { + ContextProvider, children, backgroundImages = [], backgroundFilter: bgFilterFromProps = undefined, @@ -340,7 +168,7 @@ export const BackgroundFiltersProvider = ( const reasons: Array = []; if (showLowFpsWarning) { - reasons.push(PerformanceDegradationReason.FRAME_DROP); + reasons.push('frame-drop'); } const qualityLimitationReasons = @@ -351,7 +179,7 @@ export const BackgroundFiltersProvider = ( qualityLimitationReasons && qualityLimitationReasons?.includes('cpu') ) { - reasons.push(PerformanceDegradationReason.CPU_THROTTLING); + reasons.push('cpu-throttling'); } return { @@ -458,52 +286,54 @@ export const BackgroundFiltersProvider = ( ); const isReady = useLegacyFilter ? !!tfLite : !!mediaPipe; + const contextValue: BackgroundFiltersContextValue = { + isSupported, + performance, + isReady, + backgroundImage, + backgroundBlurLevel, + backgroundFilter, + disableBackgroundFilter, + applyBackgroundBlurFilter, + applyBackgroundImageFilter, + backgroundImages, + tfFilePath, + modelFilePath, + basePath, + onError: handleError, + }; return ( - + {children} {isReady && ( )} - + ); }; const BackgroundFilters = (props: { + api: BackgroundFiltersContextValue; tfLite?: TFLite; engine: FilterEngine; onStats: (stats: PerformanceStats) => void; }) => { const call = useCall(); - const { children, start } = useRenderer(props.tfLite, call, props.engine); - const { onError, backgroundFilter } = useBackgroundFilters(); + const { engine, api, tfLite, onStats } = props; + const { children, start } = useRenderer(api, tfLite, call, engine); + const { onError, backgroundFilter } = api; const handleErrorRef = useRef<((error: any) => void) | undefined>(undefined); handleErrorRef.current = onError; const handleStatsRef = useRef< ((stats: PerformanceStats) => void) | undefined >(undefined); - handleStatsRef.current = props.onStats; + handleStatsRef.current = onStats; useEffect(() => { if (!call || !backgroundFilter) return; @@ -524,6 +354,7 @@ const BackgroundFilters = (props: { }; const useRenderer = ( + api: BackgroundFiltersContextValue, tfLite: TFLite | undefined, call: Call | undefined, engine: FilterEngine, @@ -534,7 +365,7 @@ const useRenderer = ( backgroundImage, modelFilePath, basePath, - } = useBackgroundFilters(); + } = api; const videoRef = useRef(null); const canvasRef = useRef(null); diff --git a/packages/react-sdk/src/components/BackgroundFilters/BackgroundFiltersProvider.tsx b/packages/react-sdk/src/components/BackgroundFilters/BackgroundFiltersProvider.tsx new file mode 100644 index 0000000000..d208c8c65f --- /dev/null +++ b/packages/react-sdk/src/components/BackgroundFilters/BackgroundFiltersProvider.tsx @@ -0,0 +1,60 @@ +import { + createContext, + lazy, + PropsWithChildren, + ReactNode, + Suspense, + useContext, +} from 'react'; +import type { + BackgroundFiltersProps, + BackgroundFiltersContextValue, +} from './types'; + +const BackgroundFiltersProviderImpl = lazy(() => + import('./BackgroundFilters').then((m) => ({ + default: m.BackgroundFiltersProvider, + })), +); + +/** + * The context for the background filters. + */ +const BackgroundFiltersContext = createContext< + BackgroundFiltersContextValue | undefined +>(undefined); + +/** + * A hook to access the background filters context API. + */ +export const useBackgroundFilters = () => { + const context = useContext(BackgroundFiltersContext); + if (!context) { + throw new Error( + 'useBackgroundFilters must be used within a BackgroundFiltersProvider', + ); + } + return context; +}; + +/** + * A provider component that enables the use of background filters in your app. + * + * Please make sure you have the `@stream-io/video-filters-web` package installed + * in your project before using this component. + */ +export const BackgroundFiltersProvider = ( + props: PropsWithChildren & { + SuspenseFallback?: ReactNode; + }, +) => { + const { SuspenseFallback = null, ...filterProps } = props; + return ( + + + + ); +}; diff --git a/packages/react-sdk/src/components/BackgroundFilters/index.ts b/packages/react-sdk/src/components/BackgroundFilters/index.ts index c8492dd550..35e7acb46a 100644 --- a/packages/react-sdk/src/components/BackgroundFilters/index.ts +++ b/packages/react-sdk/src/components/BackgroundFilters/index.ts @@ -1 +1,4 @@ -export * from './BackgroundFilters'; +// don't export BackgroundFilters.tsx as it is lazily loaded through +// the BackgroundFiltersProvider +export * from './BackgroundFiltersProvider'; +export * from './types'; diff --git a/packages/react-sdk/src/components/BackgroundFilters/types.ts b/packages/react-sdk/src/components/BackgroundFilters/types.ts new file mode 100644 index 0000000000..3ffee12979 --- /dev/null +++ b/packages/react-sdk/src/components/BackgroundFilters/types.ts @@ -0,0 +1,162 @@ +import type { + BackgroundBlurLevel, + BackgroundFilter, + PlatformSupportFlags, +} from '@stream-io/video-filters-web'; + +/** + * Configuration for performance metric thresholds. + */ +export type BackgroundFiltersPerformanceThresholds = { + /** + * The lower FPS threshold for triggering a performance warning. + * When the EMA FPS falls below this value, a warning is shown. + * @default 23 + */ + fpsWarningThresholdLower?: number; + + /** + * The upper FPS threshold for clearing a performance warning. + * When the EMA FPS rises above this value, the warning is cleared. + * @default 25 + */ + fpsWarningThresholdUpper?: number; + + /** + * The default FPS value used as the initial value for the EMA (Exponential Moving Average) + * calculation and when stats are unavailable or when resetting the filter. + * @default 30 + */ + defaultFps?: number; +}; + +export type BackgroundFiltersProps = PlatformSupportFlags & { + /** + * A list of URLs to use as background images. + */ + backgroundImages?: string[]; + + /** + * The background filter to apply to the video (by default). + * @default undefined no filter applied + */ + backgroundFilter?: BackgroundFilter; + + /** + * The URL of the image to use as the background (by default). + */ + backgroundImage?: string; + + /** + * The level of blur to apply to the background (by default). + * @default 'high'. + */ + backgroundBlurLevel?: BackgroundBlurLevel; + + /** + * The base path for the TensorFlow Lite files. + * @default 'https://unpkg.com/@stream-io/video-filters-web/mediapipe'. + */ + basePath?: string; + + /** + * The path to the TensorFlow Lite WebAssembly file. + * + * Override this prop to use a custom path to the TensorFlow Lite WebAssembly file + * (e.g., if you choose to host it yourself). + */ + tfFilePath?: string; + + /** + * The path to the MediaPipe model file. + * Override this prop to use a custom path to the MediaPipe model file + * (e.g., if you choose to host it yourself). + */ + modelFilePath?: string; + + /** + * When true, the filter uses the legacy TensorFlow-based segmentation model. + * When false, it uses the default MediaPipe Tasks Vision model. + * + * Only enable this if you need to mimic the behavior of older SDK versions. + */ + useLegacyFilter?: boolean; + + /** + * When a started filter encounters an error, this callback will be executed. + * The default behavior (not overridable) is unregistering a failed filter. + * Use this callback to display UI error message, disable the corresponding stream, + * or to try registering the filter again. + */ + onError?: (error: any) => void; + + /** + * Configuration for performance metric thresholds. + * Use this to customize when performance warnings are triggered. + */ + performanceThresholds?: BackgroundFiltersPerformanceThresholds; +}; + +/** + * Represents the possible reasons for background filter performance degradation. + */ +export type PerformanceDegradationReason = 'frame-drop' | 'cpu-throttling'; + +/** + * Performance degradation information for background filters. + * + * Performance is calculated using an Exponential Moving Average (EMA) of FPS values + * to smooth out quick spikes and provide stable performance warnings. + */ +export type BackgroundFiltersPerformance = { + /** + * Whether performance is currently degraded. + */ + degraded: boolean; + /** + * Reasons for performance degradation. + */ + reason?: Array; +}; + +export type BackgroundFiltersAPI = { + /** + * Whether the current platform supports the background filters. + */ + isSupported: boolean; + + /** + * Indicates whether the background filters engine is loaded and ready. + */ + isReady: boolean; + + /** + * Performance information for background filters. + */ + performance: BackgroundFiltersPerformance; + + /** + * Disables all background filters applied to the video. + */ + disableBackgroundFilter: () => void; + + /** + * Applies a background blur filter to the video. + * + * @param blurLevel the level of blur to apply to the background. + */ + applyBackgroundBlurFilter: (blurLevel: BackgroundBlurLevel) => void; + + /** + * Applies a background image filter to the video. + * + * @param imageUrl the URL of the image to use as the background. + */ + applyBackgroundImageFilter: (imageUrl: string) => void; +}; + +/** + * The context value for the background filters context. + */ +export type BackgroundFiltersContextValue = BackgroundFiltersProps & + BackgroundFiltersAPI; diff --git a/sample-apps/react/react-dogfood/components/DegradedPerformanceNotification.tsx b/sample-apps/react/react-dogfood/components/DegradedPerformanceNotification.tsx index 63e5d88c16..16bdef9911 100644 --- a/sample-apps/react/react-dogfood/components/DegradedPerformanceNotification.tsx +++ b/sample-apps/react/react-dogfood/components/DegradedPerformanceNotification.tsx @@ -2,7 +2,6 @@ import { PropsWithChildren, useMemo } from 'react'; import { Placement } from '@floating-ui/react'; import { Notification, - PerformanceDegradationReason, useBackgroundFilters, useI18n, } from '@stream-io/video-react-sdk'; @@ -29,12 +28,8 @@ export const DegradedPerformanceNotification = ({ } const reasons = performance?.reason || []; - const hasFrameDrop = reasons.includes( - PerformanceDegradationReason.FRAME_DROP, - ); - const hasCpuThrottling = reasons.includes( - PerformanceDegradationReason.CPU_THROTTLING, - ); + const hasFrameDrop = reasons.includes('frame-drop'); + const hasCpuThrottling = reasons.includes('cpu-throttling'); if (hasFrameDrop && hasCpuThrottling) { return t(