Event driven session pushing

sdk/highlight-run/src/client/constants/sessions.ts

@@ -6,7 +6,14 @@
  * The amount of time between sending the client-side payload to Highlight backend client.
  * In milliseconds.
  */
-export const SEND_FREQUENCY = 1000 * 2
+export const SEND_FREQUENCY = 1000 * 15
+
+/**
+ * Payload size threshold for triggering sends.
+ * In bytes.
+ */
+// TODO: This is a temporary low value for testing.
+export const UNCOMPRESSED_PAYLOAD_SIZE_THRESHOLD = 1024 * 10 // 10KB
 
 /**
  * Maximum length of a session

sdk/highlight-run/src/client/listeners/page-visibility-listener.tsx

@@ -1,34 +1,8 @@
 export const PageVisibilityListener = (
 	callback: (isTabHidden: boolean) => void,
 ) => {
-	let hidden: string | undefined = undefined
-	let visibilityChangeEventName: string | undefined = undefined
-
-	if (typeof document.hidden !== 'undefined') {
-		// Opera 12.10 and Firefox 18 and later support
-		hidden = 'hidden'
-		visibilityChangeEventName = 'visibilitychange'
-		// @ts-expect-error
-	} else if (typeof document.msHidden !== 'undefined') {
-		hidden = 'msHidden'
-		visibilityChangeEventName = 'msvisibilitychange'
-		// @ts-expect-error
-	} else if (typeof document.webkitHidden !== 'undefined') {
-		hidden = 'webkitHidden'
-		visibilityChangeEventName = 'webkitvisibilitychange'
-	}
-
-	if (visibilityChangeEventName === undefined) {
-		return () => {}
-	}
-	if (hidden === undefined) {
-		return () => {}
-	}
-
-	const hiddenPropertyName = hidden
 	const listener = () => {
-		// @ts-expect-error
-		const tabState = document[hiddenPropertyName]
+		const tabState = document.hidden
 
 		if (tabState) {
 			callback(true)
@@ -37,9 +11,6 @@ export const PageVisibilityListener = (
 		}
 	}
 
-	document.addEventListener(visibilityChangeEventName, listener)
-
-	const eventNameToRemove = visibilityChangeEventName
-
-	return () => document.removeEventListener(eventNameToRemove, listener)
+	document.addEventListener('visibilitychange', listener)
+	return () => document.removeEventListener('visibilitychange', listener)
 }

sdk/highlight-run/src/sdk/record.ts

@@ -30,6 +30,7 @@ import {
 	LAUNCHDARKLY_PATH_PREFIX,
 	LAUNCHDARKLY_URL,
 	MAX_SESSION_LENGTH,
+	UNCOMPRESSED_PAYLOAD_SIZE_THRESHOLD,
 	SEND_FREQUENCY,
 	SNAPSHOT_SETTINGS,
 	VISIBILITY_DEBOUNCE_MS,
@@ -86,6 +87,7 @@ import type { Hook, LDClient } from '../integrations/launchdarkly'
 import { LaunchDarklyIntegration } from '../integrations/launchdarkly'
 import { LDPluginEnvironmentMetadata } from '../plugins/plugin'
 import { RecordOptions } from '../client/types/record'
+import { strToU8 } from 'fflate'
 
 interface HighlightWindow extends Window {
 	Highlight: Highlight
@@ -142,6 +144,8 @@ export class RecordSDK implements Record {
 	hasSessionUnloaded!: boolean
 	hasPushedData!: boolean
 	reloaded!: boolean
+	saving!: boolean
+	_estimatedEventsByteSize!: number
 	_hasPreviouslyInitialized!: boolean
 	_recordStop!: listenerHandler | undefined
 	_integrations: IntegrationClient[] = []
@@ -336,6 +340,8 @@ export class RecordSDK implements Record {
 		this.events = []
 		this.hasSessionUnloaded = false
 		this.hasPushedData = false
+		this.saving = false
+		this._estimatedEventsByteSize = 0
 
 		if (window.Intercom) {
 			window.Intercom('onShow', () => {
@@ -581,6 +587,12 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 					this.logger.log('received isCheckout emit', { event })
 				}
 				this.events.push(event)
+
+				// Check if we should send payload early based on size
+				if (!this.saving && this._checkForImmediateSave(event)) {
+					this.logger.log('Triggering save due to large payload size')
+					this._save()
+				}
 			}
 			emit.bind(this)
 
@@ -704,6 +716,7 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 			this.addCustomEvent('TabHidden', false)
 		} else {
 			this.addCustomEvent('TabHidden', true)
+			this._saveOnUnload()
 			if (this.options.disableBackgroundRecording) {
 				this.stop()
 			}
@@ -911,7 +924,7 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 			}
 			window.addEventListener('pagehide', unloadListener)
 			this.listeners.push(() =>
-				window.removeEventListener('beforeunload', unloadListener),
+				window.removeEventListener('pagehide', unloadListener),
 			)
 		}
 	}
@@ -987,6 +1000,11 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 
 	// Reset the events array and push to a backend.
 	async _save() {
+		if (this.saving) {
+			return
+		}
+		this.saving = true
+
 		try {
 			if (
 				this.state === 'Recording' &&
@@ -1001,6 +1019,9 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 				})
 				await this._reset({})
 			}
+
+			const eventsToSend = this._captureAndResetEventsState()
+
 			let sendFn = undefined
 			if (this.options?.sendMode === 'local') {
 				sendFn = async (payload: any) => {
@@ -1022,7 +1043,7 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 					return 0
 				}
 			}
-			await this._sendPayload({ sendFn })
+			await this._sendPayload({ sendFn, events: eventsToSend })
 			this.hasPushedData = true
 			this.sessionData.lastPushTime = Date.now()
 			setSessionData(this.sessionData)
@@ -1035,9 +1056,12 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 				this.pushPayloadTimerId = undefined
 			}
 			this.pushPayloadTimerId = setTimeout(() => {
+				this.logger.log(`Triggering save due to timeout`)
 				this._save()
 			}, SEND_FREQUENCY)
 		}
+
+		this.saving = false
 	}
 
 	/**
@@ -1066,13 +1090,13 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 
 	async _sendPayload({
 		sendFn,
+		events,
 	}: {
 		sendFn?: (
 			payload: PushSessionEventsMutationVariables,
 		) => Promise<number>
+		events: eventWithTime[]
 	}) {
-		const events = [...this.events]
-
 		// if it is time to take a full snapshot,
 		// ensure the snapshot is at the beginning of the next payload
 		// After snapshot thresholds have been met,
@@ -1108,7 +1132,9 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 				highlight_logs: highlightLogs || undefined,
 			}
 
-			const { compressedBase64 } = await payloadToBase64(sessionPayload)
+			const { compressedBase64, compressedSize } =
+				await payloadToBase64(sessionPayload)
+			this.logger.log(`Compressed payload size: ${compressedSize} bytes`)
 			await sendFn({
 				session_secure_id: this.sessionData.sessionSecureID,
 				payload_id: payloadId.toString(),
@@ -1133,15 +1159,6 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 		}
 		setSessionData(this.sessionData)
 
-		// We are creating a weak copy of the events. rrweb could have pushed more events to this.events while we send the request with the events as a payload.
-		// Originally, we would clear this.events but this could lead to a race condition.
-		// Example Scenario:
-		// 1. Create the events payload from this.events (with N events)
-		// 2. rrweb pushes to this.events (with M events)
-		// 3. Network request made to push payload (Only includes N events)
-		// 4. this.events is cleared (we lose M events)
-		this.events = this.events.slice(events.length)
-
 		clearHighlightLogs(highlightLogs)
 	}
 
@@ -1159,6 +1176,69 @@ SessionSecureID: ${this.sessionData.sessionSecureID}`,
 		this._lastSnapshotTime = new Date().getTime()
 	}
 
+	// Flush data if size threshold is exceeded
+	private _checkForImmediateSave(newEvent: eventWithTime): boolean {
+		if (this.state !== 'Recording' || !this.pushPayloadTimerId) {
+			return false
+		}
+
+		const newEventByteSize = strToU8(JSON.stringify(newEvent)).length
+		this._estimatedEventsByteSize += newEventByteSize
+		return (
+			this._estimatedEventsByteSize >= UNCOMPRESSED_PAYLOAD_SIZE_THRESHOLD
+		)
+	}
+
+	private _saveWithBeacon(
+		backendUrl: string,
+		payload: PushSessionEventsMutationVariables,
+	): Promise<number> {
+		let blob = new Blob(
+			[
+				JSON.stringify({
+					query: print(PushSessionEventsDocument),
+					variables: payload,
+				}),
+			],
+			{
+				type: 'application/json',
+			},
+		)
+		window.fetch(`${backendUrl}`, {
+			method: 'POST',
+			body: blob,
+			keepalive: true,
+		})
+		return Promise.resolve(0)
+	}
+
+	private _saveOnUnload(): void {
+		console.log('saving on unload', this.events.length)
+		if (this.events.length === 0) {
+			return
+		}
+
+		try {
+			const eventsToSend = this._captureAndResetEventsState()
+
+			this._sendPayload({
+				sendFn: (payload) =>
+					this._saveWithBeacon(this._backendUrl, payload),
+				events: eventsToSend,
+			})
+		} catch (error) {
+			this.logger.log('Failed to save session data on unload:', error)
+		}
+	}
+
+	// Atomic capture and reset events state synchronously to avoid race conditions
+	private _captureAndResetEventsState(): eventWithTime[] {
+		const eventsToSend = [...this.events]
+		this.events = []
+		this._estimatedEventsByteSize = 0
+		return eventsToSend
+	}
+
 	register(client: LDClient, metadata: LDPluginEnvironmentMetadata) {
 		this._integrations.push(new LaunchDarklyIntegration(client, metadata))
 	}