feat(core): add an API to take a snapshot

Author: ThibaultBee

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt

@@ -1,17 +1,39 @@
+/*
+ * Copyright 2022 The Android Open Source Project
+ * Copyright 2025 Thibault B.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 package io.github.thibaultbee.streampack.core.elements.processing.video
 
+import android.graphics.Bitmap
 import android.graphics.SurfaceTexture
 import android.util.Size
 import android.view.Surface
+import androidx.annotation.IntRange
 import androidx.concurrent.futures.CallbackToFutureAdapter
 import com.google.common.util.concurrent.ListenableFuture
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils
+import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preRotate
+import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preVerticalFlip
 import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile
+import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate
 import io.github.thibaultbee.streampack.core.logger.Logger
 import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_GL
 import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider
 import io.github.thibaultbee.streampack.core.pipelines.utils.HandlerThreadExecutor
+import java.io.IOException
 import java.util.concurrent.atomic.AtomicBoolean
 
 
@@ -21,6 +43,8 @@ private class DefaultSurfaceProcessor(
 ) : ISurfaceProcessorInternal, SurfaceTexture.OnFrameAvailableListener {
     private val renderer = OpenGlRenderer()
 
+    private val glHandler = glThread.handler
+
     private val isReleaseRequested = AtomicBoolean(false)
     private var isReleased = false
 
@@ -31,7 +55,7 @@ private class DefaultSurfaceProcessor(
     private val surfaceInputs: MutableList<SurfaceInput> = mutableListOf()
     private val surfaceInputsTimestampInNsMap: MutableMap<SurfaceTexture, Long> = hashMapOf()
 
-    private val glHandler = glThread.handler
+    private val pendingSnapshots = mutableListOf<PendingSnapshot>()
 
     init {
         val future = submitSafely {
@@ -177,6 +201,19 @@ private class DefaultSurfaceProcessor(
         }
     }
 
+    override fun snapshot(
+        @IntRange(from = 0, to = 359) rotationDegrees: Int
+    ): ListenableFuture<Bitmap> {
+        if (isReleaseRequested.get()) {
+            throw IllegalStateException("SurfaceProcessor is released")
+        }
+        return CallbackToFutureAdapter.getFuture { completer ->
+            executeSafely {
+                pendingSnapshots.add(PendingSnapshot(rotationDegrees, completer))
+            }
+        }
+    }
+
     // Executed on GL thread
     override fun onFrameAvailable(surfaceTexture: SurfaceTexture) {
         if (isReleaseRequested.get()) {
@@ -201,6 +238,78 @@ private class DefaultSurfaceProcessor(
                 Logger.e(TAG, "Error while rendering frame", t)
             }
         }
+
+        // Surface, size and transform matrix for JPEG Surface if exists
+        if (pendingSnapshots.isNotEmpty()) {
+            try {
+                val first = surfaceOutputs.first()
+                val snapshotOutput = Pair(
+                    first.descriptor.resolution,
+                    surfaceOutputMatrix.clone()
+                )
+
+                // Execute all pending snapshots.
+                takeSnapshot(snapshotOutput)
+            } catch (e: RuntimeException) {
+                // Propagates error back to the app if failed to take snapshot.
+                failAllPendingSnapshots(e)
+            }
+        }
+    }
+
+    /**
+     * Takes a snapshot of the current frame and draws it to given JPEG surface.
+     *
+     * @param snapshotOutput The <Surface size, transform matrix> pair for drawing.
+     */
+    private fun takeSnapshot(snapshotOutput: Pair<Size, FloatArray>) {
+        if (pendingSnapshots.isEmpty()) {
+            // No pending snapshot requests, do nothing.
+            return
+        }
+
+        // Write to JPEG surface, once for each snapshot request.
+        try {
+            for (pendingSnapshot in pendingSnapshots) {
+                val (size, transform) = snapshotOutput
+
+                // Take a snapshot of the current frame.
+                val bitmap = getBitmap(size, transform, pendingSnapshot.rotationDegrees)
+
+                // Complete the snapshot request.
+                pendingSnapshot.completer.set(bitmap)
+            }
+            pendingSnapshots.clear()
+        } catch (e: IOException) {
+            failAllPendingSnapshots(e)
+        }
+    }
+
+    private fun failAllPendingSnapshots(throwable: Throwable) {
+        for (pendingSnapshot in pendingSnapshots) {
+            pendingSnapshot.completer.setException(throwable)
+        }
+        pendingSnapshots.clear()
+    }
+
+    private fun getBitmap(
+        size: Size,
+        textureTransform: FloatArray,
+        rotationDegrees: Int
+    ): Bitmap {
+        val snapshotTransform = textureTransform.clone()
+
+        // Rotate the output if requested.
+        snapshotTransform.preRotate(rotationDegrees.toFloat(), 0.5f, 0.5f)
+
+        // Flip the snapshot. This is for reverting the GL transform added in SurfaceOutputImpl.
+        snapshotTransform.preVerticalFlip(0.5f)
+
+        // Update the size based on the rotation degrees.
+        val rotatedSize = size.rotate(rotationDegrees)
+
+        // Take a snapshot Bitmap and compress it to JPEG.
+        return renderer.snapshot(rotatedSize, snapshotTransform)
     }
 
     private fun executeSafely(
@@ -242,6 +351,12 @@ private class DefaultSurfaceProcessor(
     }
 
     private data class SurfaceInput(val surface: Surface, val surfaceTexture: SurfaceTexture)
+
+    private data class PendingSnapshot(
+        @IntRange(from = 0, to = 359)
+        val rotationDegrees: Int,
+        val completer: CallbackToFutureAdapter.Completer<Bitmap>
+    )
 }
 
 class DefaultSurfaceProcessorFactory :

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt

@@ -15,8 +15,11 @@
  */
 package io.github.thibaultbee.streampack.core.elements.processing.video
 
+import android.graphics.Bitmap
 import android.util.Size
 import android.view.Surface
+import androidx.annotation.IntRange
+import com.google.common.util.concurrent.ListenableFuture
 import io.github.thibaultbee.streampack.core.elements.interfaces.Releasable
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile
@@ -45,6 +48,8 @@ interface ISurfaceProcessorInternal : ISurfaceProcessor, Releasable {
 
     fun removeAllOutputSurfaces()
 
+    fun snapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int): ListenableFuture<Bitmap>
+
     /**
      * Factory interface for creating instances of [ISurfaceProcessorInternal].
      */
@@ -54,4 +59,4 @@ interface ISurfaceProcessorInternal : ISurfaceProcessor, Releasable {
             dispatcherProvider: IVideoDispatcherProvider
         ): ISurfaceProcessorInternal
     }
-}
+}

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/OpenGlRenderer.kt

@@ -16,6 +16,7 @@
  */
 package io.github.thibaultbee.streampack.core.elements.processing.video
 
+import android.graphics.Bitmap
 import android.graphics.Rect
 import android.opengl.EGL14
 import android.opengl.EGLConfig
@@ -29,10 +30,13 @@ import android.util.Log
 import android.util.Size
 import android.view.Surface
 import androidx.annotation.WorkerThread
+import androidx.core.graphics.createBitmap
 import androidx.core.util.Pair
+import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.EMPTY_ATTRIBS
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.InputFormat
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.NO_OUTPUT_SURFACE
+import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.PIXEL_STRIDE
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.Program2D
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.SamplerShaderProgram
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils.checkEglErrorOrLog
@@ -58,6 +62,7 @@ import java.nio.ByteBuffer
 import java.util.concurrent.atomic.AtomicBoolean
 import javax.microedition.khronos.egl.EGL10
 
+
 /**
  * OpenGLRenderer renders texture image to the output surface.
  *
@@ -310,6 +315,29 @@ class OpenGlRenderer {
         }
     }
 
+    /**
+     * Takes a snapshot of the current external texture and returns a Bitmap.
+     *
+     * @param size             the size of the output [Bitmap].
+     * @param textureTransform the transformation matrix.
+     * See: [SurfaceOutput.updateTransformMatrix]
+     */
+    fun snapshot(size: Size, textureTransform: FloatArray): Bitmap {
+        // Allocate buffer.
+        val byteBuffer = ByteBuffer.allocateDirect(
+            size.width * size.height * PIXEL_STRIDE
+        )
+
+        // Take a snapshot.
+        snapshot(byteBuffer, size, textureTransform)
+        byteBuffer.rewind()
+
+        // Create a Bitmap and copy the bytes over.
+        val bitmap = createBitmap(size.width, size.height, Bitmap.Config.ARGB_8888)
+        bitmap.copyPixelsFromBuffer(byteBuffer)
+        return bitmap
+    }
+
     /**
      * Takes a snapshot of the current external texture and stores it in the given byte buffer.
      *

core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/VideoInput.kt

@@ -16,7 +16,9 @@
 package io.github.thibaultbee.streampack.core.pipelines.inputs
 
 import android.content.Context
+import android.graphics.Bitmap
 import android.view.Surface
+import androidx.annotation.IntRange
 import io.github.thibaultbee.streampack.core.elements.processing.video.ISurfaceProcessorInternal
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.processing.video.source.ISourceInfoProvider
@@ -40,7 +42,12 @@ import kotlinx.coroutines.launch
 import kotlinx.coroutines.sync.Mutex
 import kotlinx.coroutines.sync.withLock
 import kotlinx.coroutines.withContext
+import java.io.File
+import java.io.FileOutputStream
+import java.io.OutputStream
 import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.coroutines.resume
+import kotlin.coroutines.suspendCoroutine
 
 /**
  * The public interface for the video input.
@@ -75,6 +82,64 @@ interface IVideoInput {
      * The video processor for adding effects to the video frames.
      */
     val processor: ISurfaceProcessorInternal
+
+    /**
+     * Takes a snapshot of the current video frame.
+     *
+     * The snapshot is returned as a [Bitmap].
+     *
+     * @param rotationDegrees The rotation to apply to the snapshot, in degrees. 0 means no rotation.
+     * @return The snapshot as a [Bitmap].
+     */
+    suspend fun takeSnapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int = 0): Bitmap
+}
+
+/**
+ * Takes a JPEG snapshot of the current video frame.
+ *
+ * The snapshot is saved to the specified file.
+ *
+ * @param filePathString The path of the file to save the snapshot to.
+ * @param quality The quality of the JPEG, from 0 to 100.
+ * @param rotationDegrees The rotation to apply to the snapshot, in degrees.
+ */
+suspend fun IVideoInput.takeJpegSnapshot(
+    filePathString: String,
+    @IntRange(from = 0, to = 100) quality: Int = 100,
+    @IntRange(from = 0, to = 359) rotationDegrees: Int = 0
+) = takeJpegSnapshot(FileOutputStream(filePathString), quality, rotationDegrees)
+
+
+/**
+ * Takes a JPEG snapshot of the current video frame.
+ *
+ * The snapshot is saved to the specified file.
+ *
+ * @param file The file to save the snapshot to.
+ * @param quality The quality of the JPEG, from 0 to 100.
+ * @param rotationDegrees The rotation to apply to the snapshot, in degrees.
+ */
+suspend fun IVideoInput.takeJpegSnapshot(
+    file: File,
+    @IntRange(from = 0, to = 100) quality: Int = 100,
+    @IntRange(from = 0, to = 359) rotationDegrees: Int = 0
+) = takeJpegSnapshot(FileOutputStream(file), quality, rotationDegrees)
+
+/**
+ * Takes a snapshot of the current video frame.
+ *
+ * The snapshot is saved as a JPEG to the specified output stream.
+ * @param outputStream The output stream to save the snapshot to.
+ * @param quality The quality of the JPEG, from 0 to 100.
+ * @param rotationDegrees The rotation to apply to the snapshot, in degrees.
+ */
+suspend fun IVideoInput.takeJpegSnapshot(
+    outputStream: OutputStream,
+    @IntRange(from = 0, to = 100) quality: Int = 100,
+    @IntRange(from = 0, to = 359) rotationDegrees: Int = 0
+) {
+    val bitmap = takeSnapshot(rotationDegrees)
+    bitmap.compress(Bitmap.CompressFormat.JPEG, quality, outputStream)
 }
 
 /**
@@ -349,7 +414,25 @@ internal class VideoInput(
         return newSurfaceProcessor
     }
 
-    suspend fun addOutputSurface(output: ISurfaceOutput) {
+    override suspend fun takeSnapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int): Bitmap {
+        if (isReleaseRequested.get()) {
+            throw IllegalStateException("Input is released")
+        }
+        return withContext(dispatcherProvider.default) {
+            suspendCoroutine { continuation ->
+                val listener = processor.snapshot(rotationDegrees)
+                try {
+                    val bitmap = listener.get()
+                    continuation.resume(bitmap)
+                } catch (e: Exception) {
+                    continuation.resumeWith(Result.failure(e))
+                }
+            }
+        }
+    }
+
+
+    internal suspend fun addOutputSurface(output: ISurfaceOutput) {
         if (isReleaseRequested.get()) {
             throw IllegalStateException("Input is released")
         }
@@ -360,7 +443,7 @@ internal class VideoInput(
         }
     }
 
-    suspend fun removeOutputSurface(output: Surface) {
+    internal suspend fun removeOutputSurface(output: Surface) {
         outputMutex.withLock {
             surfaceOutput.firstOrNull { it.descriptor.surface == output }?.let {
                 surfaceOutput.remove(it)