Feature/snapshot

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt

@@ -16,14 +16,20 @@
  */
 package io.github.thibaultbee.streampack.core.elements.processing.video
 
+import android.graphics.Bitmap
 import android.graphics.SurfaceTexture
 import android.util.Size
 import android.view.Surface
+import androidx.annotation.IntRange
 import androidx.concurrent.futures.CallbackToFutureAdapter
 import com.google.common.util.concurrent.ListenableFuture
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
+import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils
+import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preRotate
+import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preVerticalFlip
 import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile
+import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate
 import io.github.thibaultbee.streampack.core.logger.Logger
 import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_GL
 import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider
@@ -37,6 +43,8 @@ private class DefaultSurfaceProcessor(
 ) : ISurfaceProcessorInternal, SurfaceTexture.OnFrameAvailableListener {
     private val renderer = OpenGlRenderer()
 
+    private val glHandler = glThread.handler
+
     private val isReleaseRequested = AtomicBoolean(false)
     private var isReleased = false
 
@@ -47,7 +55,7 @@ private class DefaultSurfaceProcessor(
     private val surfaceInputs: MutableList<SurfaceInput> = mutableListOf()
     private val surfaceInputsTimestampInNsMap: MutableMap<SurfaceTexture, Long> = hashMapOf()
 
-    private val glHandler = glThread.handler
+    private val pendingSnapshots = mutableListOf<PendingSnapshot>()
 
     init {
         val future = submitSafely {
@@ -110,7 +118,7 @@ private class DefaultSurfaceProcessor(
 
         executeSafely {
             if (!surfaceOutputs.contains(surfaceOutput)) {
-                renderer.registerOutputSurface(surfaceOutput.descriptor.surface)
+                renderer.registerOutputSurface(surfaceOutput.surface)
                 surfaceOutputs.add(surfaceOutput)
             } else {
                 Logger.w(TAG, "Surface already added")
@@ -120,7 +128,7 @@ private class DefaultSurfaceProcessor(
 
     private fun removeOutputSurfaceInternal(surfaceOutput: ISurfaceOutput) {
         if (surfaceOutputs.contains(surfaceOutput)) {
-            renderer.unregisterOutputSurface(surfaceOutput.descriptor.surface)
+            renderer.unregisterOutputSurface(surfaceOutput.surface)
             surfaceOutputs.remove(surfaceOutput)
         } else {
             Logger.w(TAG, "Surface not found")
@@ -144,7 +152,7 @@ private class DefaultSurfaceProcessor(
 
         executeSafely {
             val surfaceOutput =
-                surfaceOutputs.firstOrNull { it.descriptor.surface == surface }
+                surfaceOutputs.firstOrNull { it.surface == surface }
             if (surfaceOutput != null) {
                 removeOutputSurfaceInternal(surfaceOutput)
             } else {
@@ -155,7 +163,7 @@ private class DefaultSurfaceProcessor(
 
     private fun removeAllOutputSurfacesInternal() {
         surfaceOutputs.forEach { surfaceOutput ->
-            renderer.unregisterOutputSurface(surfaceOutput.descriptor.surface)
+            renderer.unregisterOutputSurface(surfaceOutput.surface)
         }
         surfaceOutputs.clear()
     }
@@ -193,6 +201,19 @@ private class DefaultSurfaceProcessor(
         }
     }
 
+    override fun snapshot(
+        @IntRange(from = 0, to = 359) rotationDegrees: Int
+    ): ListenableFuture<Bitmap> {
+        if (isReleaseRequested.get()) {
+            throw IllegalStateException("SurfaceProcessor is released")
+        }
+        return CallbackToFutureAdapter.getFuture { completer ->
+            executeSafely {
+                pendingSnapshots.add(PendingSnapshot(rotationDegrees, completer))
+            }
+        }
+    }
+
     // Executed on GL thread
     override fun onFrameAvailable(surfaceTexture: SurfaceTexture) {
         if (isReleaseRequested.get()) {
@@ -204,19 +225,96 @@ private class DefaultSurfaceProcessor(
 
         val timestamp =
             surfaceTexture.timestamp + (surfaceInputsTimestampInNsMap[surfaceTexture] ?: 0L)
-        surfaceOutputs.filter { it.isStreaming() }.forEach {
+        surfaceOutputs.filterIsInstance<SurfaceOutput>().forEach {
             try {
                 it.updateTransformMatrix(surfaceOutputMatrix, textureMatrix)
-                renderer.render(
-                    timestamp,
-                    surfaceOutputMatrix,
-                    it.descriptor.surface,
-                    it.viewportRect
-                )
+                if (it.isStreaming()) {
+                    renderer.render(
+                        timestamp,
+                        surfaceOutputMatrix,
+                        it.surface,
+                        it.viewportRect
+                    )
+                }
             } catch (t: Throwable) {
                 Logger.e(TAG, "Error while rendering frame", t)
             }
         }
+
+        // Surface, size and transform matrix for JPEG Surface if exists
+        if (pendingSnapshots.isNotEmpty()) {
+            try {
+                val bitmapSurface =
+                    surfaceOutputs.maxByOrNull { it.resolution.width * it.resolution.height }
+                        ?: throw IllegalStateException(
+                            "No output surface available for snapshot"
+                        )
+
+                // Execute all pending snapshots.
+                takeSnapshot(bitmapSurface.resolution, surfaceOutputMatrix.clone())
+            } catch (e: RuntimeException) {
+                // Propagates error back to the app if failed to take snapshot.
+                failAllPendingSnapshots(e)
+            }
+        }
+    }
+
+    /**
+     * Takes a snapshot of the current frame and draws it to given JPEG surface.
+     *
+     * @param snapshotSize The size of the snapshot.
+     * @param snapshotTransform The GL transform matrix to apply to the snapshot.
+     */
+    private fun takeSnapshot(snapshotSize: Size, snapshotTransform: FloatArray) {
+        if (pendingSnapshots.isEmpty()) {
+            // No pending snapshot requests, do nothing.
+            return
+        }
+
+        // Write to Bitmap, once for each snapshot request.
+        try {
+            for (pendingSnapshot in pendingSnapshots) {
+                try {
+                    // Take a snapshot of the current frame.
+                    val bitmap =
+                        getBitmap(snapshotSize, snapshotTransform, pendingSnapshot.rotationDegrees)
+
+                    // Complete the snapshot request.
+                    pendingSnapshot.completer.set(bitmap)
+                } catch (t: Throwable) {
+                    // Propagate error back to the app if failed to take snapshot.
+                    pendingSnapshot.completer.setException(t)
+                }
+            }
+        } finally {
+            pendingSnapshots.clear()
+        }
+    }
+
+    private fun failAllPendingSnapshots(throwable: Throwable) {
+        for (pendingSnapshot in pendingSnapshots) {
+            pendingSnapshot.completer.setException(throwable)
+        }
+    }
+
+    private fun getBitmap(
+        size: Size,
+        textureTransform: FloatArray,
+        rotationDegrees: Int
+    ): Bitmap {
+        val snapshotTransform = textureTransform.clone()
+
+        // Rotate the output if requested.
+        snapshotTransform.preRotate(rotationDegrees.toFloat(), 0.5f, 0.5f)
+
+        // Flip the snapshot. This is for reverting the GL transform added in SurfaceOutputImpl.
+        snapshotTransform.preVerticalFlip(0.5f)
+
+        // Update the size based on the rotation degrees.
+        val rotatedSize = size.rotate(rotationDegrees)
+
+        // Take a snapshot Bitmap and compress it to JPEG.
+        return renderer.snapshot(rotatedSize, snapshotTransform)
     }
 
     private fun executeSafely(
@@ -258,6 +356,12 @@ private class DefaultSurfaceProcessor(
     }
 
     private data class SurfaceInput(val surface: Surface, val surfaceTexture: SurfaceTexture)
+
+    private data class PendingSnapshot(
+        @IntRange(from = 0, to = 359)
+        val rotationDegrees: Int,
+        val completer: CallbackToFutureAdapter.Completer<Bitmap>
+    )
 }
 
 class DefaultSurfaceProcessorFactory :

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/ISurfaceProcessor.kt

@@ -15,8 +15,11 @@
  */
 package io.github.thibaultbee.streampack.core.elements.processing.video
 
+import android.graphics.Bitmap
 import android.util.Size
 import android.view.Surface
+import androidx.annotation.IntRange
+import com.google.common.util.concurrent.ListenableFuture
 import io.github.thibaultbee.streampack.core.elements.interfaces.Releasable
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile
@@ -45,6 +48,8 @@ interface ISurfaceProcessorInternal : ISurfaceProcessor, Releasable {
 
     fun removeAllOutputSurfaces()
 
+    fun snapshot(@IntRange(from = 0, to = 359) rotationDegrees: Int): ListenableFuture<Bitmap>
+
     /**
      * Factory interface for creating instances of [ISurfaceProcessorInternal].
      */
@@ -54,4 +59,4 @@ interface ISurfaceProcessorInternal : ISurfaceProcessor, Releasable {
             dispatcherProvider: IVideoDispatcherProvider
         ): ISurfaceProcessorInternal
     }
-}
+}

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/ISurfaceOutput.kt

@@ -1,12 +1,17 @@
 package io.github.thibaultbee.streampack.core.elements.processing.video.outputs
 
-import android.graphics.Rect
-import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor
+import android.util.Size
+import android.view.Surface
 
 interface ISurfaceOutput {
-    val descriptor: SurfaceDescriptor
-    val isStreaming: () -> Boolean
-    val viewportRect: Rect
+    val surface: Surface
+    val resolution: Size
+    val type: OutputType
 
     fun updateTransformMatrix(output: FloatArray, input: FloatArray)
+
+    enum class OutputType {
+        INTERNAL,
+        BITMAP
+    }
 }

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/SurfaceOutput.kt

@@ -18,7 +18,10 @@ package io.github.thibaultbee.streampack.core.elements.processing.video.outputs
 import android.graphics.Rect
 import android.graphics.RectF
 import android.opengl.Matrix
+import android.util.Size
+import android.view.Surface
 import androidx.annotation.IntRange
+import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput.TransformationInfo
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ViewPortUtils.calculateViewportRect
 import io.github.thibaultbee.streampack.core.elements.processing.video.source.ISourceInfoProvider
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.TransformUtils
@@ -29,14 +32,27 @@ import io.github.thibaultbee.streampack.core.elements.utils.RotationValue
 import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate
 import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor
 
+fun SurfaceOutput(
+    descriptor: SurfaceDescriptor,
+    isStreaming: () -> Boolean,
+    transformationInfo: TransformationInfo
+) =
+    SurfaceOutput(
+        descriptor.surface,
+        descriptor.resolution,
+        isStreaming,
+        transformationInfo
+    )
+
 class SurfaceOutput(
-    override val descriptor: SurfaceDescriptor,
-    override val isStreaming: () -> Boolean,
+    override val surface: Surface,
+    override val resolution: Size,
+    val isStreaming: () -> Boolean,
     private val transformationInfo: TransformationInfo
 ) :
     ISurfaceOutput {
-    private val resolution = descriptor.resolution
-    
+    override val type = ISurfaceOutput.OutputType.INTERNAL
+
     private val infoProvider: ISourceInfoProvider
         get() = transformationInfo.infoProvider
 
@@ -52,7 +68,7 @@ class SurfaceOutput(
     private val additionalTransform = FloatArray(16)
     private val invertedTextureTransform = FloatArray(16)
 
-    override val viewportRect = calculateViewportRect(
+    val viewportRect = calculateViewportRect(
         transformationInfo.aspectRatioMode,
         transformationInfo.infoProvider.getSurfaceSize(resolution),
         resolution