feat(core): startStream to take a snapshot if needed

Author: ThibaultBee

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt

@@ -24,6 +24,7 @@ import androidx.annotation.IntRange
 import androidx.concurrent.futures.CallbackToFutureAdapter
 import com.google.common.util.concurrent.ListenableFuture
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
+import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preRotate
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preVerticalFlip
@@ -33,7 +34,6 @@ import io.github.thibaultbee.streampack.core.logger.Logger
 import io.github.thibaultbee.streampack.core.pipelines.DispatcherProvider.Companion.THREAD_NAME_GL
 import io.github.thibaultbee.streampack.core.pipelines.IVideoDispatcherProvider
 import io.github.thibaultbee.streampack.core.pipelines.utils.HandlerThreadExecutor
-import java.io.IOException
 import java.util.concurrent.atomic.AtomicBoolean
 
 
@@ -118,7 +118,7 @@ private class DefaultSurfaceProcessor(
 
         executeSafely {
             if (!surfaceOutputs.contains(surfaceOutput)) {
-                renderer.registerOutputSurface(surfaceOutput.descriptor.surface)
+                renderer.registerOutputSurface(surfaceOutput.surface)
                 surfaceOutputs.add(surfaceOutput)
             } else {
                 Logger.w(TAG, "Surface already added")
@@ -128,7 +128,7 @@ private class DefaultSurfaceProcessor(
 
     private fun removeOutputSurfaceInternal(surfaceOutput: ISurfaceOutput) {
         if (surfaceOutputs.contains(surfaceOutput)) {
-            renderer.unregisterOutputSurface(surfaceOutput.descriptor.surface)
+            renderer.unregisterOutputSurface(surfaceOutput.surface)
             surfaceOutputs.remove(surfaceOutput)
         } else {
             Logger.w(TAG, "Surface not found")
@@ -152,7 +152,7 @@ private class DefaultSurfaceProcessor(
 
         executeSafely {
             val surfaceOutput =
-                surfaceOutputs.firstOrNull { it.descriptor.surface == surface }
+                surfaceOutputs.firstOrNull { it.surface == surface }
             if (surfaceOutput != null) {
                 removeOutputSurfaceInternal(surfaceOutput)
             } else {
@@ -163,7 +163,7 @@ private class DefaultSurfaceProcessor(
 
     private fun removeAllOutputSurfacesInternal() {
         surfaceOutputs.forEach { surfaceOutput ->
-            renderer.unregisterOutputSurface(surfaceOutput.descriptor.surface)
+            renderer.unregisterOutputSurface(surfaceOutput.surface)
         }
         surfaceOutputs.clear()
     }
@@ -225,15 +225,17 @@ private class DefaultSurfaceProcessor(
 
         val timestamp =
             surfaceTexture.timestamp + (surfaceInputsTimestampInNsMap[surfaceTexture] ?: 0L)
-        surfaceOutputs.filter { it.isStreaming() }.forEach {
+        surfaceOutputs.filterIsInstance<SurfaceOutput>().forEach {
             try {
                 it.updateTransformMatrix(surfaceOutputMatrix, textureMatrix)
-                renderer.render(
-                    timestamp,
-                    surfaceOutputMatrix,
-                    it.descriptor.surface,
-                    it.viewportRect
-                )
+                if (it.isStreaming()) {
+                    renderer.render(
+                        timestamp,
+                        surfaceOutputMatrix,
+                        it.surface,
+                        it.viewportRect
+                    )
+                }
             } catch (t: Throwable) {
                 Logger.e(TAG, "Error while rendering frame", t)
             }
@@ -242,14 +244,14 @@ private class DefaultSurfaceProcessor(
         // Surface, size and transform matrix for JPEG Surface if exists
         if (pendingSnapshots.isNotEmpty()) {
             try {
-                val first = surfaceOutputs.first()
-                val snapshotOutput = Pair(
-                    first.descriptor.resolution,
-                    surfaceOutputMatrix.clone()
-                )
+                val bitmapSurface =
+                    surfaceOutputs.maxByOrNull { it.resolution.width * it.resolution.height }
+                        ?: throw IllegalStateException(
+                            "No output surface available for snapshot"
+                        )
 
                 // Execute all pending snapshots.
-                takeSnapshot(snapshotOutput)
+                takeSnapshot(bitmapSurface.resolution, surfaceOutputMatrix.clone())
             } catch (e: RuntimeException) {
                 // Propagates error back to the app if failed to take snapshot.
                 failAllPendingSnapshots(e)
@@ -260,36 +262,39 @@ private class DefaultSurfaceProcessor(
     /**
      * Takes a snapshot of the current frame and draws it to given JPEG surface.
      *
-     * @param snapshotOutput The <Surface size, transform matrix> pair for drawing.
+     * @param snapshotSize The size of the snapshot.
+     * @param snapshotTransform The GL transform matrix to apply to the snapshot.
      */
-    private fun takeSnapshot(snapshotOutput: Pair<Size, FloatArray>) {
+    private fun takeSnapshot(snapshotSize: Size, snapshotTransform: FloatArray) {
         if (pendingSnapshots.isEmpty()) {
             // No pending snapshot requests, do nothing.
             return
         }
 
-        // Write to JPEG surface, once for each snapshot request.
+        // Write to Bitmap, once for each snapshot request.
         try {
             for (pendingSnapshot in pendingSnapshots) {
-                val (size, transform) = snapshotOutput
-
-                // Take a snapshot of the current frame.
-                val bitmap = getBitmap(size, transform, pendingSnapshot.rotationDegrees)
-
-                // Complete the snapshot request.
-                pendingSnapshot.completer.set(bitmap)
+                try {
+                    // Take a snapshot of the current frame.
+                    val bitmap =
+                        getBitmap(snapshotSize, snapshotTransform, pendingSnapshot.rotationDegrees)
+
+                    // Complete the snapshot request.
+                    pendingSnapshot.completer.set(bitmap)
+                } catch (t: Throwable) {
+                    // Propagate error back to the app if failed to take snapshot.
+                    pendingSnapshot.completer.setException(t)
+                }
             }
+        } finally {
             pendingSnapshots.clear()
-        } catch (e: IOException) {
-            failAllPendingSnapshots(e)
         }
     }
 
     private fun failAllPendingSnapshots(throwable: Throwable) {
         for (pendingSnapshot in pendingSnapshots) {
             pendingSnapshot.completer.setException(throwable)
         }
-        pendingSnapshots.clear()
     }
 
     private fun getBitmap(

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/ISurfaceOutput.kt

@@ -1,12 +1,17 @@
 package io.github.thibaultbee.streampack.core.elements.processing.video.outputs
 
-import android.graphics.Rect
-import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor
+import android.util.Size
+import android.view.Surface
 
 interface ISurfaceOutput {
-    val descriptor: SurfaceDescriptor
-    val isStreaming: () -> Boolean
-    val viewportRect: Rect
+    val surface: Surface
+    val resolution: Size
+    val type: OutputType
 
     fun updateTransformMatrix(output: FloatArray, input: FloatArray)
+
+    enum class OutputType {
+        INTERNAL,
+        BITMAP
+    }
 }

core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/IdentitySurfaceOutput.kt

@@ -18,7 +18,10 @@ package io.github.thibaultbee.streampack.core.elements.processing.video.outputs
 import android.graphics.Rect
 import android.graphics.RectF
 import android.opengl.Matrix
+import android.util.Size
+import android.view.Surface
 import androidx.annotation.IntRange
+import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput.TransformationInfo
 import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ViewPortUtils.calculateViewportRect
 import io.github.thibaultbee.streampack.core.elements.processing.video.source.ISourceInfoProvider
 import io.github.thibaultbee.streampack.core.elements.processing.video.utils.TransformUtils
@@ -29,14 +32,27 @@ import io.github.thibaultbee.streampack.core.elements.utils.RotationValue
 import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate
 import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor
 
+fun SurfaceOutput(
+    descriptor: SurfaceDescriptor,
+    isStreaming: () -> Boolean,
+    transformationInfo: TransformationInfo
+) =
+    SurfaceOutput(
+        descriptor.surface,
+        descriptor.resolution,
+        isStreaming,
+        transformationInfo
+    )
+
 class SurfaceOutput(
-    override val descriptor: SurfaceDescriptor,
-    override val isStreaming: () -> Boolean,
+    override val surface: Surface,
+    override val resolution: Size,
+    val isStreaming: () -> Boolean,
     private val transformationInfo: TransformationInfo
 ) :
     ISurfaceOutput {
-    private val resolution = descriptor.resolution
-    
+    override val type = ISurfaceOutput.OutputType.INTERNAL
+
     private val infoProvider: ISourceInfoProvider
         get() = transformationInfo.infoProvider
 
@@ -52,7 +68,7 @@ class SurfaceOutput(
     private val additionalTransform = FloatArray(16)
     private val invertedTextureTransform = FloatArray(16)
 
-    override val viewportRect = calculateViewportRect(
+    val viewportRect = calculateViewportRect(
         transformationInfo.aspectRatioMode,
         transformationInfo.infoProvider.getSurfaceSize(resolution),
         resolution