Skip to content

Commit 2ba3614

Browse files
ThibaultBeeThibaultBee
committed
feat(core): startStream to take a snapshot if needed
1 parent e17bb56 commit 2ba3614

File tree

6 files changed

+130
-110
lines changed

6 files changed

+130
-110
lines changed

‎core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/DefaultSurfaceProcessor.kt‎

Lines changed: 36 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -26,13 +26,13 @@ import androidx.annotation.IntRange
2626
import androidx.concurrent.futures.CallbackToFutureAdapter
2727
import com.google.common.util.concurrent.ListenableFuture
2828
import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ISurfaceOutput
29+
import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput
2930
import io.github.thibaultbee.streampack.core.elements.processing.video.utils.GLUtils
3031
import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preRotate
3132
import io.github.thibaultbee.streampack.core.elements.processing.video.utils.extensions.preVerticalFlip
3233
import io.github.thibaultbee.streampack.core.elements.utils.av.video.DynamicRangeProfile
3334
import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate
3435
import io.github.thibaultbee.streampack.core.logger.Logger
35-
import java.io.IOException
3636
import java.util.concurrent.atomic.AtomicBoolean
3737

3838

@@ -119,7 +119,7 @@ private class DefaultSurfaceProcessor(
119119

120120
executeSafely {
121121
if (!surfaceOutputs.contains(surfaceOutput)) {
122-
renderer.registerOutputSurface(surfaceOutput.descriptor.surface)
122+
renderer.registerOutputSurface(surfaceOutput.surface)
123123
surfaceOutputs.add(surfaceOutput)
124124
} else {
125125
Logger.w(TAG, "Surface already added")
@@ -129,7 +129,7 @@ private class DefaultSurfaceProcessor(
129129

130130
private fun removeOutputSurfaceInternal(surfaceOutput: ISurfaceOutput) {
131131
if (surfaceOutputs.contains(surfaceOutput)) {
132-
renderer.unregisterOutputSurface(surfaceOutput.descriptor.surface)
132+
renderer.unregisterOutputSurface(surfaceOutput.surface)
133133
surfaceOutputs.remove(surfaceOutput)
134134
} else {
135135
Logger.w(TAG, "Surface not found")
@@ -153,7 +153,7 @@ private class DefaultSurfaceProcessor(
153153

154154
executeSafely {
155155
val surfaceOutput =
156-
surfaceOutputs.firstOrNull { it.descriptor.surface == surface }
156+
surfaceOutputs.firstOrNull { it.surface == surface }
157157
if (surfaceOutput != null) {
158158
removeOutputSurfaceInternal(surfaceOutput)
159159
} else {
@@ -164,7 +164,7 @@ private class DefaultSurfaceProcessor(
164164

165165
private fun removeAllOutputSurfacesInternal() {
166166
surfaceOutputs.forEach { surfaceOutput ->
167-
renderer.unregisterOutputSurface(surfaceOutput.descriptor.surface)
167+
renderer.unregisterOutputSurface(surfaceOutput.surface)
168168
}
169169
surfaceOutputs.clear()
170170
}
@@ -226,15 +226,17 @@ private class DefaultSurfaceProcessor(
226226

227227
val timestamp =
228228
surfaceTexture.timestamp + (surfaceInputsTimestampInNsMap[surfaceTexture] ?: 0L)
229-
surfaceOutputs.filter { it.isStreaming() }.forEach {
229+
surfaceOutputs.filterIsInstance<SurfaceOutput>().forEach {
230230
try {
231231
it.updateTransformMatrix(surfaceOutputMatrix, textureMatrix)
232-
renderer.render(
233-
timestamp,
234-
surfaceOutputMatrix,
235-
it.descriptor.surface,
236-
it.viewportRect
237-
)
232+
if (it.isStreaming()) {
233+
renderer.render(
234+
timestamp,
235+
surfaceOutputMatrix,
236+
it.surface,
237+
it.viewportRect
238+
)
239+
}
238240
} catch (t: Throwable) {
239241
Logger.e(TAG, "Error while rendering frame", t)
240242
}
@@ -243,14 +245,14 @@ private class DefaultSurfaceProcessor(
243245
// Surface, size and transform matrix for JPEG Surface if exists
244246
if (pendingSnapshots.isNotEmpty()) {
245247
try {
246-
val first = surfaceOutputs.first()
247-
val snapshotOutput = Pair(
248-
first.descriptor.resolution,
249-
surfaceOutputMatrix.clone()
250-
)
248+
val bitmapSurface =
249+
surfaceOutputs.maxByOrNull { it.resolution.width * it.resolution.height }
250+
?: throw IllegalStateException(
251+
"No output surface available for snapshot"
252+
)
251253

252254
// Execute all pending snapshots.
253-
takeSnapshot(snapshotOutput)
255+
takeSnapshot(bitmapSurface.resolution, surfaceOutputMatrix.clone())
254256
} catch (e: RuntimeException) {
255257
// Propagates error back to the app if failed to take snapshot.
256258
failAllPendingSnapshots(e)
@@ -261,36 +263,39 @@ private class DefaultSurfaceProcessor(
261263
/**
262264
* Takes a snapshot of the current frame and draws it to given JPEG surface.
263265
*
264-
* @param snapshotOutput The <Surface size, transform matrix> pair for drawing.
266+
* @param snapshotSize The size of the snapshot.
267+
* @param snapshotTransform The GL transform matrix to apply to the snapshot.
265268
*/
266-
private fun takeSnapshot(snapshotOutput: Pair<Size, FloatArray>) {
269+
private fun takeSnapshot(snapshotSize: Size, snapshotTransform: FloatArray) {
267270
if (pendingSnapshots.isEmpty()) {
268271
// No pending snapshot requests, do nothing.
269272
return
270273
}
271274

272-
// Write to JPEG surface, once for each snapshot request.
275+
// Write to Bitmap, once for each snapshot request.
273276
try {
274277
for (pendingSnapshot in pendingSnapshots) {
275-
val (size, transform) = snapshotOutput
276-
277-
// Take a snapshot of the current frame.
278-
val bitmap = getBitmap(size, transform, pendingSnapshot.rotationDegrees)
279-
280-
// Complete the snapshot request.
281-
pendingSnapshot.completer.set(bitmap)
278+
try {
279+
// Take a snapshot of the current frame.
280+
val bitmap =
281+
getBitmap(snapshotSize, snapshotTransform, pendingSnapshot.rotationDegrees)
282+
283+
// Complete the snapshot request.
284+
pendingSnapshot.completer.set(bitmap)
285+
} catch (t: Throwable) {
286+
// Propagate error back to the app if failed to take snapshot.
287+
pendingSnapshot.completer.setException(t)
288+
}
282289
}
290+
} finally {
283291
pendingSnapshots.clear()
284-
} catch (e: IOException) {
285-
failAllPendingSnapshots(e)
286292
}
287293
}
288294

289295
private fun failAllPendingSnapshots(throwable: Throwable) {
290296
for (pendingSnapshot in pendingSnapshots) {
291297
pendingSnapshot.completer.setException(throwable)
292298
}
293-
pendingSnapshots.clear()
294299
}
295300

296301
private fun getBitmap(

‎core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/ISurfaceOutput.kt‎

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
11
package io.github.thibaultbee.streampack.core.elements.processing.video.outputs
22

3-
import android.graphics.Rect
4-
import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor
3+
import android.util.Size
4+
import android.view.Surface
55

66
interface ISurfaceOutput {
7-
val descriptor: SurfaceDescriptor
8-
val isStreaming: () -> Boolean
9-
val viewportRect: Rect
7+
val surface: Surface
8+
val resolution: Size
9+
val type: OutputType
1010

1111
fun updateTransformMatrix(output: FloatArray, input: FloatArray)
12+
13+
enum class OutputType {
14+
INTERNAL,
15+
BITMAP
16+
}
1217
}

‎core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/IdentitySurfaceOutput.kt‎

Lines changed: 0 additions & 40 deletions
This file was deleted.

‎core/src/main/java/io/github/thibaultbee/streampack/core/elements/processing/video/outputs/SurfaceOutput.kt‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ package io.github.thibaultbee.streampack.core.elements.processing.video.outputs
1818
import android.graphics.Rect
1919
import android.graphics.RectF
2020
import android.opengl.Matrix
21+
import android.util.Size
22+
import android.view.Surface
2123
import androidx.annotation.IntRange
24+
import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.SurfaceOutput.TransformationInfo
2225
import io.github.thibaultbee.streampack.core.elements.processing.video.outputs.ViewPortUtils.calculateViewportRect
2326
import io.github.thibaultbee.streampack.core.elements.processing.video.source.ISourceInfoProvider
2427
import io.github.thibaultbee.streampack.core.elements.processing.video.utils.TransformUtils
@@ -29,14 +32,27 @@ import io.github.thibaultbee.streampack.core.elements.utils.RotationValue
2932
import io.github.thibaultbee.streampack.core.elements.utils.extensions.rotate
3033
import io.github.thibaultbee.streampack.core.pipelines.outputs.SurfaceDescriptor
3134

35+
fun SurfaceOutput(
36+
descriptor: SurfaceDescriptor,
37+
isStreaming: () -> Boolean,
38+
transformationInfo: TransformationInfo
39+
) =
40+
SurfaceOutput(
41+
descriptor.surface,
42+
descriptor.resolution,
43+
isStreaming,
44+
transformationInfo
45+
)
46+
3247
class SurfaceOutput(
33-
override val descriptor: SurfaceDescriptor,
34-
override val isStreaming: () -> Boolean,
48+
override val surface: Surface,
49+
override val resolution: Size,
50+
val isStreaming: () -> Boolean,
3551
private val transformationInfo: TransformationInfo
3652
) :
3753
ISurfaceOutput {
38-
private val resolution = descriptor.resolution
39-
54+
override val type = ISurfaceOutput.OutputType.INTERNAL
55+
4056
private val infoProvider: ISourceInfoProvider
4157
get() = transformationInfo.infoProvider
4258

@@ -52,7 +68,7 @@ class SurfaceOutput(
5268
private val additionalTransform = FloatArray(16)
5369
private val invertedTextureTransform = FloatArray(16)
5470

55-
override val viewportRect = calculateViewportRect(
71+
val viewportRect = calculateViewportRect(
5672
transformationInfo.aspectRatioMode,
5773
transformationInfo.infoProvider.getSurfaceSize(resolution),
5874
resolution

‎core/src/main/java/io/github/thibaultbee/streampack/core/pipelines/inputs/AudioInput.kt‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ internal class AudioInput(
110110

111111
private var isReleaseRequested = AtomicBoolean(false)
112112

113-
private val audioSourceMutex = Mutex()
113+
private val sourceMutex = Mutex()
114114

115115
// SOURCE
116116
private var sourceInternalFlow = MutableStateFlow<IAudioSourceInternal?>(null)
@@ -174,7 +174,7 @@ internal class AudioInput(
174174
}
175175

176176
withContext(coroutineDispatcher) {
177-
audioSourceMutex.withLock {
177+
sourceMutex.withLock {
178178
val previousAudioSource = sourceInternalFlow.value
179179
val isStreaming = previousAudioSource?.isStreamingFlow?.value ?: false
180180

@@ -231,7 +231,7 @@ internal class AudioInput(
231231
}
232232

233233
withContext(coroutineDispatcher) {
234-
audioSourceMutex.withLock {
234+
sourceMutex.withLock {
235235
if (sourceConfig == newAudioSourceConfig) {
236236
Logger.i(TAG, "Audio source configuration is the same, skipping configuration")
237237
return@withContext
@@ -270,7 +270,7 @@ internal class AudioInput(
270270
}
271271

272272
withContext(coroutineDispatcher) {
273-
audioSourceMutex.withLock {
273+
sourceMutex.withLock {
274274
val source = requireNotNull(sourceInternalFlow.value) {
275275
"Audio source is not set yet"
276276
}
@@ -300,7 +300,7 @@ internal class AudioInput(
300300
}
301301

302302
withContext(coroutineDispatcher) {
303-
audioSourceMutex.withLock {
303+
sourceMutex.withLock {
304304
_isStreamingFlow.emit(false)
305305
try {
306306
port.stopStream()
@@ -323,7 +323,7 @@ internal class AudioInput(
323323
}
324324

325325
withContext(coroutineDispatcher) {
326-
audioSourceMutex.withLock {
326+
sourceMutex.withLock {
327327
_isStreamingFlow.emit(false)
328328
try {
329329
port.removeInput()

0 commit comments

Comments
 (0)