feat(core): mirror front camera

See #72
ThibaultBee · Nov 16, 2023 · c713559 · c713559
1 parent a13439e
commit c713559
Show file tree

Hide file tree

Showing 13 changed files with 148 additions and 29 deletions.
diff --git a/...rc/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt b/...rc/main/java/io/github/thibaultbee/streampack/internal/encoders/VideoMediaCodecEncoder.kt
@@ -28,7 +28,8 @@ import io.github.thibaultbee.streampack.data.VideoConfig
 import io.github.thibaultbee.streampack.internal.gl.EglWindowSurface
 import io.github.thibaultbee.streampack.internal.gl.FullFrameRect
 import io.github.thibaultbee.streampack.internal.gl.Texture2DProgram
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationListener
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.listeners.OnErrorListener
 import java.util.concurrent.Executors
 
@@ -112,7 +113,7 @@ class VideoMediaCodecEncoder(
     class CodecSurface(
         private val orientationProvider: ISourceOrientationProvider?
     ) :
-        SurfaceTexture.OnFrameAvailableListener {
+        SurfaceTexture.OnFrameAvailableListener, ISourceOrientationListener {
         private var eglSurface: EglWindowSurface? = null
         private var fullFrameRect: FullFrameRect? = null
         private var textureId = -1
@@ -121,8 +122,9 @@ class VideoMediaCodecEncoder(
         private var surfaceTexture: SurfaceTexture? = null
         private val stMatrix = FloatArray(16)
 
+        private var _inputSurface: Surface? = null
         val inputSurface: Surface?
-            get() = surfaceTexture?.let { Surface(surfaceTexture) }
+            get() = _inputSurface
 
         var outputSurface: Surface? = null
             set(value) {
@@ -144,6 +146,10 @@ class VideoMediaCodecEncoder(
                 field = value
             }
 
+        init {
+            orientationProvider?.addListener(this)
+        }
+
         private fun initOrUpdateSurfaceTexture(surface: Surface) {
             eglSurface = ensureGlContext(EglWindowSurface(surface)) {
                 val width = it.getWidth()
@@ -155,7 +161,8 @@ class VideoMediaCodecEncoder(
                     textureId = createTextureObject()
                     setMVPMatrixAndViewPort(
                         orientation.toFloat(),
-                        size
+                        size,
+                        orientationProvider?.mirroredVertically ?: false
                     )
                 }
 
@@ -171,7 +178,9 @@ class VideoMediaCodecEncoder(
         @SuppressLint("Recycle")
         private fun attachOrBuildSurfaceTexture(surfaceTexture: SurfaceTexture?): SurfaceTexture {
             return if (surfaceTexture == null) {
-                SurfaceTexture(textureId)
+                SurfaceTexture(textureId).apply {
+                    _inputSurface = Surface(this)
+                }
             } else {
                 surfaceTexture.attachToGLContext(textureId)
                 surfaceTexture
@@ -190,12 +199,40 @@ class VideoMediaCodecEncoder(
             return surface
         }
 
-        override fun onFrameAvailable(surfaceTexture: SurfaceTexture) {
-            synchronized(this) {
-                if (!isRunning) {
-                    return
+        override fun onOrientationChanged() {
+            executor.execute {
+                synchronized(this) {
+                    ensureGlContext(eglSurface) {
+                        val width = it.getWidth()
+                        val height = it.getHeight()
+
+                        fullFrameRect?.setMVPMatrixAndViewPort(
+                            (orientationProvider?.orientation ?: 0).toFloat(),
+                            orientationProvider?.getOrientedSize(Size(width, height)) ?: Size(
+                                width,
+                                height
+                            ),
+                            orientationProvider?.mirroredVertically ?: false
+                        )
+
+                        /**
+                         * Flushing spurious latest camera frames that block SurfaceTexture buffer
+                         * to avoid having a misoriented frame.
+                         */
+                        surfaceTexture?.updateTexImage()
+                        surfaceTexture?.releaseTexImage()
+                    }
                 }
-                executor.execute {
+            }
+        }
+
+        override fun onFrameAvailable(surfaceTexture: SurfaceTexture) {
+            if (!isRunning) {
+                return
+            }
+
+            executor.execute {
+                synchronized(this) {
                     eglSurface?.let {
                         it.makeCurrent()
                         surfaceTexture.updateTexImage()
@@ -238,8 +275,10 @@ class VideoMediaCodecEncoder(
             }.get()
         }
 
-        fun dispose() {
+        fun release() {
+            orientationProvider?.removeListener(this)
             stopStream()
+            surfaceTexture?.setOnFrameAvailableListener(null)
             surfaceTexture?.release()
             surfaceTexture = null
         }

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/gl/FullFrameRect.kt
@@ -103,12 +103,13 @@ class FullFrameRect(var program: Texture2DProgram) {
         return program.createTextureObject()
     }
 
-    fun setMVPMatrixAndViewPort(rotation: Float, resolution: Size) {
+    fun setMVPMatrixAndViewPort(rotation: Float, resolution: Size, mirroredVertically: Boolean) {
         Matrix.setIdentityM(mvpMatrix, 0)
         Matrix.rotateM(
             mvpMatrix, 0,
             rotation, 0f, 0f, -1f
         )
+        Matrix.scaleM(mvpMatrix, 0, if (mirroredVertically) -1f else 1f, 1f, 0f)
         GLES20.glViewport(0, 0, resolution.width, resolution.height)
     }
 

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/IMuxer.kt
@@ -17,7 +17,7 @@ package io.github.thibaultbee.streampack.internal.muxers
 
 import io.github.thibaultbee.streampack.data.Config
 import io.github.thibaultbee.streampack.internal.data.Frame
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.interfaces.Streamable
 
 interface IMuxer : Streamable<Unit> {

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/FlvMuxer.kt
@@ -19,7 +19,7 @@ import io.github.thibaultbee.streampack.data.Config
 import io.github.thibaultbee.streampack.internal.data.Frame
 import io.github.thibaultbee.streampack.internal.data.Packet
 import io.github.thibaultbee.streampack.internal.data.PacketType
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.muxers.IMuxer
 import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener
 import io.github.thibaultbee.streampack.internal.muxers.flv.tags.AVTagsFactory

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/flv/tags/OnMetadata.kt
@@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.internal.muxers.flv.tags
 import io.github.thibaultbee.streampack.data.AudioConfig
 import io.github.thibaultbee.streampack.data.Config
 import io.github.thibaultbee.streampack.data.VideoConfig
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.muxers.flv.amf.containers.AmfContainer
 import io.github.thibaultbee.streampack.internal.muxers.flv.amf.containers.AmfEcmaArray
 import io.github.thibaultbee.streampack.internal.muxers.flv.tags.video.CodecID

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/mp4/MP4Muxer.kt
@@ -18,7 +18,7 @@ package io.github.thibaultbee.streampack.internal.muxers.mp4
 import io.github.thibaultbee.streampack.data.Config
 import io.github.thibaultbee.streampack.internal.data.Frame
 import io.github.thibaultbee.streampack.internal.data.Packet
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.muxers.IMuxer
 import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener
 import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.FileTypeBox

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/muxers/ts/TSMuxer.kt
@@ -20,7 +20,7 @@ import android.media.MediaFormat
 import io.github.thibaultbee.streampack.data.AudioConfig
 import io.github.thibaultbee.streampack.data.Config
 import io.github.thibaultbee.streampack.internal.data.Frame
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.muxers.IMuxer
 import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener
 import io.github.thibaultbee.streampack.internal.muxers.ts.data.Service

diff --git a/...o/github/thibaultbee/streampack/internal/orientation/AbstractSourceOrientationProvider.kt b/...o/github/thibaultbee/streampack/internal/orientation/AbstractSourceOrientationProvider.kt
@@ -0,0 +1,18 @@
+package io.github.thibaultbee.streampack.internal.orientation
+
+abstract class AbstractSourceOrientationProvider : ISourceOrientationProvider {
+    protected val listeners = mutableSetOf<ISourceOrientationListener>()
+
+    override val mirroredVertically = false
+    override fun addListener(listener: ISourceOrientationListener) {
+        listeners.add(listener)
+    }
+
+    override fun removeListener(listener: ISourceOrientationListener) {
+        listeners.remove(listener)
+    }
+
+    override fun removeAllListeners() {
+        listeners.clear()
+    }
+}
diff --git a/.../interfaces/ISourceOrientationProvider.kt → ...orientation/ISourceOrientationProvider.kt b/.../interfaces/ISourceOrientationProvider.kt → ...orientation/ISourceOrientationProvider.kt
@@ -13,7 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package io.github.thibaultbee.streampack.internal.interfaces
+package io.github.thibaultbee.streampack.internal.orientation
 
 import android.graphics.SurfaceTexture
 import android.util.Size
@@ -31,16 +31,52 @@ interface ISourceOrientationProvider {
     val orientation: Int
 
     /**
-     * Return the size with the correct orientation.
+     * If true, the source is mirrored vertically.
+     * Example: should be true for a front camera.
+     */
+    val mirroredVertically: Boolean
+
+    /**
+     * Returns the size with the correct orientation.
      * If orientation is portrait, it returns a portrait size.
      * Example:
      *  - Size = 1920x1080, if orientation is portrait, it returns 1080x1920.
      */
     fun getOrientedSize(size: Size): Size
 
     /**
-     * Return the size for [SurfaceTexture.setDefaultBufferSize].
+     * Returns the size for [SurfaceTexture.setDefaultBufferSize].
      * Override this method if the image is stretched.
      */
     fun getDefaultBufferSize(size: Size) = size
+
+    /**
+     * Adds a listener to be notified when the orientation changes.
+     *
+     * @param listener to add.
+     */
+    fun addListener(listener: ISourceOrientationListener)
+
+    /**
+     * Removes a listener.
+     *
+     * @param listener to remove.
+     */
+    fun removeListener(listener: ISourceOrientationListener)
+
+    /**
+     * Removes all registered listeners.
+     */
+    fun removeAllListeners()
+}
+
+/**
+ * Interface to be notified when the orientation changes.
+ */
+interface ISourceOrientationListener {
+    /**
+     * Called when the orientation changes.
+     * Only called if [ISourceOrientationProvider.mirroredVertically] changes for now.
+     */
+    fun onOrientationChanged()
 }
diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/IVideoSource.kt
@@ -16,7 +16,7 @@
 package io.github.thibaultbee.streampack.internal.sources
 
 import io.github.thibaultbee.streampack.data.VideoConfig
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 
 interface IVideoSource : IFrameSource<VideoConfig>, ISurfaceSource {
     /**

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/camera/CameraSource.kt
@@ -17,19 +17,22 @@ package io.github.thibaultbee.streampack.internal.sources.camera
 
 import android.Manifest
 import android.content.Context
+import android.hardware.camera2.CameraCharacteristics
 import android.util.Size
 import android.view.Surface
 import androidx.annotation.RequiresPermission
 import io.github.thibaultbee.streampack.data.VideoConfig
 import io.github.thibaultbee.streampack.internal.data.Frame
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.AbstractSourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.sources.IVideoSource
 import io.github.thibaultbee.streampack.internal.utils.extensions.deviceOrientation
 import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait
 import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize
 import io.github.thibaultbee.streampack.internal.utils.extensions.portraitize
 import io.github.thibaultbee.streampack.utils.CameraSettings
+import io.github.thibaultbee.streampack.utils.cameraList
 import io.github.thibaultbee.streampack.utils.defaultCameraId
+import io.github.thibaultbee.streampack.utils.getFacingDirection
 import io.github.thibaultbee.streampack.utils.isFrameRateSupported
 import kotlinx.coroutines.runBlocking
 import java.nio.ByteBuffer
@@ -65,7 +68,7 @@ class CameraSource(
     override val timestampOffset = CameraHelper.getTimeOffsetToMonoClock(context, cameraId)
     override val hasSurface = true
     override val hasFrames = false
-    override val orientationProvider = CameraOrientationProvider(context)
+    override val orientationProvider = CameraOrientationProvider(context, cameraId)
 
     override fun getFrame(buffer: ByteBuffer): Frame {
         throw UnsupportedOperationException("Camera expects to run in Surface mode")
@@ -93,6 +96,7 @@ class CameraSource(
         }
         cameraController.startRequestSession(fps, targets)
         isPreviewing = true
+        orientationProvider.cameraId = cameraId
     }
 
     fun stopPreview() {
@@ -127,8 +131,22 @@ class CameraSource(
     }
 
 
-    class CameraOrientationProvider(private val context: Context) :
-        ISourceOrientationProvider {
+    class CameraOrientationProvider(private val context: Context, initialCameraId: String) :
+        AbstractSourceOrientationProvider() {
+        private val isFrontFacingMap =
+            context.cameraList.associateWith { (context.getFacingDirection(it) == CameraCharacteristics.LENS_FACING_FRONT) }
+
+        var cameraId: String = initialCameraId
+            set(value) {
+                if (field == value) {
+                    return
+                }
+                val orientationChanged = mirroredVertically != isFrontFacing(value)
+                field = value
+                if (orientationChanged) {
+                    listeners.forEach { it.onOrientationChanged() }
+                }
+            }
 
         override val orientation: Int
             get() = when (context.deviceOrientation) {
@@ -139,6 +157,13 @@ class CameraSource(
                 else -> 0
             }
 
+        private fun isFrontFacing(cameraId: String): Boolean {
+            return isFrontFacingMap[cameraId] ?: false
+        }
+
+        override val mirroredVertically: Boolean
+            get() = isFrontFacing(cameraId)
+
         override fun getOrientedSize(size: Size): Size {
             return if (context.isDevicePortrait) {
                 size.portraitize()

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt b/core/src/main/java/io/github/thibaultbee/streampack/internal/sources/screen/ScreenSource.kt
@@ -28,7 +28,7 @@ import androidx.activity.result.ActivityResult
 import io.github.thibaultbee.streampack.data.VideoConfig
 import io.github.thibaultbee.streampack.error.StreamPackError
 import io.github.thibaultbee.streampack.internal.data.Frame
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.AbstractSourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.sources.IVideoSource
 import io.github.thibaultbee.streampack.internal.utils.extensions.isDevicePortrait
 import io.github.thibaultbee.streampack.internal.utils.extensions.landscapize
@@ -147,7 +147,7 @@ class ScreenSource(
     }
 
     class ScreenSourceOrientationProvider(private val context: Context) :
-        ISourceOrientationProvider {
+        AbstractSourceOrientationProvider() {
         override val orientation = 0
 
         override fun getOrientedSize(size: Size): Size {

diff --git a/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt b/core/src/main/java/io/github/thibaultbee/streampack/streamers/bases/BaseStreamer.kt
@@ -29,7 +29,7 @@ import io.github.thibaultbee.streampack.internal.encoders.IEncoderListener
 import io.github.thibaultbee.streampack.internal.encoders.VideoMediaCodecEncoder
 import io.github.thibaultbee.streampack.internal.endpoints.IEndpoint
 import io.github.thibaultbee.streampack.internal.events.EventHandler
-import io.github.thibaultbee.streampack.internal.interfaces.ISourceOrientationProvider
+import io.github.thibaultbee.streampack.internal.orientation.ISourceOrientationProvider
 import io.github.thibaultbee.streampack.internal.muxers.IMuxer
 import io.github.thibaultbee.streampack.internal.muxers.IMuxerListener
 import io.github.thibaultbee.streampack.internal.sources.IAudioSource
@@ -372,7 +372,7 @@ abstract class BaseStreamer(
      */
     override fun release() {
         audioEncoder?.release()
-        videoEncoder?.codecSurface?.dispose()
+        videoEncoder?.codecSurface?.release()
         videoEncoder?.release()
         audioSource?.release()
         videoSource?.release()