Skip to content

Commit

Permalink
feat(core): add experimental support for AV1
Browse files Browse the repository at this point in the history
  • Loading branch information
ThibaultBee committed Oct 28, 2023
1 parent b0b7e45 commit 6b177e4
Show file tree
Hide file tree
Showing 13 changed files with 314 additions and 33 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package io.github.thibaultbee.streampack.data

import android.content.Context
import android.media.MediaCodecInfo.CodecProfileLevel
import android.media.MediaCodecInfo.CodecProfileLevel.AV1ProfileMain8
import android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileBaseline
import android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileConstrainedBaseline
import android.media.MediaCodecInfo.CodecProfileLevel.AVCProfileConstrainedHigh
Expand Down Expand Up @@ -48,8 +49,8 @@ import kotlin.math.roundToInt
class VideoConfig(
/**
* Video encoder mime type.
* Only [MediaFormat.MIMETYPE_VIDEO_AVC], [MediaFormat.MIMETYPE_VIDEO_HEVC] and
* [MediaFormat.MIMETYPE_VIDEO_VP9] are supported yet.
* Only [MediaFormat.MIMETYPE_VIDEO_AVC], [MediaFormat.MIMETYPE_VIDEO_HEVC],
* [MediaFormat.MIMETYPE_VIDEO_VP9] and [MediaFormat.MIMETYPE_VIDEO_AV1] are supported yet.
*
* **See Also:** [MediaFormat MIMETYPE_VIDEO_*](https://developer.android.com/reference/android/media/MediaFormat)
*/
Expand Down Expand Up @@ -223,6 +224,10 @@ class VideoConfig(
VP9Profile0
)

private val av1ProfilePriority = listOf(
AV1ProfileMain8
)

/**
* Return the higher profile with the higher level
*/
Expand All @@ -231,6 +236,7 @@ class VideoConfig(
MediaFormat.MIMETYPE_VIDEO_AVC -> avcProfilePriority
MediaFormat.MIMETYPE_VIDEO_HEVC -> hevcProfilePriority
MediaFormat.MIMETYPE_VIDEO_VP9 -> vp9ProfilePriority
MediaFormat.MIMETYPE_VIDEO_AV1 -> av1ProfilePriority
else -> throw InvalidParameterException("Profile for $mimeType is not supported")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ class VideoFlvMuxerHelper : IVideoMuxerHelper {
get() {
val extendedSupportedCodec = listOf(
MediaFormat.MIMETYPE_VIDEO_HEVC,
MediaFormat.MIMETYPE_VIDEO_VP9
MediaFormat.MIMETYPE_VIDEO_VP9,
MediaFormat.MIMETYPE_VIDEO_AV1
)
val supportedCodecList = CodecID.entries.mapNotNull {
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,15 @@ class AVTagsFactory(
VPCodecConfigurationRecord.fromMediaFormat(frame.format)
}

MediaFormat.MIMETYPE_VIDEO_AV1 -> {
if (frame.extra != null) {
// Extra is AV1CodecConfigurationRecord
PassthroughBufferWriter(frame.extra[0])
} else {
throw IOException("AV1 sequence header without CSD buffer is not supported")
}
}

else -> {
throw IOException("Unsupported video codec: ${config.mimeType}")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ class VideoMP4MuxerHelper : IVideoMuxerHelper {
listOf(
MediaFormat.MIMETYPE_VIDEO_AVC,
MediaFormat.MIMETYPE_VIDEO_HEVC,
MediaFormat.MIMETYPE_VIDEO_VP9
MediaFormat.MIMETYPE_VIDEO_VP9,
MediaFormat.MIMETYPE_VIDEO_AV1
)
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2023 Thibault B.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.github.thibaultbee.streampack.internal.muxers.mp4.boxes

import io.github.thibaultbee.streampack.internal.utils.av.video.av1.AV1CodecConfigurationRecord
import java.nio.ByteBuffer

sealed class AV1CodecConfigurationBox : Box("av1C")

class AV1CodecConfigurationBox1(private val config: AV1CodecConfigurationRecord) :
AV1CodecConfigurationBox() {
override val size: Int = super.size + config.size

override fun write(output: ByteBuffer) {
super.write(output)
config.write(output)
}
}

class AV1CodecConfigurationBox2(private val buffer: ByteBuffer) : AV1CodecConfigurationBox() {
override val size: Int = super.size + buffer.remaining()

override fun write(output: ByteBuffer) {
super.write(output)
output.put(buffer)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ class HEVCSampleEntry(
pasp
)


class VP9SampleEntry(
resolution: Size,
horizontalResolution: Int = 72,
Expand Down Expand Up @@ -161,6 +160,34 @@ class VP9SampleEntry(
pasp
)

class AV1SampleEntry(
resolution: Size,
horizontalResolution: Int = 72,
verticalResolution: Int = 72,
frameCount: Short = 1,
compressorName: String? = "AV1 Coding",
depth: Short = 0x0018,
av1C: AV1CodecConfigurationBox,
btrt: BitRateBox? = null,
extensionDescriptorsBox: List<Box> = emptyList(),
clap: CleanApertureBox? = null,
pasp: PixelAspectRatioBox? = null
) : VisualSampleEntry(
"av01",
resolution,
horizontalResolution,
verticalResolution,
frameCount,
compressorName,
depth,
mutableListOf<Box>(av1C).apply {
btrt?.let { add(it) }
addAll(extensionDescriptorsBox)
},
clap,
pasp
)

class OpusSampleEntry(
channelCount: Short,
dOps: OpusSpecificBox,
Expand All @@ -174,7 +201,8 @@ class OpusSampleEntry(
48000,
mutableListOf<Box>(dOps).apply {
btrt?.let { add(it) }
})
}
)

class MP4AudioSampleEntry(
channelCount: Short,
Expand All @@ -191,7 +219,8 @@ class MP4AudioSampleEntry(
sampleRate,
mutableListOf<Box>(esds).apply {
btrt?.let { add(it) }
})
}
)

open class AudioSampleEntry(
type: String,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import io.github.thibaultbee.streampack.data.Config
import io.github.thibaultbee.streampack.data.VideoConfig
import io.github.thibaultbee.streampack.internal.data.Frame
import io.github.thibaultbee.streampack.internal.interfaces.IOrientationProvider
import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AV1CodecConfigurationBox2
import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AV1SampleEntry
import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AVCConfigurationBox
import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.AVCSampleEntry
import io.github.thibaultbee.streampack.internal.muxers.mp4.boxes.ChunkLargeOffsetBox
Expand Down Expand Up @@ -97,6 +99,10 @@ class TrackChunks(
this.format.isNotEmpty()
}

MediaFormat.MIMETYPE_VIDEO_AV1 -> {
this.extra.size == 1
}

MediaFormat.MIMETYPE_AUDIO_AAC -> {
this.extra.size == 1
}
Expand Down Expand Up @@ -343,6 +349,18 @@ class TrackChunks(
)
}

MediaFormat.MIMETYPE_VIDEO_AV1 -> {
val extra = this.extra
require(extra.size == 1) { "For AV1, extra must contain 1 extra" }
(track.config as VideoConfig)
AV1SampleEntry(
orientationProvider.orientedSize(track.config.resolution),
av1C = AV1CodecConfigurationBox2(
extra[0][0]
),
)
}

MediaFormat.MIMETYPE_AUDIO_AAC -> {
(track.config as AudioConfig)
MP4AudioSampleEntry(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
* Copyright (C) 2023 Thibault B.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.github.thibaultbee.streampack.internal.utils.av.video.av1

import android.media.MediaFormat
import io.github.thibaultbee.streampack.internal.utils.av.buffer.ByteBufferWriter
import io.github.thibaultbee.streampack.internal.utils.extensions.put
import io.github.thibaultbee.streampack.internal.utils.extensions.putShort
import io.github.thibaultbee.streampack.internal.utils.extensions.shl
import io.github.thibaultbee.streampack.internal.utils.extensions.toInt
import java.nio.ByteBuffer

class AV1CodecConfigurationRecord(
private val marker: Boolean = true,
private val version: Byte = 1,
private val seqProfile: Byte,
private val seqLevelIdx0: Byte,
private val seqTier0: Boolean,
private val highBitdepth: Boolean,
private val twelveBit: Boolean,
private val monochrome: Boolean,
private val chromaSubsamplingX: Boolean,
private val chromaSubsamplingY: Boolean,
private val chromaSamplePosition: Byte,
private val initialPresentationDelayMinusOne: Int?,
private val configOBUs: ByteBuffer,
) : ByteBufferWriter() {
override val size: Int = AV1_DECODER_CONFIGURATION_RECORD_SIZE + configOBUs.remaining()

override fun write(output: ByteBuffer) {
output.put((marker.toInt() shl 7) or version.toInt())
output.put((seqProfile shl 5) or seqLevelIdx0.toInt())
output.putShort(
(seqTier0 shl 15) or
(highBitdepth shl 14) or
(twelveBit shl 13) or
(monochrome shl 12) or
(chromaSubsamplingX shl 11) or
(chromaSubsamplingY shl 10) or
(chromaSamplePosition shl 8) or
if (initialPresentationDelayMinusOne != null) {
(0b1 shl 4) or (initialPresentationDelayMinusOne)
} else {
0
}
)

output.put(configOBUs)
}

companion object {
private const val AV1_DECODER_CONFIGURATION_RECORD_SIZE = 4

/**
* {max-bitrate=2000000, crop-right=719, level=32, latency=0, mime=video/av01, profile=1,
* bitrate=2000000, priority=0, color-standard=1, color-transfer=3,
* hdr10-plus-info=java.nio.HeapByteBuffer[pos=0 lim=0 cap=0], crop-bottom=1279,
* video-qp-average=0, crop-left=0, width=720, bitrate-mode=2, color-range=2, crop-top=0,
* frame-rate=30, height=1280, csd-0=java.nio.HeapByteBuffer[pos=0 lim=4 cap=4]}
*
*/
fun fromMediaFormat(mediaFormat: MediaFormat): AV1CodecConfigurationRecord {
throw NotImplementedError("AV1CodecConfigurationRecord.fromMediaFormat() is not implemented")
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -176,21 +176,13 @@ open class VideoStreamerConfigurationHelper(private val videoMuxerHelper: IVideo
MediaFormat.MIMETYPE_VIDEO_AVC -> avcProfiles
MediaFormat.MIMETYPE_VIDEO_HEVC -> hevcProfiles
MediaFormat.MIMETYPE_VIDEO_VP9 -> vp9Profiles
MediaFormat.MIMETYPE_VIDEO_AV1 -> av1Profiles
else -> throw InvalidParameterException("Unknown mimetype $mimeType")
}
val supportedProfiles = MediaCodecHelper.getProfiles(mimeType)
return supportedProfiles.filter { profiles.contains(it) }
}

private val vp9Profiles = listOf(
VP9Profile0,
VP9Profile1
)

private val hevcProfiles = listOf(
HEVCProfileMain
)

private val avcProfiles = listOf(
AVCProfileBaseline,
AVCProfileConstrainedBaseline,
Expand All @@ -199,4 +191,18 @@ open class VideoStreamerConfigurationHelper(private val videoMuxerHelper: IVideo
AVCProfileHigh,
AVCProfileMain
)

private val hevcProfiles = listOf(
HEVCProfileMain
)

private val vp9Profiles = listOf(
VP9Profile0,
VP9Profile1
)

private val av1Profiles = listOf(
AV1ProfileMain8
)
}

Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,8 @@ class SettingsFragment : PreferenceFragmentCompat() {
MediaFormat.MIMETYPE_VIDEO_AVC to getString(R.string.video_encoder_h264),
MediaFormat.MIMETYPE_VIDEO_HEVC to getString(R.string.video_encoder_h265),
MediaFormat.MIMETYPE_VIDEO_H263 to getString(R.string.video_encoder_h263),
MediaFormat.MIMETYPE_VIDEO_VP9 to getString(R.string.video_encoder_vp9)
MediaFormat.MIMETYPE_VIDEO_VP9 to getString(R.string.video_encoder_vp9),
MediaFormat.MIMETYPE_VIDEO_AV1 to getString(R.string.video_encoder_av1)
)

val supportedVideoEncoder = streamerHelper.video.supportedEncoders
Expand Down
Loading

0 comments on commit 6b177e4

Please sign in to comment.