Make Media3 write the container and codec it was asked for #2

Merged
JMR-dev merged 2 commits from fix/media3-honours-output-format into main 2026-08-22 12:23:58 +00:00
10 changed files with 297 additions and 47 deletions
@@ -100,7 +100,7 @@ class RealMediaBenchmark {
val hwOut = File(context.cacheDir, "bench_hw.mp4").apply { delete() } val hwOut = File(context.cacheDir, "bench_hw.mp4").apply { delete() }
val engine = Media3Engine(context) val engine = Media3Engine(context)
val hwMs = try { val hwMs = try {
runCatching { timed { engine.transcode(uri, hwOut, MimeTypes.VIDEO_H265) } } runCatching { timed { engine.transcode(uri, hwOut, OutputFormat.MP4_H265) } }
.onFailure { Log.w(TAG, "BENCH hardware: UNSUPPORTED (${it.message})") } .onFailure { Log.w(TAG, "BENCH hardware: UNSUPPORTED (${it.message})") }
.getOrNull() .getOrNull()
} finally { } finally {
@@ -162,7 +162,7 @@ class RealMediaBenchmark {
val out = File(context.cacheDir, "bench_av1_out.mp4").apply { delete() } val out = File(context.cacheDir, "bench_av1_out.mp4").apply { delete() }
val engine = Media3Engine(context) val engine = Media3Engine(context)
val ms = try { val ms = try {
timed { engine.transcode(Uri.fromFile(input), out, MimeTypes.VIDEO_H265) } timed { engine.transcode(Uri.fromFile(input), out, OutputFormat.MP4_H265) }
} finally { } finally {
engine.close() engine.close()
} }
@@ -11,10 +11,12 @@ import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout import kotlinx.coroutines.withTimeout
import org.junit.After import org.junit.After
import org.junit.Assert.assertEquals import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue import org.junit.Assert.assertTrue
import org.junit.Before import org.junit.Before
import org.junit.Test import org.junit.Test
import org.junit.runner.RunWith import org.junit.runner.RunWith
import org.libremediaconverter.model.OutputFormat
import java.io.File import java.io.File
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
@@ -60,7 +62,7 @@ class Media3EngineTest {
engine.transcode( engine.transcode(
input = Uri.fromFile(input), input = Uri.fromFile(input),
output = output, output = output,
videoMimeType = MimeTypes.VIDEO_H265, format = OutputFormat.MP4_H265,
) { percent -> seen += percent } ) { percent -> seen += percent }
assertTrue("export produced no file", output.exists()) assertTrue("export produced no file", output.exists())
@@ -77,6 +79,35 @@ class Media3EngineTest {
seen.forEach { assertTrue("progress out of range: $it", it in 0..100) } seen.forEach { assertTrue("progress out of range: $it", it in 0..100) }
} }
/**
* Regression guard for the audio-extraction bug.
*
* [OutputFormat.M4A_AAC] declares `VideoCodec.NONE`, and the router sends it to Media3. But
* the engine used to build a bare `EditedMediaItem` and take a video MIME type that defaulted
* to HEVC, so "extract the audio" transcoded the *video* to H.265 and wrote it to a file named
* `.m4a`. Nothing failed; the output was simply not what was asked for.
*
* This is the test the suite was missing — [Media3EngineTest] had no audio-only case at all,
* which is why the defect survived.
*/
@Test
fun audioOnlyExportDropsTheVideoTrack(): Unit = runBlocking {
val audio = File(context.cacheDir, "out_audio.m4a")
audio.delete()
try {
engine.transcode(Uri.fromFile(input), audio, OutputFormat.M4A_AAC)
assertTrue("export produced no file", audio.exists() && audio.length() > 0)
val tracks = trackMimeTypesOf(audio)
assertEquals("expected exactly one track, got $tracks", 1, tracks.size)
assertEquals(MimeTypes.AUDIO_AAC, tracks.single())
assertNull("an audio-only export must carry no video track", videoMimeTypeOf(audio))
assertTrue("output has no duration", durationMsOf(audio) > 0)
} finally {
audio.delete()
}
}
/** /**
* Regression guard for the Transformer threading trap. * Regression guard for the Transformer threading trap.
* *
@@ -124,7 +155,7 @@ class Media3EngineTest {
val failure = runCatching { val failure = runCatching {
runBlocking { runBlocking {
withTimeout(30_000) { withTimeout(30_000) {
engine.transcode(Uri.fromFile(input), impossible, MimeTypes.VIDEO_H265) {} engine.transcode(Uri.fromFile(input), impossible, OutputFormat.MP4_H265) {}
} }
} }
}.exceptionOrNull() }.exceptionOrNull()
@@ -148,6 +179,17 @@ class Media3EngineTest {
} }
} }
private fun trackMimeTypesOf(file: File): List<String> {
val extractor = MediaExtractor()
return try {
extractor.setDataSource(file.absolutePath)
(0 until extractor.trackCount)
.map { extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME).orEmpty() }
} finally {
extractor.release()
}
}
private fun videoMimeTypeOf(file: File): String? { private fun videoMimeTypeOf(file: File): String? {
val extractor = MediaExtractor() val extractor = MediaExtractor()
try { try {
@@ -7,6 +7,7 @@ import org.libremediaconverter.convert.HardwareTranscoder
import org.libremediaconverter.convert.OutputPublisher import org.libremediaconverter.convert.OutputPublisher
import org.libremediaconverter.convert.SoftwareTranscoder import org.libremediaconverter.convert.SoftwareTranscoder
import org.libremediaconverter.model.ConversionRequest import org.libremediaconverter.model.ConversionRequest
import org.libremediaconverter.model.OutputFormat
import java.io.File import java.io.File
/** /**
@@ -24,7 +25,7 @@ object FakeFailures {
override suspend fun transcode( override suspend fun transcode(
input: Uri, input: Uri,
output: File, output: File,
videoMimeType: String, format: OutputFormat,
onProgress: (Int) -> Unit, onProgress: (Int) -> Unit,
) { ) {
called = true called = true
@@ -15,6 +15,9 @@ import androidx.media3.transformer.ProgressHolder
import androidx.media3.transformer.Transformer import androidx.media3.transformer.Transformer
import kotlinx.coroutines.CancellableContinuation import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.suspendCancellableCoroutine import kotlinx.coroutines.suspendCancellableCoroutine
import org.libremediaconverter.model.AudioCodec
import org.libremediaconverter.model.OutputFormat
import org.libremediaconverter.model.VideoCodec
import java.io.File import java.io.File
import kotlin.coroutines.resume import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException import kotlin.coroutines.resumeWithException
@@ -55,12 +58,19 @@ class Media3Engine(private val context: Context) : HardwareTranscoder {
override suspend fun transcode( override suspend fun transcode(
input: Uri, input: Uri,
output: File, output: File,
videoMimeType: String, format: OutputFormat,
onProgress: (Int) -> Unit, onProgress: (Int) -> Unit,
): Unit = suspendCancellableCoroutine { cont -> ): Unit = suspendCancellableCoroutine { cont ->
handler.post { handler.post {
val transformer = buildTransformer(videoMimeType, cont) val transformer = runCatching { buildTransformer(format, cont) }
val item = EditedMediaItem.Builder(MediaItem.fromUri(input)).build() .getOrElse { cont.resumeWithException(it); return@post }
// Dropping the tracks the target format does not have is what stops an audio-only
// export from carrying a re-encoded video track. Without setRemoveVideo, asking for
// M4A produced an HEVC stream in a file named .m4a.
val item = EditedMediaItem.Builder(MediaItem.fromUri(input))
.setRemoveVideo(format.videoCodec == VideoCodec.NONE)
.setRemoveAudio(format.audioCodec == AudioCodec.NONE)
.build()
cont.invokeOnCancellation { cont.invokeOnCancellation {
// cancel() has the same single-thread requirement as start(). // cancel() has the same single-thread requirement as start().
@@ -74,26 +84,69 @@ class Media3Engine(private val context: Context) : HardwareTranscoder {
} }
} }
/**
* @throws IllegalArgumentException if [format] names a container Media3 cannot mux. That is a
* routing bug rather than a runtime condition — [org.libremediaconverter.model.ConversionRouter]
* is supposed to have sent such a job to FFmpeg — so it fails loudly instead of quietly
* writing MP4, which is what the old code did.
*/
private fun buildTransformer( private fun buildTransformer(
videoMimeType: String, format: OutputFormat,
cont: CancellableContinuation<Unit>, cont: CancellableContinuation<Unit>,
): Transformer = Transformer.Builder(context) ): Transformer {
.setLooper(thread.looper) val muxerFactory = requireNotNull(Media3Muxers.factoryFor(format.container)) {
.setVideoMimeType(videoMimeType) "Media3 cannot mux ${format.container}; this job should have routed to FFmpeg."
.addListener(object : Transformer.Listener { }
override fun onCompleted(composition: Composition, result: ExportResult) {
if (cont.isActive) cont.resume(Unit)
}
override fun onError( val builder = Transformer.Builder(context)
composition: Composition, .setLooper(thread.looper)
result: ExportResult, .setMuxerFactory(muxerFactory)
exception: ExportException,
) { // Only name a MIME type for a track the output actually keeps. Naming one for a removed
if (cont.isActive) cont.resumeWithException(exception) // track makes Transformer build an encoder for samples that will never arrive.
} videoMimeTypeFor(format.videoCodec)?.let(builder::setVideoMimeType)
}) audioMimeTypeFor(format.audioCodec)?.let(builder::setAudioMimeType)
.build()
return builder
.addListener(object : Transformer.Listener {
override fun onCompleted(composition: Composition, result: ExportResult) {
if (cont.isActive) cont.resume(Unit)
}
override fun onError(
composition: Composition,
result: ExportResult,
exception: ExportException,
) {
if (cont.isActive) cont.resumeWithException(exception)
}
})
.build()
}
/**
* Media3 encodes only H.264 and H.265 of the codecs this app offers.
*
* VP8/VP9/AV1 targets never reach here — the router sends them to FFmpeg because
* `Transformer.setVideoMimeType` rejects them — so anything unexpected returns null and lets
* Transformer pick, rather than silently substituting H.265 the way the old mapping did.
*/
private fun videoMimeTypeFor(codec: VideoCodec): String? = when (codec) {
VideoCodec.H264 -> MimeTypes.VIDEO_H264
VideoCodec.H265 -> MimeTypes.VIDEO_H265
VideoCodec.NONE -> null
VideoCodec.VP8, VideoCodec.VP9, VideoCodec.AV1 -> null
}
private fun audioMimeTypeFor(codec: AudioCodec): String? = when (codec) {
AudioCodec.AAC -> MimeTypes.AUDIO_AAC
AudioCodec.OPUS -> MimeTypes.AUDIO_OPUS
AudioCodec.VORBIS -> MimeTypes.AUDIO_VORBIS
AudioCodec.PCM -> MimeTypes.AUDIO_RAW
AudioCodec.NONE -> null
// MP3 and FLAC have no Android encoder; the router routes them to FFmpeg.
AudioCodec.MP3, AudioCodec.FLAC -> null
}
/** /**
* Polls export progress on the Transformer's own thread. * Polls export progress on the Transformer's own thread.
@@ -0,0 +1,60 @@
package org.libremediaconverter.convert
import androidx.media3.common.util.UnstableApi
import androidx.media3.muxer.Muxer
import androidx.media3.transformer.DefaultMuxer
import org.libremediaconverter.model.Container
/**
* Which containers Media3 can actually write, and why it is only one.
*
* ## The four other muxers cannot be driven by Transformer
*
* `media3-muxer` 1.11.0 ships `WebmMuxer`, `OggMuxer`, `WavMuxer` and `AacMuxer` alongside the MP4
* ones, which reads like four more containers on the hardware path. It is not: **all four throw
* `UnsupportedOperationException` from `addMetadataEntry`**, and
* `MuxerWrapper.addTrackFormat` calls it for every metadata entry on the track format. Any real
* recording carries some — a creation timestamp is enough — so the export dies partway through:
*
* ```
* Caused by: java.lang.UnsupportedOperationException
* at androidx.media3.muxer.OggMuxer.addMetadataEntry(OggMuxer.java:123)
* at androidx.media3.transformer.MuxerWrapper.addTrackFormat(MuxerWrapper.java:488)
* ```
*
* They are standalone muxers, not Transformer-compatible ones. WAV fails a second way even before
* that: `DefaultEncoderFactory` has no PCM encoder, so Transformer reports "No MIME type is
* supported by both encoder and muxer" rather than passing raw samples through.
*
* Both were observed on a CI API 35 emulator, not inferred — the tests that found them were written
* on the assumption these containers worked.
*
* So Media3 writes MP4 and nothing else. WebM, Ogg, WAV and raw AAC belong to FFmpeg, which already
* produces all of them and has instrumented coverage asserting the produced files.
*/
@UnstableApi
object Media3Muxers {
/**
* The factory for [container], or null when Media3 cannot write it.
*
* A null here must agree with [org.libremediaconverter.model.ConversionRouter]'s container set;
* `Media3MuxersTest` asserts they do. They drifted once already, and expensively: the router
* claimed five containers while the engine silently wrote MP4 for all of them.
*/
fun factoryFor(container: Container): Muxer.Factory? = when (container) {
// Transformer's own default. Named explicitly so the engine states its container rather
// than inheriting one, which is how the MP4-for-everything bug went unnoticed.
Container.MP4 -> DefaultMuxer.Factory()
Container.WEBM,
Container.OGG,
Container.WAV,
Container.AAC_ADTS,
Container.MKV,
Container.MP3,
Container.GIF,
Container.IMAGE_SEQUENCE,
-> null
}
}
@@ -5,15 +5,23 @@ import android.net.Uri
import org.libremediaconverter.ffmpeg.FFmpegEngine import org.libremediaconverter.ffmpeg.FFmpegEngine
import org.libremediaconverter.model.ConversionRequest import org.libremediaconverter.model.ConversionRequest
import org.libremediaconverter.model.DeviceCodecs import org.libremediaconverter.model.DeviceCodecs
import org.libremediaconverter.model.OutputFormat
import org.libremediaconverter.codec.AndroidDeviceCodecs import org.libremediaconverter.codec.AndroidDeviceCodecs
import java.io.File import java.io.File
/** The hardware conversion path. Implemented by [Media3Engine]. */ /** The hardware conversion path. Implemented by [Media3Engine]. */
interface HardwareTranscoder : AutoCloseable { interface HardwareTranscoder : AutoCloseable {
/**
* Takes the whole [OutputFormat] rather than just a video MIME type.
*
* The narrower signature was the reason "extract audio to M4A" produced an HEVC video track:
* the container, the audio codec and "this output has no video at all" had nowhere to travel,
* so the engine defaulted all three.
*/
suspend fun transcode( suspend fun transcode(
input: Uri, input: Uri,
output: File, output: File,
videoMimeType: String = androidx.media3.common.MimeTypes.VIDEO_H265, format: OutputFormat = OutputFormat.MP4_H265,
onProgress: (Int) -> Unit = {}, onProgress: (Int) -> Unit = {},
) )
} }
@@ -13,14 +13,19 @@ package org.libremediaconverter.model
*/ */
object ConversionRouter { object ConversionRouter {
/** Containers Media3 can mux. Anything else has to go to FFmpeg. */ /**
private val MEDIA3_CONTAINERS = setOf( * Containers Media3 can mux. Anything else has to go to FFmpeg.
Container.MP4, *
Container.WEBM, * MP4 alone. This set used to name WebM, Ogg, WAV and AAC-ADTS as well, on the strength of
Container.OGG, * `media3-muxer` shipping a muxer for each — but none of those four can be driven by
Container.WAV, * Transformer at all, for the reasons `Media3Muxers` records. Nothing caught it because the
Container.AAC_ADTS, * engine ignored the container entirely and wrote MP4 regardless, so the set being wrong and
) * the engine being wrong cancelled out.
*
* Not private: `Media3Muxers` has to supply a `Muxer.Factory` for every entry, and a test
* asserts the two agree.
*/
internal val MEDIA3_CONTAINERS = setOf(Container.MP4)
/** WebM is codec-restricted: Media3's WebmMuxer writes only these. */ /** WebM is codec-restricted: Media3's WebmMuxer writes only these. */
private val WEBM_AUDIO = setOf(AudioCodec.OPUS, AudioCodec.VORBIS) private val WEBM_AUDIO = setOf(AudioCodec.OPUS, AudioCodec.VORBIS)
@@ -3,7 +3,6 @@ package org.libremediaconverter.work
import android.content.Context import android.content.Context
import android.net.Uri import android.net.Uri
import android.util.Log import android.util.Log
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi import androidx.media3.common.util.UnstableApi
import androidx.work.CoroutineWorker import androidx.work.CoroutineWorker
import androidx.work.Data import androidx.work.Data
@@ -21,7 +20,6 @@ import org.libremediaconverter.model.Engine
import org.libremediaconverter.model.EnginePreference import org.libremediaconverter.model.EnginePreference
import org.libremediaconverter.model.OutputFormat import org.libremediaconverter.model.OutputFormat
import org.libremediaconverter.model.QualityTier import org.libremediaconverter.model.QualityTier
import org.libremediaconverter.model.VideoCodec
import java.io.File import java.io.File
/** /**
@@ -115,7 +113,7 @@ class ConversionWorker(
) { ) {
val engine = ConversionDependencies.hardware(applicationContext) val engine = ConversionDependencies.hardware(applicationContext)
try { try {
engine.transcode(inputUri, staged, media3MimeType(request.format)) { percent -> engine.transcode(inputUri, staged, request.format) { percent ->
publishProgress(displayName, percent) publishProgress(displayName, percent)
} }
return return
@@ -189,11 +187,6 @@ class ConversionWorker(
} }
} }
private fun media3MimeType(format: OutputFormat): String = when (format.videoCodec) {
VideoCodec.H264 -> MimeTypes.VIDEO_H264
else -> MimeTypes.VIDEO_H265
}
override suspend fun getForegroundInfo(): ForegroundInfo = override suspend fun getForegroundInfo(): ForegroundInfo =
foregroundInfo( foregroundInfo(
inputData.getString(KEY_DISPLAY_NAME) ?: "input", inputData.getString(KEY_DISPLAY_NAME) ?: "input",
@@ -0,0 +1,60 @@
package org.libremediaconverter.convert
import androidx.media3.common.util.UnstableApi
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Test
import org.libremediaconverter.model.Container
import org.libremediaconverter.model.ConversionRouter
/**
* Guards the agreement between the router's container set and the muxers that back it.
*
* These two drifted once already, and the failure was invisible in both directions: the router
* claimed WebM, Ogg, WAV and AAC-ADTS for Media3 — none of which Transformer can actually write —
* while [Media3Engine] ignored the container and produced MP4 for all of them. Two bugs that
* cancelled out, so nothing failed and the file was simply the wrong container.
*
* A JVM test rather than an instrumented one: constructing a factory touches no Android APIs, only
* `create()` does, so the mapping is checkable without a device.
*/
@UnstableApi
class Media3MuxersTest {
@Test
fun `every container the router sends to Media3 has a muxer factory`() {
ConversionRouter.MEDIA3_CONTAINERS.forEach { container ->
assertNotNull(
"$container is routed to Media3 but has no Muxer.Factory",
Media3Muxers.factoryFor(container),
)
}
}
@Test
fun `containers the router withholds from Media3 have no factory`() {
Container.entries
.filter { it !in ConversionRouter.MEDIA3_CONTAINERS }
.forEach { container ->
assertNull(
"$container has a Muxer.Factory but the router never sends it to Media3",
Media3Muxers.factoryFor(container),
)
}
}
/**
* MP4 is the whole of it.
*
* Pinned as a value rather than left implicit, because `media3-muxer` shipping a `WebmMuxer`,
* `OggMuxer`, `WavMuxer` and `AacMuxer` makes it look like there should be five. Those four
* throw `UnsupportedOperationException` from `addMetadataEntry`, which `MuxerWrapper` calls
* unconditionally — see [Media3Muxers]. If a future Media3 release fixes that, this test is
* where the decision to widen the set gets made deliberately.
*/
@Test
fun `Media3 can write MP4 and nothing else`() {
assertEquals(setOf(Container.MP4), ConversionRouter.MEDIA3_CONTAINERS)
}
}
@@ -39,13 +39,16 @@ class ConversionRouterTest {
} }
@Test @Test
fun `webm vp9 routes to ffmpeg because media3 cannot encode vp9`() { fun `webm vp9 routes to ffmpeg`() {
// Transformer.setVideoMimeType accepts only H.263/H.264/H.265/MP4V. The WebM // Two independent reasons, either of which is sufficient: Transformer.setVideoMimeType
// muxer exists, but there is no VP9 *encoder* behind it, so producing WebM // accepts only H.263/H.264/H.265/MP4V, so there is no VP9 encoder — and Media3 cannot mux
// video is FFmpeg's job even though the container is nominally supported. // WebM at all, which it was previously credited with. The container check runs first and
// is the more fundamental of the two: even given a VP9 encoder, the file could not be
// written. This asserted NO_PLATFORM_ENCODER while the container was wrongly believed to
// be supported.
val d = route(OutputFormat.WEBM_VP9) val d = route(OutputFormat.WEBM_VP9)
assertEquals(Engine.FFMPEG, d.engine) assertEquals(Engine.FFMPEG, d.engine)
assertEquals(Reason.NO_PLATFORM_ENCODER, d.reason) assertEquals(Reason.CONTAINER_UNSUPPORTED, d.reason)
} }
@Test @Test
@@ -173,6 +176,31 @@ class ConversionRouterTest {
assertEquals(Reason.USER_FORCED_SOFTWARE, d.reason) assertEquals(Reason.USER_FORCED_SOFTWARE, d.reason)
} }
// --- containers Media3 cannot write -------------------------------------
/**
* WAV, Opus and raw AAC used to be claimed for Media3 and are not any more.
*
* `media3-muxer` ships a muxer for each, which is why they were listed — but none can be driven
* by Transformer, so the export dies at the muxer. They were never actually produced on the
* hardware path: the engine ignored the container and wrote MP4 into a file named `.wav`.
* FFmpeg produces all three, and `FFmpegEngineTest` asserts the produced files.
*/
@Test
fun `audio containers Media3 cannot mux route to FFmpeg`() {
listOf(OutputFormat.WAV, OutputFormat.OPUS).forEach { format ->
val d = route(format)
assertEquals("$format should need FFmpeg", Engine.FFMPEG, d.engine)
assertEquals(Reason.CONTAINER_UNSUPPORTED, d.reason)
}
}
/** M4A is the audio format that does stay on hardware, because its container is MP4. */
@Test
fun `m4a stays on hardware because MP4 is a container Media3 can write`() {
assertEquals(Engine.MEDIA3, route(OutputFormat.M4A_AAC).engine)
}
// --- exhaustiveness ---------------------------------------------------- // --- exhaustiveness ----------------------------------------------------
@Test @Test