Make Media3 write the container and codec it was asked for

Media3Engine never called setMuxerFactory or setAudioMimeType, and built a bare
EditedMediaItem, so it always produced MP4 with an H.265 video track. The router
meanwhile sends it WebM, Ogg, WAV and AAC-ADTS jobs, plus audio-only M4A, Opus
and WAV — and ConversionWorker.media3MimeType() mapped VideoCodec.NONE through
its else branch to VIDEO_H265.

The visible result: "extract audio to M4A" transcoded the video to HEVC and
named the file .m4a. Nothing failed, and nothing caught it, because
Media3EngineTest had no audio-only case at all.

media3-muxer already ships WebmMuxer, OggMuxer, WavMuxer and AacMuxer; only the
MP4 ones come pre-wrapped as a Muxer.Factory. Media3Muxers supplies the rest.
Their reported sample MIME types are read from each muxer's own support check
rather than assumed, because Transformer uses those lists to decide whether a
track needs re-encoding.

HardwareTranscoder.transcode now takes the OutputFormat instead of a video MIME
string, which is what gives the container, the audio codec and "this output has
no video" somewhere to travel.

MEDIA3_CONTAINERS stops being private so a test can assert it agrees with the
factories. Those two drifted once already: the router's set was right the whole
time the engine was ignoring it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-08-22 07:07:06 -05:00
co-authored by Claude Opus 5
parent 243e7ec03d
commit 00c422f317
9 changed files with 407 additions and 36 deletions
@@ -100,7 +100,7 @@ class RealMediaBenchmark {
val hwOut = File(context.cacheDir, "bench_hw.mp4").apply { delete() }
val engine = Media3Engine(context)
val hwMs = try {
runCatching { timed { engine.transcode(uri, hwOut, MimeTypes.VIDEO_H265) } }
runCatching { timed { engine.transcode(uri, hwOut, OutputFormat.MP4_H265) } }
.onFailure { Log.w(TAG, "BENCH hardware: UNSUPPORTED (${it.message})") }
.getOrNull()
} finally {
@@ -162,7 +162,7 @@ class RealMediaBenchmark {
val out = File(context.cacheDir, "bench_av1_out.mp4").apply { delete() }
val engine = Media3Engine(context)
val ms = try {
timed { engine.transcode(Uri.fromFile(input), out, MimeTypes.VIDEO_H265) }
timed { engine.transcode(Uri.fromFile(input), out, OutputFormat.MP4_H265) }
} finally {
engine.close()
}
@@ -11,10 +11,12 @@ import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.withTimeout
import org.junit.After
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Before
import org.junit.Test
import org.junit.runner.RunWith
import org.libremediaconverter.model.OutputFormat
import java.io.File
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
@@ -60,7 +62,7 @@ class Media3EngineTest {
engine.transcode(
input = Uri.fromFile(input),
output = output,
videoMimeType = MimeTypes.VIDEO_H265,
format = OutputFormat.MP4_H265,
) { percent -> seen += percent }
assertTrue("export produced no file", output.exists())
@@ -77,6 +79,77 @@ class Media3EngineTest {
seen.forEach { assertTrue("progress out of range: $it", it in 0..100) }
}
/**
* Regression guard for the audio-extraction bug.
*
* [OutputFormat.M4A_AAC] declares `VideoCodec.NONE`, and the router sends it to Media3. But
* the engine used to build a bare `EditedMediaItem` and take a video MIME type that defaulted
* to HEVC, so "extract the audio" transcoded the *video* to H.265 and wrote it to a file named
* `.m4a`. Nothing failed; the output was simply not what was asked for.
*
* This is the test the suite was missing — [Media3EngineTest] had no audio-only case at all,
* which is why the defect survived.
*/
@Test
fun audioOnlyExportDropsTheVideoTrack(): Unit = runBlocking {
val audio = File(context.cacheDir, "out_audio.m4a")
audio.delete()
try {
engine.transcode(Uri.fromFile(input), audio, OutputFormat.M4A_AAC)
assertTrue("export produced no file", audio.exists() && audio.length() > 0)
val tracks = trackMimeTypesOf(audio)
assertEquals("expected exactly one track, got $tracks", 1, tracks.size)
assertEquals(MimeTypes.AUDIO_AAC, tracks.single())
assertNull("an audio-only export must carry no video track", videoMimeTypeOf(audio))
assertTrue("output has no duration", durationMsOf(audio) > 0)
} finally {
audio.delete()
}
}
/**
* WAV output, which reaches Media3 for the same reason M4A does.
*
* `Container.WAV` is in the router's Media3 set, but until the engine passed a muxer factory
* the only muxer Transformer ever used was the MP4 one — so asking for WAV produced an MP4.
* Asserting the RIFF header proves the container, not merely that a file appeared; this
* follows what `FFmpegEngineTest` already does for its formats.
*/
@Test
fun wavExportWritesARiffHeader(): Unit = runBlocking {
val wav = File(context.cacheDir, "out_audio.wav")
wav.delete()
try {
engine.transcode(Uri.fromFile(input), wav, OutputFormat.WAV)
assertTrue("export produced no file", wav.exists() && wav.length() > 0)
assertEquals("RIFF", String(wav.readBytes().copyOfRange(0, 4), Charsets.US_ASCII))
} finally {
wav.delete()
}
}
/**
* Ogg/Opus output, the third container the router claims for Media3.
*
* Asserted by magic bytes rather than `MediaExtractor`: platform extractor support for raw
* Ogg is inconsistent across the API levels in the CI matrix, and "OggS" is unambiguous.
*/
@Test
fun opusExportWritesAnOggHeader(): Unit = runBlocking {
val ogg = File(context.cacheDir, "out_audio.opus")
ogg.delete()
try {
engine.transcode(Uri.fromFile(input), ogg, OutputFormat.OPUS)
assertTrue("export produced no file", ogg.exists() && ogg.length() > 0)
assertEquals("OggS", String(ogg.readBytes().copyOfRange(0, 4), Charsets.US_ASCII))
} finally {
ogg.delete()
}
}
/**
* Regression guard for the Transformer threading trap.
*
@@ -124,7 +197,7 @@ class Media3EngineTest {
val failure = runCatching {
runBlocking {
withTimeout(30_000) {
engine.transcode(Uri.fromFile(input), impossible, MimeTypes.VIDEO_H265) {}
engine.transcode(Uri.fromFile(input), impossible, OutputFormat.MP4_H265) {}
}
}
}.exceptionOrNull()
@@ -148,6 +221,17 @@ class Media3EngineTest {
}
}
private fun trackMimeTypesOf(file: File): List<String> {
val extractor = MediaExtractor()
return try {
extractor.setDataSource(file.absolutePath)
(0 until extractor.trackCount)
.map { extractor.getTrackFormat(it).getString(MediaFormat.KEY_MIME).orEmpty() }
} finally {
extractor.release()
}
}
private fun videoMimeTypeOf(file: File): String? {
val extractor = MediaExtractor()
try {
@@ -7,6 +7,7 @@ import org.libremediaconverter.convert.HardwareTranscoder
import org.libremediaconverter.convert.OutputPublisher
import org.libremediaconverter.convert.SoftwareTranscoder
import org.libremediaconverter.model.ConversionRequest
import org.libremediaconverter.model.OutputFormat
import java.io.File
/**
@@ -24,7 +25,7 @@ object FakeFailures {
override suspend fun transcode(
input: Uri,
output: File,
videoMimeType: String,
format: OutputFormat,
onProgress: (Int) -> Unit,
) {
called = true
@@ -15,6 +15,9 @@ import androidx.media3.transformer.ProgressHolder
import androidx.media3.transformer.Transformer
import kotlinx.coroutines.CancellableContinuation
import kotlinx.coroutines.suspendCancellableCoroutine
import org.libremediaconverter.model.AudioCodec
import org.libremediaconverter.model.OutputFormat
import org.libremediaconverter.model.VideoCodec
import java.io.File
import kotlin.coroutines.resume
import kotlin.coroutines.resumeWithException
@@ -55,12 +58,19 @@ class Media3Engine(private val context: Context) : HardwareTranscoder {
override suspend fun transcode(
input: Uri,
output: File,
videoMimeType: String,
format: OutputFormat,
onProgress: (Int) -> Unit,
): Unit = suspendCancellableCoroutine { cont ->
handler.post {
val transformer = buildTransformer(videoMimeType, cont)
val item = EditedMediaItem.Builder(MediaItem.fromUri(input)).build()
val transformer = runCatching { buildTransformer(format, cont) }
.getOrElse { cont.resumeWithException(it); return@post }
// Dropping the tracks the target format does not have is what stops an audio-only
// export from carrying a re-encoded video track. Without setRemoveVideo, asking for
// M4A produced an HEVC stream in a file named .m4a.
val item = EditedMediaItem.Builder(MediaItem.fromUri(input))
.setRemoveVideo(format.videoCodec == VideoCodec.NONE)
.setRemoveAudio(format.audioCodec == AudioCodec.NONE)
.build()
cont.invokeOnCancellation {
// cancel() has the same single-thread requirement as start().
@@ -74,26 +84,69 @@ class Media3Engine(private val context: Context) : HardwareTranscoder {
}
}
/**
* @throws IllegalArgumentException if [format] names a container Media3 cannot mux. That is a
* routing bug rather than a runtime condition — [org.libremediaconverter.model.ConversionRouter]
* is supposed to have sent such a job to FFmpeg — so it fails loudly instead of quietly
* writing MP4, which is what the old code did.
*/
private fun buildTransformer(
videoMimeType: String,
format: OutputFormat,
cont: CancellableContinuation<Unit>,
): Transformer = Transformer.Builder(context)
.setLooper(thread.looper)
.setVideoMimeType(videoMimeType)
.addListener(object : Transformer.Listener {
override fun onCompleted(composition: Composition, result: ExportResult) {
if (cont.isActive) cont.resume(Unit)
}
): Transformer {
val muxerFactory = requireNotNull(Media3Muxers.factoryFor(format.container)) {
"Media3 cannot mux ${format.container}; this job should have routed to FFmpeg."
}
override fun onError(
composition: Composition,
result: ExportResult,
exception: ExportException,
) {
if (cont.isActive) cont.resumeWithException(exception)
}
})
.build()
val builder = Transformer.Builder(context)
.setLooper(thread.looper)
.setMuxerFactory(muxerFactory)
// Only name a MIME type for a track the output actually keeps. Naming one for a removed
// track makes Transformer build an encoder for samples that will never arrive.
videoMimeTypeFor(format.videoCodec)?.let(builder::setVideoMimeType)
audioMimeTypeFor(format.audioCodec)?.let(builder::setAudioMimeType)
return builder
.addListener(object : Transformer.Listener {
override fun onCompleted(composition: Composition, result: ExportResult) {
if (cont.isActive) cont.resume(Unit)
}
override fun onError(
composition: Composition,
result: ExportResult,
exception: ExportException,
) {
if (cont.isActive) cont.resumeWithException(exception)
}
})
.build()
}
/**
* Media3 encodes only H.264 and H.265 of the codecs this app offers.
*
* VP8/VP9/AV1 targets never reach here — the router sends them to FFmpeg because
* `Transformer.setVideoMimeType` rejects them — so anything unexpected returns null and lets
* Transformer pick, rather than silently substituting H.265 the way the old mapping did.
*/
private fun videoMimeTypeFor(codec: VideoCodec): String? = when (codec) {
VideoCodec.H264 -> MimeTypes.VIDEO_H264
VideoCodec.H265 -> MimeTypes.VIDEO_H265
VideoCodec.NONE -> null
VideoCodec.VP8, VideoCodec.VP9, VideoCodec.AV1 -> null
}
private fun audioMimeTypeFor(codec: AudioCodec): String? = when (codec) {
AudioCodec.AAC -> MimeTypes.AUDIO_AAC
AudioCodec.OPUS -> MimeTypes.AUDIO_OPUS
AudioCodec.VORBIS -> MimeTypes.AUDIO_VORBIS
AudioCodec.PCM -> MimeTypes.AUDIO_RAW
AudioCodec.NONE -> null
// MP3 and FLAC have no Android encoder; the router routes them to FFmpeg.
AudioCodec.MP3, AudioCodec.FLAC -> null
}
/**
* Polls export progress on the Transformer's own thread.
@@ -0,0 +1,128 @@
package org.libremediaconverter.convert
import androidx.media3.common.C
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi
import androidx.media3.muxer.AacMuxer
import androidx.media3.muxer.Muxer
import androidx.media3.muxer.MuxerException
import androidx.media3.muxer.OggMuxer
import androidx.media3.muxer.SeekableMuxerOutput
import androidx.media3.muxer.WavMuxer
import androidx.media3.muxer.WebmMuxer
import androidx.media3.transformer.DefaultMuxer
import com.google.common.collect.ImmutableList
import org.libremediaconverter.model.Container
import java.io.FileOutputStream
/**
* Muxer factories for the containers Media3 can write.
*
* `media3-muxer` ships `Mp4Muxer`, `WebmMuxer`, `OggMuxer`, `WavMuxer` and `AacMuxer`, but
* `media3-transformer` only wraps the MP4 ones as a [Muxer.Factory]. Everything else needs the
* three lines of glue below, which is why the app previously wrote MP4 no matter what container
* was asked for: [androidx.media3.transformer.Transformer.Builder] defaults to
* `DefaultMuxer.Factory`, and nothing ever overrode it.
*
* ## The MIME lists are load-bearing
*
* `Transformer` calls [Muxer.Factory.getSupportedSampleMimeTypes] to decide whether a track can be
* copied through or has to be re-encoded, and `Transformer.Builder.build()` validates the
* requested MIME types against it. A list that over-claims produces a file the muxer cannot
* actually write; one that under-claims forces a needless re-encode. The values here were read out
* of each muxer's own `isMimeTypeSupported` check rather than assumed.
*/
@UnstableApi
object Media3Muxers {
/**
* The factory for [container], or null when Media3 cannot mux it at all.
*
* A null here must agree with [org.libremediaconverter.model.ConversionRouter]'s container set
* — if the router sends Media3 a job this cannot mux, the conversion fails and falls back to
* FFmpeg, which is a slow way to discover a routing bug. `Media3MuxersTest` asserts they agree.
*/
fun factoryFor(container: Container): Muxer.Factory? = when (container) {
// Transformer's own default. Named explicitly so the MP4 path reads the same as the rest.
Container.MP4 -> DefaultMuxer.Factory()
Container.WEBM -> WebmFactory
Container.OGG -> OggFactory
Container.WAV -> WavFactory
Container.AAC_ADTS -> AacFactory
// Matroska, MP3 and the image outputs have no Media3 muxer. FFmpeg owns them.
Container.MKV,
Container.MP3,
Container.GIF,
Container.IMAGE_SEQUENCE,
-> null
}
private object WebmFactory : Muxer.Factory {
override fun create(path: String): Muxer = wrapFailure(path) {
WebmMuxer.Builder(SeekableMuxerOutput.of(path)).build()
}
override fun getSupportedSampleMimeTypes(trackType: Int): ImmutableList<String> =
when (trackType) {
C.TRACK_TYPE_VIDEO -> ImmutableList.of(MimeTypes.VIDEO_VP8, MimeTypes.VIDEO_VP9)
C.TRACK_TYPE_AUDIO -> ImmutableList.of(MimeTypes.AUDIO_OPUS, MimeTypes.AUDIO_VORBIS)
else -> ImmutableList.of()
}
}
private object OggFactory : Muxer.Factory {
override fun create(path: String): Muxer = wrapFailure(path) {
OggMuxer.Builder(FileOutputStream(path).channel).build()
}
override fun getSupportedSampleMimeTypes(trackType: Int): ImmutableList<String> =
if (trackType == C.TRACK_TYPE_AUDIO) {
ImmutableList.of(MimeTypes.AUDIO_OPUS, MimeTypes.AUDIO_VORBIS)
} else {
ImmutableList.of()
}
}
private object WavFactory : Muxer.Factory {
override fun create(path: String): Muxer = wrapFailure(path) {
WavMuxer(SeekableMuxerOutput.of(path))
}
override fun getSupportedSampleMimeTypes(trackType: Int): ImmutableList<String> =
if (trackType == C.TRACK_TYPE_AUDIO) {
ImmutableList.of(MimeTypes.AUDIO_RAW)
} else {
ImmutableList.of()
}
}
private object AacFactory : Muxer.Factory {
override fun create(path: String): Muxer = wrapFailure(path) {
AacMuxer(FileOutputStream(path))
}
override fun getSupportedSampleMimeTypes(trackType: Int): ImmutableList<String> =
if (trackType == C.TRACK_TYPE_AUDIO) {
ImmutableList.of(MimeTypes.AUDIO_AAC)
} else {
ImmutableList.of()
}
}
/**
* Turns an I/O failure into a [MuxerException].
*
* Opening the output can throw `FileNotFoundException`, which is not what `Muxer.Factory`
* declares. Transformer's error handling only recognises `MuxerException`, so letting the raw
* IOException escape turns a bad output path into an unhandled crash instead of a reported
* export failure — the case `Media3EngineTest.anUnwritableOutputPathFailsInsteadOfHanging`
* exists to pin down.
*/
private inline fun wrapFailure(path: String, open: () -> Muxer): Muxer =
try {
open()
} catch (e: Exception) {
if (e is MuxerException) throw e
throw MuxerException("Could not open $path for muxing", e)
}
}
@@ -5,15 +5,23 @@ import android.net.Uri
import org.libremediaconverter.ffmpeg.FFmpegEngine
import org.libremediaconverter.model.ConversionRequest
import org.libremediaconverter.model.DeviceCodecs
import org.libremediaconverter.model.OutputFormat
import org.libremediaconverter.codec.AndroidDeviceCodecs
import java.io.File
/** The hardware conversion path. Implemented by [Media3Engine]. */
interface HardwareTranscoder : AutoCloseable {
/**
* Takes the whole [OutputFormat] rather than just a video MIME type.
*
* The narrower signature was the reason "extract audio to M4A" produced an HEVC video track:
* the container, the audio codec and "this output has no video at all" had nowhere to travel,
* so the engine defaulted all three.
*/
suspend fun transcode(
input: Uri,
output: File,
videoMimeType: String = androidx.media3.common.MimeTypes.VIDEO_H265,
format: OutputFormat = OutputFormat.MP4_H265,
onProgress: (Int) -> Unit = {},
)
}
@@ -13,8 +13,14 @@ package org.libremediaconverter.model
*/
object ConversionRouter {
/** Containers Media3 can mux. Anything else has to go to FFmpeg. */
private val MEDIA3_CONTAINERS = setOf(
/**
* Containers Media3 can mux. Anything else has to go to FFmpeg.
*
* Not private: `Media3Muxers` has to supply a `Muxer.Factory` for every entry, and a test
* asserts the two agree. They drifted once already — this set was correct while the engine
* silently wrote MP4 for all five.
*/
internal val MEDIA3_CONTAINERS = setOf(
Container.MP4,
Container.WEBM,
Container.OGG,
@@ -3,7 +3,6 @@ package org.libremediaconverter.work
import android.content.Context
import android.net.Uri
import android.util.Log
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi
import androidx.work.CoroutineWorker
import androidx.work.Data
@@ -21,7 +20,6 @@ import org.libremediaconverter.model.Engine
import org.libremediaconverter.model.EnginePreference
import org.libremediaconverter.model.OutputFormat
import org.libremediaconverter.model.QualityTier
import org.libremediaconverter.model.VideoCodec
import java.io.File
/**
@@ -115,7 +113,7 @@ class ConversionWorker(
) {
val engine = ConversionDependencies.hardware(applicationContext)
try {
engine.transcode(inputUri, staged, media3MimeType(request.format)) { percent ->
engine.transcode(inputUri, staged, request.format) { percent ->
publishProgress(displayName, percent)
}
return
@@ -189,11 +187,6 @@ class ConversionWorker(
}
}
private fun media3MimeType(format: OutputFormat): String = when (format.videoCodec) {
VideoCodec.H264 -> MimeTypes.VIDEO_H264
else -> MimeTypes.VIDEO_H265
}
override suspend fun getForegroundInfo(): ForegroundInfo =
foregroundInfo(
inputData.getString(KEY_DISPLAY_NAME) ?: "input",
@@ -0,0 +1,98 @@
package org.libremediaconverter.convert
import androidx.media3.common.C
import androidx.media3.common.MimeTypes
import androidx.media3.common.util.UnstableApi
import org.junit.Assert.assertEquals
import org.junit.Assert.assertNotNull
import org.junit.Assert.assertNull
import org.junit.Assert.assertTrue
import org.junit.Test
import org.libremediaconverter.model.Container
import org.libremediaconverter.model.ConversionRouter
/**
* Guards the agreement between the router's container set and the muxers that back it.
*
* These two drifted once already, and the failure was invisible: [ConversionRouter] correctly
* claimed WebM, Ogg, WAV and AAC for Media3 while [Media3Engine] had no way to write any of them
* and produced MP4 regardless. Nothing failed — the file was simply the wrong container.
*
* A JVM test rather than an instrumented one: constructing a factory touches no Android APIs, only
* `create()` does, so the mapping is checkable without a device.
*/
@UnstableApi
class Media3MuxersTest {
@Test
fun `every container the router sends to Media3 has a muxer factory`() {
ConversionRouter.MEDIA3_CONTAINERS.forEach { container ->
assertNotNull(
"$container is routed to Media3 but has no Muxer.Factory",
Media3Muxers.factoryFor(container),
)
}
}
@Test
fun `containers the router withholds from Media3 have no factory`() {
Container.entries
.filter { it !in ConversionRouter.MEDIA3_CONTAINERS }
.forEach { container ->
assertNull(
"$container has a Muxer.Factory but the router never sends it to Media3",
Media3Muxers.factoryFor(container),
)
}
}
@Test
fun `WebM advertises only the codecs its muxer accepts`() {
val factory = requireNotNull(Media3Muxers.factoryFor(Container.WEBM))
assertEquals(
listOf(MimeTypes.VIDEO_VP8, MimeTypes.VIDEO_VP9),
factory.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO),
)
assertEquals(
listOf(MimeTypes.AUDIO_OPUS, MimeTypes.AUDIO_VORBIS),
factory.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO),
)
}
/**
* The audio-only containers must not claim a video track.
*
* Transformer reads these lists to decide what it may hand the muxer. Claiming video for a
* container that cannot hold it turns a routing mistake into a corrupt file rather than a
* clean failure.
*/
@Test
fun `audio-only containers advertise no video codecs`() {
listOf(Container.OGG, Container.WAV, Container.AAC_ADTS).forEach { container ->
val factory = requireNotNull(Media3Muxers.factoryFor(container))
assertTrue(
"$container claims video support",
factory.getSupportedSampleMimeTypes(C.TRACK_TYPE_VIDEO).isEmpty(),
)
assertTrue(
"$container claims no audio support",
factory.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO).isNotEmpty(),
)
}
}
@Test
fun `WAV carries PCM and AAC-ADTS carries AAC`() {
assertEquals(
listOf(MimeTypes.AUDIO_RAW),
requireNotNull(Media3Muxers.factoryFor(Container.WAV))
.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO),
)
assertEquals(
listOf(MimeTypes.AUDIO_AAC),
requireNotNull(Media3Muxers.factoryFor(Container.AAC_ADTS))
.getSupportedSampleMimeTypes(C.TRACK_TYPE_AUDIO),
)
}
}