Gogledd-Orllewin

Analyse audio from ExoPlayer to create reactive visuals.

Audio processor

package io.orllewin.pcr.radio

import androidx.media3.common.C
import androidx.media3.common.audio.AudioProcessor
import androidx.media3.common.audio.BaseAudioProcessor
import androidx.media3.common.util.UnstableApi
import kotlinx.coroutines.flow.MutableStateFlow
import kotlinx.coroutines.flow.StateFlow
import kotlinx.coroutines.flow.asStateFlow
import java.nio.ByteBuffer
import java.nio.ByteOrder
import kotlin.math.sqrt

/**
 * Pass-through processor that calculates a smoothed RMS amplitude from the audio stream and
 * exposes it as a flow.
 */
@UnstableApi
class AmplitudeProcessor : BaseAudioProcessor() {

    private var encoding = C.ENCODING_INVALID

    private val _amplitude = MutableStateFlow(0f)
    val amplitude: StateFlow = _amplitude.asStateFlow()

    override fun onConfigure(
        inputAudioFormat: AudioProcessor.AudioFormat
    ): AudioProcessor.AudioFormat {
        encoding = inputAudioFormat.encoding
        return inputAudioFormat // pass-through, no format change
    }

    override fun queueInput(inputBuffer: ByteBuffer) {
        val remaining = inputBuffer.remaining()
        if (remaining == 0) return

        // Duplicate buffer
        computeAmplitude(inputBuffer.duplicate().order(inputBuffer.order()))

        // Then forward the original untouched
        replaceOutputBuffer(remaining).put(inputBuffer).flip()
    }

    override fun onReset() {
        super.onReset()
        encoding = C.ENCODING_INVALID
        _amplitude.value = 0f
    }

    private fun computeAmplitude(buf: ByteBuffer) {
        var sum = 0.0
        var count = 0

        when (encoding) {
            C.ENCODING_PCM_16BIT -> {
                val shorts = buf.order(ByteOrder.nativeOrder()).asShortBuffer()
                while (shorts.hasRemaining()) {
                    val s = shorts.get().toFloat() / Short.MAX_VALUE
                    sum += s * s
                    count++
                }
            }
            C.ENCODING_PCM_FLOAT -> {
                val floats = buf.order(ByteOrder.nativeOrder()).asFloatBuffer()
                while (floats.hasRemaining()) {
                    val s = floats.get()
                    sum += s * s
                    count++
                }
            }
            else -> return
        }

        if (count == 0) return

        // Root mean square:
        // https://en.wikipedia.org/wiki/Root_mean_square
        // https://rosettacode.org/wiki/Averages/Root_mean_square
        val rms = sqrt(sum / count).toFloat().coerceIn(0f, 1f)

        // Fast attack, slow decay tail.
        val current = _amplitude.value
        val smoothed = if (rms > current) {
            current + (rms - current) * 0.65f
        } else {
            current + (rms - current) * 0.15f
        }
        _amplitude.value = smoothed.coerceIn(0f, 1f)
    }
}

ExoPlayer sink

val amplitudeProcessor = AmplitudeProcessor()
//...
val renderersFactory = object : DefaultRenderersFactory(context) {
    override fun buildAudioSink(
        context: Context,
        enableFloatOutput: Boolean,
        enableAudioTrackPlaybackParams: Boolean
    ): AudioSink {
        return DefaultAudioSink.Builder(context)
            .setAudioProcessorChain(
                DefaultAudioSink.DefaultAudioProcessorChain(amplitudeProcessor)
            )
            .setEnableFloatOutput(enableFloatOutput)
            .setEnableAudioTrackPlaybackParams(enableAudioTrackPlaybackParams)
            .build()
    }
}

val exoPlayer = ExoPlayer.Builder(context, renderersFactory).build()

Read amplitude from flow

val amplitude: StateFlow = amplitudeProcessor.amplitude