From da63e2bb72183ca4d8db0f856fee556ef1667edc Mon Sep 17 00:00:00 2001 From: MarcelineVPQ Date: Sun, 24 May 2026 09:27:11 -0600 Subject: [PATCH] feat(audio): mic packet-loss concealment (jitter buffer + Opus PLC) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BT mic decode path gained a decoded-frame jitter buffer (8 frames) drained at a steady 10 ms playout cadence. Bursty BT delivery is smoothed; a dropped mic frame during an active session is concealed with an Opus PLC frame (opus_decode(decoder, NULL, 0, ...)) instead of leaving a hole the host hears as a click/dropout. Playout pre-buffers 3 frames and stops after 300 ms of no real frames so it never emits comfort noise when the mic is idle. New "Mic PLC:" Diagnostics counter climbs only when concealment fires (a live link-quality gauge). Verified on hardware: forced BT loss kept captured audio gap-free (longest zero-run ~0 ms) while the counter climbed; clean link leaves it idle. Adds ~30 ms mic latency (the pre-buffer). Design ported from SundayMoments/DS5_Bridge (credit). Unreleased — batching with the next feature. Co-Authored-By: Claude Opus 4.7 (1M context) --- CHANGELOG.md | 4 +++ README.md | 1 + src/audio.cpp | 87 ++++++++++++++++++++++++++++++++++++++++++--------- src/audio.h | 1 + src/oled.cpp | 7 +++-- 5 files changed, 83 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1031dbb..b488b03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Version ## [Unreleased] +### Changed + +- **BT microphone now has packet-loss concealment (PLC).** The mic decode path gained a small decoded-frame jitter buffer (8 frames) drained at a steady 10 ms playout cadence: bursty BT delivery is smoothed, and a dropped mic frame during an active session is concealed with an Opus PLC frame (`opus_decode(decoder, NULL, 0, …)`) instead of leaving a hole the host hears as a click/dropout. Playout pre-buffers 3 frames and stops after 300 ms of no real frames (so it never emits comfort noise when the mic is idle). A new **`Mic PLC:`** counter on the Diagnostics screen climbs only when concealment fires — effectively a live BT link-quality gauge. Verified: forced BT loss kept the captured audio gap-free (longest zero-run ~0 ms) while the counter climbed. Design ported from [SundayMoments/DS5_Bridge](https://github.com/SundayMoments/DS5_Bridge) (credit). Adds ~30 ms mic latency (the pre-buffer). + --- ## [0.6.8-oled-edition] — 2026-05-24 diff --git a/README.md b/README.md index 632699e..50f38f6 100644 --- a/README.md +++ b/README.md @@ -157,6 +157,7 @@ This was long believed impossible — earlier versions of this fork documented i - Mic audio is **mono** (decoded mono, duplicated across the stereo capture endpoint). - Toggling off mid-session stops the host feed immediately, but the controller keeps streaming until it next reconnects (there's no known "stop" command); connecting fresh with the toggle off never enables it. - The OLED **Diagnostics** screen's `Mic in:` counter reads ~100/s while the mic is streaming — a quick way to confirm it's live. +- **Packet-loss concealment:** dropped mic frames (weak BT link, distance, interference) are concealed with Opus PLC so voice stays continuous instead of clicking/cutting out, at a small jitter-buffer latency (~30 ms). The Diag screen's `Mic PLC:` counter climbs only when frames are being concealed — effectively a live link-quality gauge. ## Known Issues diff --git a/src/audio.cpp b/src/audio.cpp index 2f60073..33c1d99 100644 --- a/src/audio.cpp +++ b/src/audio.cpp @@ -61,6 +61,24 @@ int32_t audio_mic_last_decoded() { return g_mic_last_decoded; } uint16_t audio_mic_last_want() { return g_mic_last_want; } uint16_t audio_mic_last_wrote() { return g_mic_last_wrote; } +// Mic jitter buffer + packet-loss concealment. Decoded mono frames land here +// (filled as Opus arrives, drained at a steady 10 ms playout cadence) so bursty +// BT delivery is smoothed and a dropped frame is concealed via Opus PLC instead +// of underrunning the host with a click/hole. Design ported from +// SundayMoments/DS5_Bridge (credit there). PLC keeps voice continuous on a +// lossy BT link (e.g. controller moved away, USB 3.0 RF interference). +struct mic_decoded_element { int16_t mono[MIC_FRAMES]; }; +static queue_t mic_decode_fifo; +static constexpr int MIC_DECODE_DEPTH = 8; // jitter-buffer capacity (frames) +static constexpr int MIC_PLAYOUT_START = 3; // pre-buffer before playout begins +static constexpr uint64_t MIC_FRAME_US = 10000; // 10 ms per Opus frame @ 48 kHz +static constexpr uint64_t MIC_SESSION_US = 300000; // no real frame this long → stop playout +static bool mic_playout_started = false; +static uint64_t mic_next_playout_us = 0; +static uint64_t mic_last_real_us = 0; +static volatile uint32_t g_mic_plc_frames = 0; // concealed frames generated (Diag) +uint32_t audio_mic_plc_frames() { return g_mic_plc_frames; } + struct audio_raw_element { float data[512 * 2]; }; @@ -161,23 +179,61 @@ void audio_loop() { // push to the host via tud_audio_write. Runs once per loop iteration so // it keeps up with the ~100 Hz arrival rate of mic-tagged BT frames. if (mic_decoder != nullptr) { - static mic_element packet{}; - if (queue_try_remove(&mic_fifo, &packet)) { - static int16_t mono[MIC_FRAMES]; - const int decoded = opus_decode(mic_decoder, packet.data, - MIC_OPUS_SIZE, mono, MIC_FRAMES, 0); - g_mic_last_decoded = decoded; // observed in OLED Diag - if (decoded > 0) { - static int16_t stereo[MIC_FRAMES * 2]; - for (int i = 0; i < decoded; i++) { - stereo[i * 2] = mono[i]; - stereo[i * 2 + 1] = mono[i]; + const uint64_t now = time_us_64(); + + // Decode stage: drain incoming Opus into the jitter buffer as fast as it + // arrives (absorbs bursty BT delivery), up to the buffer's capacity. + static mic_element pkt{}; + while (queue_get_level(&mic_decode_fifo) < MIC_DECODE_DEPTH + && queue_try_remove(&mic_fifo, &pkt)) { + static mic_decoded_element dec{}; + const int n = opus_decode(mic_decoder, pkt.data, MIC_OPUS_SIZE, + dec.mono, MIC_FRAMES, 0); + g_mic_last_decoded = n; // observed in OLED Diag + if (n > 0) { + queue_try_add(&mic_decode_fifo, &dec); + mic_last_real_us = now; + } + } + + // Playout stage: emit one frame every 10 ms. Pre-buffer a few frames to + // absorb jitter, then play a real frame if buffered, else conceal with an + // Opus PLC frame during an active session (transient loss) so the host + // hears continuity instead of a hole. If real frames have been gone for a + // while (mic off/idle), stop so we don't emit comfort noise forever. + if (!mic_playout_started + && queue_get_level(&mic_decode_fifo) >= MIC_PLAYOUT_START) { + mic_playout_started = true; + mic_next_playout_us = now; + } + if (mic_playout_started && (int64_t)(now - mic_next_playout_us) >= 0) { + static mic_decoded_element out{}; + bool have = queue_try_remove(&mic_decode_fifo, &out); + if (!have) { + if (now - mic_last_real_us < MIC_SESSION_US) { + const int n = opus_decode(mic_decoder, nullptr, 0, + out.mono, MIC_FRAMES, 0); // PLC + if (n > 0) { have = true; g_mic_plc_frames++; } + } else { + mic_playout_started = false; // session ended — re-buffer next time } - const uint16_t want = (uint16_t)(decoded * 2 * sizeof(int16_t)); - const uint16_t wrote = tud_audio_write(stereo, want); + } + if (have) { + static int16_t stereo[MIC_FRAMES * 2]; + for (int i = 0; i < MIC_FRAMES; i++) { + stereo[i * 2] = out.mono[i]; + stereo[i * 2 + 1] = out.mono[i]; + } + const uint16_t want = (uint16_t)(MIC_FRAMES * 2 * sizeof(int16_t)); + g_mic_last_wrote = tud_audio_write(stereo, want); g_mic_last_want = want; - g_mic_last_wrote = wrote; g_mic_frames++; + mic_next_playout_us += MIC_FRAME_US; + // Drift guard: if we've fallen many frames behind (loop stall), + // resync the cadence instead of bursting to catch up. + if ((int64_t)(now - mic_next_playout_us) > (int64_t)(4 * MIC_FRAME_US)) { + mic_next_playout_us = now + MIC_FRAME_US; + } } } } @@ -376,7 +432,8 @@ void audio_init() { // Mic path: queue + decoder live on core0 (audio_loop), separate from // the core1 speaker encoder. Mic Opus is mono / 48 kHz / 10 ms frames. - queue_init(&mic_fifo, sizeof(mic_element), 2); + queue_init(&mic_fifo, sizeof(mic_element), MIC_DECODE_DEPTH); // deeper: tolerate BT bursts + queue_init(&mic_decode_fifo, sizeof(mic_decoded_element), MIC_DECODE_DEPTH); // decoded-PCM jitter buffer int dec_error = 0; mic_decoder = opus_decoder_create(48000, MIC_CHANNELS, &dec_error); if (dec_error != 0 || mic_decoder == nullptr) { diff --git a/src/audio.h b/src/audio.h index bd00c5b..ce6f96a 100644 --- a/src/audio.h +++ b/src/audio.h @@ -26,6 +26,7 @@ int32_t audio_mic_last_decoded(); // last opus_decode return — neg = error, 4 uint16_t audio_mic_last_want(); // bytes asked of tud_audio_write uint16_t audio_mic_last_wrote(); // bytes TinyUSB FIFO actually accepted uint8_t audio_mic_last_toc(); // first byte of last Opus packet (frame config) +uint32_t audio_mic_plc_frames(); // count of packet-loss-concealment frames generated // Called from on_bt_data() in main.cpp when the DS5 sends a mic-tagged // 0x31 input report. Buffer must point at MIC_OPUS_SIZE (71) bytes of diff --git a/src/oled.cpp b/src/oled.cpp index dc25c5d..7b35d94 100644 --- a/src/oled.cpp +++ b/src/oled.cpp @@ -827,7 +827,7 @@ void sample_diag_rates() { // Row list ordered by relevance: always-useful at top, parked-mic-investigation // data at bottom. To add a row, bump kNumDiagRows and add a case. -constexpr int kNumDiagRows = 11; +constexpr int kNumDiagRows = 12; __attribute__((noinline)) void format_diag_row(int idx, char* line, size_t n) { switch (idx) { @@ -872,7 +872,10 @@ void format_diag_row(int idx, char* line, size_t n) { (long)audio_mic_last_decoded(), (unsigned)audio_mic_last_wrote()); break; - case 10: { + case 10: + snprintf(line, n, "Mic PLC: %lu", (unsigned long)audio_mic_plc_frames()); + break; + case 11: { uint8_t pfx[6]; bt_31_mic_prefix(pfx); snprintf(line, n, "%02X %02X %02X %02X %02X %02X", pfx[0], pfx[1], pfx[2], pfx[3], pfx[4], pfx[5]);