feat(audio): mic packet-loss concealment (jitter buffer + Opus PLC)
The BT mic decode path gained a decoded-frame jitter buffer (8 frames) drained at a steady 10 ms playout cadence. Bursty BT delivery is smoothed; a dropped mic frame during an active session is concealed with an Opus PLC frame (opus_decode(decoder, NULL, 0, ...)) instead of leaving a hole the host hears as a click/dropout. Playout pre-buffers 3 frames and stops after 300 ms of no real frames so it never emits comfort noise when the mic is idle. New "Mic PLC:" Diagnostics counter climbs only when concealment fires (a live link-quality gauge). Verified on hardware: forced BT loss kept captured audio gap-free (longest zero-run ~0 ms) while the counter climbed; clean link leaves it idle. Adds ~30 ms mic latency (the pre-buffer). Design ported from SundayMoments/DS5_Bridge (credit). Unreleased — batching with the next feature. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
9920516f52
commit
da63e2bb72
@@ -8,6 +8,10 @@ Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). Version
|
|||||||
|
|
||||||
## [Unreleased]
|
## [Unreleased]
|
||||||
|
|
||||||
|
### Changed
|
||||||
|
|
||||||
|
- **BT microphone now has packet-loss concealment (PLC).** The mic decode path gained a small decoded-frame jitter buffer (8 frames) drained at a steady 10 ms playout cadence: bursty BT delivery is smoothed, and a dropped mic frame during an active session is concealed with an Opus PLC frame (`opus_decode(decoder, NULL, 0, …)`) instead of leaving a hole the host hears as a click/dropout. Playout pre-buffers 3 frames and stops after 300 ms of no real frames (so it never emits comfort noise when the mic is idle). A new **`Mic PLC:`** counter on the Diagnostics screen climbs only when concealment fires — effectively a live BT link-quality gauge. Verified: forced BT loss kept the captured audio gap-free (longest zero-run ~0 ms) while the counter climbed. Design ported from [SundayMoments/DS5_Bridge](https://github.com/SundayMoments/DS5_Bridge) (credit). Adds ~30 ms mic latency (the pre-buffer).
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## [0.6.8-oled-edition] — 2026-05-24
|
## [0.6.8-oled-edition] — 2026-05-24
|
||||||
|
|||||||
@@ -157,6 +157,7 @@ This was long believed impossible — earlier versions of this fork documented i
|
|||||||
- Mic audio is **mono** (decoded mono, duplicated across the stereo capture endpoint).
|
- Mic audio is **mono** (decoded mono, duplicated across the stereo capture endpoint).
|
||||||
- Toggling off mid-session stops the host feed immediately, but the controller keeps streaming until it next reconnects (there's no known "stop" command); connecting fresh with the toggle off never enables it.
|
- Toggling off mid-session stops the host feed immediately, but the controller keeps streaming until it next reconnects (there's no known "stop" command); connecting fresh with the toggle off never enables it.
|
||||||
- The OLED **Diagnostics** screen's `Mic in:` counter reads ~100/s while the mic is streaming — a quick way to confirm it's live.
|
- The OLED **Diagnostics** screen's `Mic in:` counter reads ~100/s while the mic is streaming — a quick way to confirm it's live.
|
||||||
|
- **Packet-loss concealment:** dropped mic frames (weak BT link, distance, interference) are concealed with Opus PLC so voice stays continuous instead of clicking/cutting out, at a small jitter-buffer latency (~30 ms). The Diag screen's `Mic PLC:` counter climbs only when frames are being concealed — effectively a live link-quality gauge.
|
||||||
|
|
||||||
## Known Issues
|
## Known Issues
|
||||||
|
|
||||||
|
|||||||
+72
-15
@@ -61,6 +61,24 @@ int32_t audio_mic_last_decoded() { return g_mic_last_decoded; }
|
|||||||
uint16_t audio_mic_last_want() { return g_mic_last_want; }
|
uint16_t audio_mic_last_want() { return g_mic_last_want; }
|
||||||
uint16_t audio_mic_last_wrote() { return g_mic_last_wrote; }
|
uint16_t audio_mic_last_wrote() { return g_mic_last_wrote; }
|
||||||
|
|
||||||
|
// Mic jitter buffer + packet-loss concealment. Decoded mono frames land here
|
||||||
|
// (filled as Opus arrives, drained at a steady 10 ms playout cadence) so bursty
|
||||||
|
// BT delivery is smoothed and a dropped frame is concealed via Opus PLC instead
|
||||||
|
// of underrunning the host with a click/hole. Design ported from
|
||||||
|
// SundayMoments/DS5_Bridge (credit there). PLC keeps voice continuous on a
|
||||||
|
// lossy BT link (e.g. controller moved away, USB 3.0 RF interference).
|
||||||
|
struct mic_decoded_element { int16_t mono[MIC_FRAMES]; };
|
||||||
|
static queue_t mic_decode_fifo;
|
||||||
|
static constexpr int MIC_DECODE_DEPTH = 8; // jitter-buffer capacity (frames)
|
||||||
|
static constexpr int MIC_PLAYOUT_START = 3; // pre-buffer before playout begins
|
||||||
|
static constexpr uint64_t MIC_FRAME_US = 10000; // 10 ms per Opus frame @ 48 kHz
|
||||||
|
static constexpr uint64_t MIC_SESSION_US = 300000; // no real frame this long → stop playout
|
||||||
|
static bool mic_playout_started = false;
|
||||||
|
static uint64_t mic_next_playout_us = 0;
|
||||||
|
static uint64_t mic_last_real_us = 0;
|
||||||
|
static volatile uint32_t g_mic_plc_frames = 0; // concealed frames generated (Diag)
|
||||||
|
uint32_t audio_mic_plc_frames() { return g_mic_plc_frames; }
|
||||||
|
|
||||||
struct audio_raw_element {
|
struct audio_raw_element {
|
||||||
float data[512 * 2];
|
float data[512 * 2];
|
||||||
};
|
};
|
||||||
@@ -161,23 +179,61 @@ void audio_loop() {
|
|||||||
// push to the host via tud_audio_write. Runs once per loop iteration so
|
// push to the host via tud_audio_write. Runs once per loop iteration so
|
||||||
// it keeps up with the ~100 Hz arrival rate of mic-tagged BT frames.
|
// it keeps up with the ~100 Hz arrival rate of mic-tagged BT frames.
|
||||||
if (mic_decoder != nullptr) {
|
if (mic_decoder != nullptr) {
|
||||||
static mic_element packet{};
|
const uint64_t now = time_us_64();
|
||||||
if (queue_try_remove(&mic_fifo, &packet)) {
|
|
||||||
static int16_t mono[MIC_FRAMES];
|
// Decode stage: drain incoming Opus into the jitter buffer as fast as it
|
||||||
const int decoded = opus_decode(mic_decoder, packet.data,
|
// arrives (absorbs bursty BT delivery), up to the buffer's capacity.
|
||||||
MIC_OPUS_SIZE, mono, MIC_FRAMES, 0);
|
static mic_element pkt{};
|
||||||
g_mic_last_decoded = decoded; // observed in OLED Diag
|
while (queue_get_level(&mic_decode_fifo) < MIC_DECODE_DEPTH
|
||||||
if (decoded > 0) {
|
&& queue_try_remove(&mic_fifo, &pkt)) {
|
||||||
static int16_t stereo[MIC_FRAMES * 2];
|
static mic_decoded_element dec{};
|
||||||
for (int i = 0; i < decoded; i++) {
|
const int n = opus_decode(mic_decoder, pkt.data, MIC_OPUS_SIZE,
|
||||||
stereo[i * 2] = mono[i];
|
dec.mono, MIC_FRAMES, 0);
|
||||||
stereo[i * 2 + 1] = mono[i];
|
g_mic_last_decoded = n; // observed in OLED Diag
|
||||||
|
if (n > 0) {
|
||||||
|
queue_try_add(&mic_decode_fifo, &dec);
|
||||||
|
mic_last_real_us = now;
|
||||||
}
|
}
|
||||||
const uint16_t want = (uint16_t)(decoded * 2 * sizeof(int16_t));
|
}
|
||||||
const uint16_t wrote = tud_audio_write(stereo, want);
|
|
||||||
|
// Playout stage: emit one frame every 10 ms. Pre-buffer a few frames to
|
||||||
|
// absorb jitter, then play a real frame if buffered, else conceal with an
|
||||||
|
// Opus PLC frame during an active session (transient loss) so the host
|
||||||
|
// hears continuity instead of a hole. If real frames have been gone for a
|
||||||
|
// while (mic off/idle), stop so we don't emit comfort noise forever.
|
||||||
|
if (!mic_playout_started
|
||||||
|
&& queue_get_level(&mic_decode_fifo) >= MIC_PLAYOUT_START) {
|
||||||
|
mic_playout_started = true;
|
||||||
|
mic_next_playout_us = now;
|
||||||
|
}
|
||||||
|
if (mic_playout_started && (int64_t)(now - mic_next_playout_us) >= 0) {
|
||||||
|
static mic_decoded_element out{};
|
||||||
|
bool have = queue_try_remove(&mic_decode_fifo, &out);
|
||||||
|
if (!have) {
|
||||||
|
if (now - mic_last_real_us < MIC_SESSION_US) {
|
||||||
|
const int n = opus_decode(mic_decoder, nullptr, 0,
|
||||||
|
out.mono, MIC_FRAMES, 0); // PLC
|
||||||
|
if (n > 0) { have = true; g_mic_plc_frames++; }
|
||||||
|
} else {
|
||||||
|
mic_playout_started = false; // session ended — re-buffer next time
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (have) {
|
||||||
|
static int16_t stereo[MIC_FRAMES * 2];
|
||||||
|
for (int i = 0; i < MIC_FRAMES; i++) {
|
||||||
|
stereo[i * 2] = out.mono[i];
|
||||||
|
stereo[i * 2 + 1] = out.mono[i];
|
||||||
|
}
|
||||||
|
const uint16_t want = (uint16_t)(MIC_FRAMES * 2 * sizeof(int16_t));
|
||||||
|
g_mic_last_wrote = tud_audio_write(stereo, want);
|
||||||
g_mic_last_want = want;
|
g_mic_last_want = want;
|
||||||
g_mic_last_wrote = wrote;
|
|
||||||
g_mic_frames++;
|
g_mic_frames++;
|
||||||
|
mic_next_playout_us += MIC_FRAME_US;
|
||||||
|
// Drift guard: if we've fallen many frames behind (loop stall),
|
||||||
|
// resync the cadence instead of bursting to catch up.
|
||||||
|
if ((int64_t)(now - mic_next_playout_us) > (int64_t)(4 * MIC_FRAME_US)) {
|
||||||
|
mic_next_playout_us = now + MIC_FRAME_US;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -376,7 +432,8 @@ void audio_init() {
|
|||||||
|
|
||||||
// Mic path: queue + decoder live on core0 (audio_loop), separate from
|
// Mic path: queue + decoder live on core0 (audio_loop), separate from
|
||||||
// the core1 speaker encoder. Mic Opus is mono / 48 kHz / 10 ms frames.
|
// the core1 speaker encoder. Mic Opus is mono / 48 kHz / 10 ms frames.
|
||||||
queue_init(&mic_fifo, sizeof(mic_element), 2);
|
queue_init(&mic_fifo, sizeof(mic_element), MIC_DECODE_DEPTH); // deeper: tolerate BT bursts
|
||||||
|
queue_init(&mic_decode_fifo, sizeof(mic_decoded_element), MIC_DECODE_DEPTH); // decoded-PCM jitter buffer
|
||||||
int dec_error = 0;
|
int dec_error = 0;
|
||||||
mic_decoder = opus_decoder_create(48000, MIC_CHANNELS, &dec_error);
|
mic_decoder = opus_decoder_create(48000, MIC_CHANNELS, &dec_error);
|
||||||
if (dec_error != 0 || mic_decoder == nullptr) {
|
if (dec_error != 0 || mic_decoder == nullptr) {
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ int32_t audio_mic_last_decoded(); // last opus_decode return — neg = error, 4
|
|||||||
uint16_t audio_mic_last_want(); // bytes asked of tud_audio_write
|
uint16_t audio_mic_last_want(); // bytes asked of tud_audio_write
|
||||||
uint16_t audio_mic_last_wrote(); // bytes TinyUSB FIFO actually accepted
|
uint16_t audio_mic_last_wrote(); // bytes TinyUSB FIFO actually accepted
|
||||||
uint8_t audio_mic_last_toc(); // first byte of last Opus packet (frame config)
|
uint8_t audio_mic_last_toc(); // first byte of last Opus packet (frame config)
|
||||||
|
uint32_t audio_mic_plc_frames(); // count of packet-loss-concealment frames generated
|
||||||
|
|
||||||
// Called from on_bt_data() in main.cpp when the DS5 sends a mic-tagged
|
// Called from on_bt_data() in main.cpp when the DS5 sends a mic-tagged
|
||||||
// 0x31 input report. Buffer must point at MIC_OPUS_SIZE (71) bytes of
|
// 0x31 input report. Buffer must point at MIC_OPUS_SIZE (71) bytes of
|
||||||
|
|||||||
+5
-2
@@ -827,7 +827,7 @@ void sample_diag_rates() {
|
|||||||
|
|
||||||
// Row list ordered by relevance: always-useful at top, parked-mic-investigation
|
// Row list ordered by relevance: always-useful at top, parked-mic-investigation
|
||||||
// data at bottom. To add a row, bump kNumDiagRows and add a case.
|
// data at bottom. To add a row, bump kNumDiagRows and add a case.
|
||||||
constexpr int kNumDiagRows = 11;
|
constexpr int kNumDiagRows = 12;
|
||||||
__attribute__((noinline))
|
__attribute__((noinline))
|
||||||
void format_diag_row(int idx, char* line, size_t n) {
|
void format_diag_row(int idx, char* line, size_t n) {
|
||||||
switch (idx) {
|
switch (idx) {
|
||||||
@@ -872,7 +872,10 @@ void format_diag_row(int idx, char* line, size_t n) {
|
|||||||
(long)audio_mic_last_decoded(),
|
(long)audio_mic_last_decoded(),
|
||||||
(unsigned)audio_mic_last_wrote());
|
(unsigned)audio_mic_last_wrote());
|
||||||
break;
|
break;
|
||||||
case 10: {
|
case 10:
|
||||||
|
snprintf(line, n, "Mic PLC: %lu", (unsigned long)audio_mic_plc_frames());
|
||||||
|
break;
|
||||||
|
case 11: {
|
||||||
uint8_t pfx[6]; bt_31_mic_prefix(pfx);
|
uint8_t pfx[6]; bt_31_mic_prefix(pfx);
|
||||||
snprintf(line, n, "%02X %02X %02X %02X %02X %02X",
|
snprintf(line, n, "%02X %02X %02X %02X %02X %02X",
|
||||||
pfx[0], pfx[1], pfx[2], pfx[3], pfx[4], pfx[5]);
|
pfx[0], pfx[1], pfx[2], pfx[3], pfx[4], pfx[5]);
|
||||||
|
|||||||
Reference in New Issue
Block a user