From 8c2375d402f91d3c432156c43586c1d7d43f2ff9 Mon Sep 17 00:00:00 2001
From: nick <coachward0742@gmail.com>
Date: Sun, 26 Jul 2026 15:26:42 -0600
Subject: [PATCH] dashboard: add passive CAN capture markers

---
 firmware/esp32-s3-dashboard/README.md         |  40 ++++-
 .../esp32-s3-dashboard/dashboard_config.h     |   2 +-
 .../esp32-s3-dashboard/esp32-s3-dashboard.ino | 156 ++++++++++++------
 run_tests.py                                  |   1 +
 tests/test_dashboard_source_contract.py       |  12 ++
 tools/analyze_can_coolant.py                  |  72 ++++++++
 6 files changed, 227 insertions(+), 56 deletions(-)
 create mode 100644 tools/analyze_can_coolant.py

diff --git a/firmware/esp32-s3-dashboard/README.md b/firmware/esp32-s3-dashboard/README.md
index 6c9678b..f50ffb1 100644
--- a/firmware/esp32-s3-dashboard/README.md
+++ b/firmware/esp32-s3-dashboard/README.md
@@ -76,13 +76,37 @@ The dashboard uses field-filtered `/api/v1/status` calls:
 
 This keeps the endpoint model simple while reducing payload size, JSON parsing, and UI churn.
 
-## OBD-II / CAN Diagnostics
+## Passive CAN Logger
 
-The hidden diagnostics overlay distinguishes raw CAN traffic from parsed
-OBD-II coolant responses. It reports request successes and failures, total
-raw frames received, the last frame identifier and DLC, TWAI queue depth,
-controller error counters, missed or overrun frames, arbitration losses,
-and bus errors.
+The dashboard runs its 500 kbps TWAI controller in listen-only mode and never
+transmits onto the vehicle bus. At 115200 baud it prints changed frames as:
 
-Raw RX activity confirms that the display is receiving CAN frames even when
-the vehicle does not return a matching Mode 01 PID 05 coolant response.
+    CAN,ms,id,format,dlc,b0,b1,b2,b3,b4,b5,b6,b7
+
+During a cold-start warm-up, enter the scan-tool coolant reading periodically:
+
+    TEMP,68
+    TEMP,95
+    TEMP,140
+
+Generic timestamped markers can be added with `MARK,<label>`. For a stationary
+4WD capture, start in 2H and enter each marker only after the corresponding
+dashboard indicator has settled:
+
+    MARK,2H
+    MARK,4H
+    MARK,4LO
+    MARK,4H
+    MARK,2H
+
+Wait 10-15 seconds between mode changes. Follow the vehicle owner's manual for
+the conditions required to select 4LO, keep the vehicle stationary where
+required, and save the complete serial output. The repeated return transitions
+help distinguish an actual mode signal from unrelated counters and timers.
+
+Save the serial output, then rank CAN ID/byte candidates with:
+
+    python tools/analyze_can_coolant.py can-warmup.log
+
+Verify strong candidates over multiple warm-up captures before using one for
+the dashboard gauge.
diff --git a/firmware/esp32-s3-dashboard/dashboard_config.h b/firmware/esp32-s3-dashboard/dashboard_config.h
index 1d112a3..58ef5b7 100644
--- a/firmware/esp32-s3-dashboard/dashboard_config.h
+++ b/firmware/esp32-s3-dashboard/dashboard_config.h
@@ -1,6 +1,6 @@
 #pragma once
 
-static const char *DASHBOARD_VERSION = "0.1.127-first-boot-wifi";
+static const char *DASHBOARD_VERSION = "0.1.129-can-markers";
 
 // First-boot defaults. Credentials are stored in Preferences/NVS after setup.
 static const char *DASHBOARD_DEFAULT_CARGO_WIFI_SSID = "OverlandController";
diff --git a/firmware/esp32-s3-dashboard/esp32-s3-dashboard.ino b/firmware/esp32-s3-dashboard/esp32-s3-dashboard.ino
index 76f17dc..aa75ccf 100644
--- a/firmware/esp32-s3-dashboard/esp32-s3-dashboard.ino
+++ b/firmware/esp32-s3-dashboard/esp32-s3-dashboard.ino
@@ -40,6 +40,20 @@ static constexpr uint32_t OBD_REQUEST_ID = 0x7DF;
 static constexpr uint32_t OBD_RESPONSE_ID_MIN = 0x7E8;
 static constexpr uint32_t OBD_RESPONSE_ID_MAX = 0x7EF;
 static constexpr uint8_t OBD_PID_COOLANT_TEMP = 0x05;
+static constexpr size_t CAN_TRACKED_ID_COUNT = 96;
+static constexpr unsigned long CAN_LOG_MIN_INTERVAL_MS = 100;
+
+struct CanTrackedFrame {
+    bool used = false;
+    bool extended = false;
+    uint32_t identifier = 0;
+    uint8_t dlc = 0;
+    uint8_t data[8] = {};
+    unsigned long last_log_ms = 0;
+};
+
+static CanTrackedFrame can_tracked_frames[CAN_TRACKED_ID_COUNT];
+static String can_serial_command;
 
 static bool obd_can_ready = false;
 static bool obd_can_started = false;
@@ -444,9 +458,9 @@ static bool init_obd_can()
         return true;
     }
 
-    twai_general_config_t general_config = TWAI_GENERAL_CONFIG_DEFAULT(OBD_CAN_TX_PIN, OBD_CAN_RX_PIN, TWAI_MODE_NORMAL);
-    general_config.tx_queue_len = 5;
-    general_config.rx_queue_len = 20;
+    twai_general_config_t general_config = TWAI_GENERAL_CONFIG_DEFAULT(OBD_CAN_TX_PIN, OBD_CAN_RX_PIN, TWAI_MODE_LISTEN_ONLY);
+    general_config.tx_queue_len = 0;
+    general_config.rx_queue_len = 64;
 
     twai_timing_config_t timing_config = TWAI_TIMING_CONFIG_500KBITS();
     twai_filter_config_t filter_config = TWAI_FILTER_CONFIG_ACCEPT_ALL();
@@ -471,37 +485,12 @@ static bool init_obd_can()
 
     obd_can_started = true;
     obd_last_error = "started";
-    Serial.println("OBD CAN: TWAI started on TX GPIO15 / RX GPIO16 at 500 kbps");
+    Serial.println("OBD CAN: passive logger started on RX GPIO16 at 500 kbps");
+    Serial.println("CAN CSV: CAN,ms,id,format,dlc,b0,b1,b2,b3,b4,b5,b6,b7");
+    Serial.println("Enter TEMP,<degrees F> when recording a scan-tool coolant reading.");
     return true;
 }
 
-static void send_obd_coolant_request()
-{
-    twai_message_t message = {};
-    message.identifier = OBD_REQUEST_ID;
-    message.extd = 0;
-    message.rtr = 0;
-    message.data_length_code = 8;
-    message.data[0] = 0x02;
-    message.data[1] = 0x01;
-    message.data[2] = OBD_PID_COOLANT_TEMP;
-    message.data[3] = 0x55;
-    message.data[4] = 0x55;
-    message.data[5] = 0x55;
-    message.data[6] = 0x55;
-    message.data[7] = 0x55;
-
-    esp_err_t result = twai_transmit(&message, 0);
-    if (result == ESP_OK) {
-        obd_tx_count++;
-        last_obd_tx_ms = millis();
-        obd_last_error = "coolant request sent";
-    } else {
-        obd_tx_fail_count++;
-        obd_last_error = "tx failed: " + String((int)result);
-    }
-}
-
 static void handle_obd_can_message(const twai_message_t &message)
 {
     if (message.extd || message.rtr) {
@@ -533,16 +522,97 @@ static void handle_obd_can_message(const twai_message_t &message)
     }
 }
 
+static CanTrackedFrame *find_can_tracked_frame(const twai_message_t &message)
+{
+    CanTrackedFrame *free_slot = nullptr;
+    for (size_t i = 0; i < CAN_TRACKED_ID_COUNT; i++) {
+        CanTrackedFrame &tracked = can_tracked_frames[i];
+        if (tracked.used && tracked.identifier == message.identifier && tracked.extended == message.extd) {
+            return &tracked;
+        }
+        if (!tracked.used && free_slot == nullptr) free_slot = &tracked;
+    }
+    return free_slot;
+}
+
+static void log_changed_can_frame(const twai_message_t &message)
+{
+    if (message.rtr) return;
+
+    CanTrackedFrame *tracked = find_can_tracked_frame(message);
+    if (tracked == nullptr) return;
+
+    bool changed = !tracked->used || tracked->dlc != message.data_length_code;
+    for (uint8_t i = 0; i < message.data_length_code && i < 8; i++) {
+        changed = changed || tracked->data[i] != message.data[i];
+    }
+
+    unsigned long now = millis();
+    bool should_log = changed && (!tracked->used || now - tracked->last_log_ms >= CAN_LOG_MIN_INTERVAL_MS);
+    tracked->used = true;
+    tracked->extended = message.extd;
+    tracked->identifier = message.identifier;
+    tracked->dlc = message.data_length_code;
+    memcpy(tracked->data, message.data, sizeof(tracked->data));
+
+    if (!should_log) return;
+
+    tracked->last_log_ms = now;
+    Serial.print("CAN,");
+    Serial.print(now);
+    Serial.print(",");
+    Serial.print(message.identifier, HEX);
+    Serial.print(message.extd ? ",EXT," : ",STD,");
+    Serial.print(message.data_length_code);
+    for (uint8_t i = 0; i < 8; i++) {
+        Serial.print(",");
+        if (i < message.data_length_code) {
+            if (message.data[i] < 0x10) Serial.print("0");
+            Serial.print(message.data[i], HEX);
+        }
+    }
+    Serial.println();
+}
+
+static void process_can_serial_commands()
+{
+    while (Serial.available() > 0) {
+        char value = (char)Serial.read();
+        if (value == '\r') continue;
+        if (value != '\n') {
+            if (can_serial_command.length() < 48) can_serial_command += value;
+            continue;
+        }
+
+        can_serial_command.trim();
+        if (can_serial_command.startsWith("TEMP,")) {
+            String temperature = can_serial_command.substring(5);
+            temperature.trim();
+            Serial.print("TEMP,");
+            Serial.print(millis());
+            Serial.print(",");
+            Serial.println(temperature);
+        } else if (can_serial_command.startsWith("MARK,")) {
+            String label = can_serial_command.substring(5);
+            label.trim();
+            Serial.print("MARK,");
+            Serial.print(millis());
+            Serial.print(",");
+            Serial.println(label);
+        } else if (can_serial_command.length() > 0) {
+            Serial.println("ERROR,expected TEMP,<degrees F> or MARK,<label>");
+        }
+        can_serial_command = "";
+    }
+}
+
 static void process_obd_can()
 {
     if (!init_obd_can()) {
         return;
     }
 
-    if (millis() - last_obd_poll_ms >= 1000) {
-        last_obd_poll_ms = millis();
-        send_obd_coolant_request();
-    }
+    process_can_serial_commands();
 
     twai_message_t message = {};
     while (twai_receive(&message, 0) == ESP_OK) {
@@ -552,6 +622,7 @@ static void process_obd_can()
         obd_last_rx_extended = message.extd;
         last_obd_raw_rx_ms = millis();
 
+        log_changed_can_frame(message);
         handle_obd_can_message(message);
     }
 
@@ -635,23 +706,14 @@ static void update_diagnostics_overlay()
     text += String(status_model.cell_delta_mv);
     text += " mV";
 
-    text += "\n\nOBD-II / CAN\n";
+    text += "\n\nPassive CAN Logger\n";
     text += "  TWAI: ";
     text += obd_can_started ? "started" : "offline";
     text += " @ 500 kbps";
     text += "\n  Pins: TX GPIO15 / RX GPIO16";
-    text += "\n  Coolant: ";
-    text += obd_coolant_temp_f > -100 ? String(obd_coolant_temp_f) + "°F" : String("--");
-
-    text += "\n  Requests OK/fail: ";
-    text += String(obd_tx_count);
-    text += "/";
-    text += String(obd_tx_fail_count);
-
-    text += "\n  Raw RX/coolant RX: ";
+    text += "\n  Mode: listen-only (no transmit)";
+    text += "\n  Raw RX: ";
     text += String(obd_raw_rx_count);
-    text += "/";
-    text += String(obd_rx_count);
 
     text += "\n  Last frame: ";
     if (obd_raw_rx_count > 0) {
diff --git a/run_tests.py b/run_tests.py
index 4f07906..d2602a0 100644
--- a/run_tests.py
+++ b/run_tests.py
@@ -5,6 +5,7 @@ import sys
 COMMANDS = [
     [sys.executable, "-m", "pytest", "tests/test_http_api_contract.py"],
     [sys.executable, "-m", "pytest", "tests/test_xiao_sensor_node_contract.py"],
+    [sys.executable, "-m", "pytest", "tests/test_dashboard_source_contract.py"],
 ]
 
 
diff --git a/tests/test_dashboard_source_contract.py b/tests/test_dashboard_source_contract.py
index 6fc20f0..f7dc8a7 100644
--- a/tests/test_dashboard_source_contract.py
+++ b/tests/test_dashboard_source_contract.py
@@ -15,3 +15,15 @@ def test_setup_done_button_event_is_handled():
     assert "LV_EVENT_CLICKED" in callback
     assert "lv_obj_add_flag(setup_keyboard, LV_OBJ_FLAG_HIDDEN);" in callback
     assert "lv_obj_add_event_cb(done_button, setup_keyboard_event_cb, LV_EVENT_CLICKED" in source
+
+
+def test_dashboard_can_is_passive_and_logs_changed_frames():
+    source = DASHBOARD.read_text()
+
+    assert "TWAI_MODE_LISTEN_ONLY" in source
+    assert "twai_transmit(" not in source
+    assert "log_changed_can_frame(message);" in source
+    assert 'Serial.print("CAN,");' in source
+    assert 'can_serial_command.startsWith("TEMP,")' in source
+    assert 'can_serial_command.startsWith("MARK,")' in source
+    assert 'Serial.print("MARK,");' in source
diff --git a/tools/analyze_can_coolant.py b/tools/analyze_can_coolant.py
new file mode 100644
index 0000000..48166a2
--- /dev/null
+++ b/tools/analyze_can_coolant.py
@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+"""Rank passive CAN bytes against timestamped scan-tool coolant readings."""
+
+import argparse
+import math
+from collections import defaultdict
+from pathlib import Path
+
+
+def correlation(xs, ys):
+    if len(xs) < 3 or len(set(xs)) < 2 or len(set(ys)) < 2:
+        return 0.0
+    xm, ym = sum(xs) / len(xs), sum(ys) / len(ys)
+    numerator = sum((x - xm) * (y - ym) for x, y in zip(xs, ys))
+    scale = math.sqrt(sum((x - xm) ** 2 for x in xs) * sum((y - ym) ** 2 for y in ys))
+    return numerator / scale if scale else 0.0
+
+
+def parse_capture(path):
+    frames = defaultdict(list)
+    temperatures = []
+    for line in path.read_text(errors="replace").splitlines():
+        parts = [part.strip() for part in line.split(",")]
+        try:
+            if len(parts) >= 3 and parts[0] == "TEMP":
+                temperatures.append((int(parts[1]), float(parts[2])))
+            elif len(parts) >= 6 and parts[0] == "CAN":
+                timestamp, can_id, dlc = int(parts[1]), parts[2].upper(), int(parts[4])
+                for index, value in enumerate(parts[5 : 5 + min(dlc, 8)]):
+                    if value:
+                        frames[(can_id, index)].append((timestamp, int(value, 16)))
+        except ValueError:
+            continue
+    return frames, sorted(temperatures)
+
+
+def value_at(samples, timestamp):
+    latest = None
+    for sample_time, value in samples:
+        if sample_time > timestamp:
+            break
+        latest = value
+    return latest
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Rank CAN bytes against TEMP markers.")
+    parser.add_argument("capture", type=Path)
+    parser.add_argument("--limit", type=int, default=20)
+    args = parser.parse_args()
+    frames, temperatures = parse_capture(args.capture)
+    if len(temperatures) < 3:
+        parser.error("capture needs at least three TEMP markers")
+
+    ranked = []
+    for (can_id, byte_index), samples in frames.items():
+        pairs = [(value_at(samples, timestamp), temp) for timestamp, temp in temperatures]
+        pairs = [(value, temp) for value, temp in pairs if value is not None]
+        values, temps = [p[0] for p in pairs], [p[1] for p in pairs]
+        score = correlation(values, temps)
+        if len(values) >= 3 and len(set(values)) >= 2:
+            ranked.append((abs(score), score, can_id, byte_index, len(values), min(values), max(values)))
+
+    ranked.sort(reverse=True)
+    print("rank  id     byte  corr     samples  raw-range")
+    for rank, (_, score, can_id, byte_index, count, low, high) in enumerate(ranked[:args.limit], 1):
+        print(f"{rank:>4}  0x{can_id:<4}  {byte_index:>4}  {score:>+7.3f}  {count:>7}  {low:>3}..{high:<3}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())