components/st3m/st3m_gfx.h

@@ -7,45 +7,109 @@
 #include "ctx.h"
 // clang-format on
 
-// Each buffer  takes ~116kB SPIRAM. While one framebuffer is being blitted, the
-// other one is being written to by the rasterizer.
-#define ST3M_GFX_NBUFFERS 2
-// More ctx drawlists than buffers so that micropython doesn't get starved when
-// pipeline runs in lockstep.
-#define ST3M_GFX_NCTX 2
-
-// A framebuffer descriptor, pointing at a framebuffer.
-typedef struct {
-    // The numeric ID of this descriptor.
-    int num;
-    // SPIRAM buffer.
-    uint16_t buffer[240 * 240];
-    Ctx *ctx;
-} st3m_framebuffer_desc_t;
+typedef enum {
+    st3m_gfx_default = 0,
+    // bitmask flag over base bpp to turn on OSD, only 16bpp for now will
+    // become available for other bitdepths as grayscale rather than color
+    // overlays.
+
+    // lock the graphics mode, this makes st3m_gfx_set_mode() a no-op
+    st3m_gfx_lock = 1 << 8,
+
+    // directly manipulate target framebuffer instead of having
+    // separate rasterization task - this causes the rasterization overhead
+    // to occur in the micropython task rather than the graphics rasterization
+    // task.
+    st3m_gfx_direct_ctx = 1 << 9,
+
+    // enable osd compositing, for a small performance boost
+    st3m_gfx_osd = 1 << 10,
+
+    // shallower pipeline, prioritize short time from drawing until shown on
+    // screen over frame rate
+    st3m_gfx_low_latency = 1 << 11,
+
+    // boost FPS by always reporting readiness for drawing, this gets disabled
+    // dynamically if FPS falls <13fps
+    st3m_gfx_EXPERIMENTAL_think_per_draw = 1 << 12,
+
+    // pixel-doubling
+    st3m_gfx_2x = 1 << 13,
+    st3m_gfx_3x = 1 << 14,
+    st3m_gfx_4x = st3m_gfx_2x | st3m_gfx_3x,
+
+    // keep track of what is drawn and only redraw the bounding box
+    st3m_gfx_smart_redraw = 1 << 15,
+
+    // 4 and 8bpp modes use the configured palette, the palette resides
+    // in video ram and is lost upon mode change
+    st3m_gfx_1bpp = 1,
+    st3m_gfx_2bpp = 2,
+    st3m_gfx_4bpp = 4,
+    st3m_gfx_8bpp = 8,    // bare 8bpp mode is grayscale
+    st3m_gfx_rgb332 = 9,  // variant of 8bpp mode using RGB
+    st3m_gfx_sepia = 10,  // grayscale rendering with sepia palette
+    st3m_gfx_cool = 11,   // grayscale rendering with cool palette
+    st3m_gfx_palette = 15,
+    // 16bpp modes have the lowest blit overhead - no osd for now
+    st3m_gfx_16bpp = 16,
+    st3m_gfx_24bpp = 24,
+    // 32bpp modes - are slightly faster at doing compositing, but the memory
+    // overhead of higher bitdepths cancels out the overhead of converting back
+    // and forth between rgb565 and RGBA8, 24 and 32bit modes are here
+    // mostly because it leads to nicer math in python.
+    st3m_gfx_32bpp = 32,
+} st3m_gfx_mode;
+
+// sets the system graphics mode, this is the mode you get to
+// when calling st3m_gfx_set_mode(st3m_gfx_default);
+void st3m_gfx_set_default_mode(st3m_gfx_mode mode);
+
+// sets the current graphics mode
+st3m_gfx_mode st3m_gfx_set_mode(st3m_gfx_mode mode);
+
+// gets the current graphics mode
+st3m_gfx_mode st3m_gfx_get_mode(void);
+
+// returns a ctx for drawing at the specified mode/target
+// should be paired with a st3m_ctx_end_frame
+// normal values are st3m_gfx_default and st3m_gfx_osd for base framebuffer
+// and overlay drawing context.
+Ctx *st3m_gfx_ctx(st3m_gfx_mode mode);
+
+// get the framebuffer associated with graphics mode
+// if you ask for st3m_gfx_default you get the current modes fb
+// and if you ask for st3m_gfx_osd you get the current modes overlay fb
+uint8_t *st3m_gfx_fb(st3m_gfx_mode mode, int *width, int *height, int *stride);
+
+// get the bits per pixel for a given mode
+int st3m_gfx_bpp(st3m_gfx_mode mode);
+
+// sets the palette, pal_in is an array with 3 uint8_t's per entry,
+// support values for count is 1-256, used only in 4bpp and 8bpp
+// graphics modes.
+void st3m_gfx_set_palette(uint8_t *pal_in, int count);
+
+// specifies the corners of the clipping rectangle
+// for compositing overlay
+void st3m_gfx_overlay_clip(int x0, int y0, int x1, int y1);
+
+// returns a running average of fps
+float st3m_gfx_fps(void);
+
+// temporary, signature compatible
+// with ctx_end_frame()
+void st3m_gfx_end_frame(Ctx *ctx);
 
 // Initialize the gfx subsystem of st3m, includng the rasterization and
 // crtx/blitter pipeline.
 void st3m_gfx_init(void);
 
-// A drawlist ctx descriptor, pointing to a drawlist-backed Ctx.
-typedef struct {
-    // The numeric ID of this descriptor.
-    int num;
-    Ctx *ctx;
-} st3m_ctx_desc_t;
+// Returns true if we right now cannot accept another frame
+uint8_t st3m_gfx_pipe_full(void);
 
-// Get a free drawlist ctx to draw into.
-//
-// ticks_to_wait can be used to limit the time to wait for a free ctx
-// descriptor, or portDELAY_MAX can be specified to wait forever. If the timeout
-// expires, NULL will be returned.
-st3m_ctx_desc_t *st3m_gfx_drawctx_free_get(TickType_t ticks_to_wait);
-
-// Submit a filled ctx descriptor to the rasterization pipeline.
-void st3m_gfx_drawctx_pipe_put(st3m_ctx_desc_t *desc);
-
-// Returns true if the rasterizaiton pipeline submission would block.
-uint8_t st3m_gfx_drawctx_pipe_full(void);
+// Returns true if there's a free drawlist available to retrieve
+uint8_t st3m_gfx_pipe_available(void);
 
 // Flush any in-flight pipelined work, resetting the free ctx/framebuffer queues
 // to their initial state. This should be called if there has been any drawlist
@@ -53,8 +117,8 @@ uint8_t st3m_gfx_drawctx_pipe_full(void);
 // wasn't, for exaple if Micropython restarted).
 //
 // This causes a graphical disturbance and shouldn't be called during normal
-// operation.
-void st3m_gfx_flush(void);
+// operation. wait_ms is waited for drawlits to clear.
+void st3m_gfx_flush(int wait_ms);
 
 typedef struct {
     const char *title;
@@ -75,3 +139,15 @@ void st3m_gfx_splash(const char *text);
 // Draw the flow3r multi-coloured logo at coordinates x,y and with given
 // dimension (approx. bounding box size).
 void st3m_gfx_flow3r_logo(Ctx *ctx, float x, float y, float dim);
+
+// configure virtual viewport, the default values are 0, 0, 240, 240 which
+// also gives room for a copy of the fb for separate rasterize/blit in 16bpp
+//
+// with, height: width and height of virtual framebuffer
+//
+// changing the viewport should be done after setting the graphics mode - upon
+// graphics mode setting viewport is reset to 240, 240, 0,0,0,0
+void st3m_gfx_fbconfig(int width, int height, int blit_x, int blit_y);
+
+// get fbconfig values, arguments passed NULL ptrs are ignored.
+void st3m_gfx_get_fbconfig(int *width, int *height, int *blit_x, int *blit_y);

components/st3m/st3m_io.c

@@ -5,36 +5,71 @@ static const char *TAG = "st3m-io";
 #include "esp_err.h"
 #include "esp_log.h"
 #include "freertos/FreeRTOS.h"
+#include "freertos/semphr.h"
 #include "freertos/task.h"
 
 #include "flow3r_bsp.h"
 #include "flow3r_bsp_i2c.h"
 #include "st3m_audio.h"
 
+static bool _app_button_left = true;
+static SemaphoreHandle_t _mu = NULL;
+
+static st3m_tripos _left_button_state;
+static st3m_tripos _right_button_state;
+
 static void _update_button_state() {
     esp_err_t ret = flow3r_bsp_spio_update();
     if (ret != ESP_OK) {
         ESP_LOGE(TAG, "update failed: %s", esp_err_to_name(ret));
     }
+    st3m_tripos state = flow3r_bsp_spio_left_button_get();
+    if (state) {
+        _left_button_state = state;
     }
-
-void init_buttons() {
-    esp_err_t ret = flow3r_bsp_spio_init();
-    if (ret != ESP_OK) {
-        ESP_LOGE(TAG, "init failed: %s", esp_err_to_name(ret));
-        for (;;) {
-        }
+    state = flow3r_bsp_spio_right_button_get();
+    if (state) {
+        _right_button_state = state;
     }
 }
 
 bool st3m_io_charger_state_get() { return flow3r_bsp_spio_charger_state_get(); }
 
-st3m_tripos st3m_io_left_button_get() {
-    return flow3r_bsp_spio_left_button_get();
+void st3m_io_app_button_configure(bool left) {
+    xSemaphoreTake(_mu, portMAX_DELAY);
+    _app_button_left = left;
+    xSemaphoreGive(_mu);
 }
 
-st3m_tripos st3m_io_right_button_get() {
-    return flow3r_bsp_spio_right_button_get();
+bool st3m_io_app_button_is_left(void) {
+    xSemaphoreTake(_mu, portMAX_DELAY);
+    bool res = _app_button_left;
+    xSemaphoreGive(_mu);
+    return res;
+}
+
+st3m_tripos st3m_io_app_button_get() {
+    if (st3m_io_app_button_is_left()) {
+        st3m_tripos state = _left_button_state;
+        _left_button_state = flow3r_bsp_spio_left_button_get();
+        return state;
+    } else {
+        st3m_tripos state = _right_button_state;
+        _right_button_state = flow3r_bsp_spio_right_button_get();
+        return state;
+    }
+}
+
+st3m_tripos st3m_io_os_button_get() {
+    if (st3m_io_app_button_is_left()) {
+        st3m_tripos state = _right_button_state;
+        _right_button_state = flow3r_bsp_spio_right_button_get();
+        return state;
+    } else {
+        st3m_tripos state = _left_button_state;
+        _left_button_state = flow3r_bsp_spio_left_button_get();
+        return state;
+    }
 }
 
 static uint8_t badge_link_enabled = 0;
@@ -102,6 +137,10 @@ static void _task(void *data) {
 }
 
 void st3m_io_init(void) {
+    assert(_mu == NULL);
+    _mu = xSemaphoreCreateMutex();
+    assert(_mu != NULL);
+
     esp_err_t ret = flow3r_bsp_spio_init();
     if (ret != ESP_OK) {
         ESP_LOGE(TAG, "spio init failed: %s", esp_err_to_name(ret));

components/st3m/st3m_io.h

@@ -19,11 +19,20 @@ typedef enum {
     st3m_tripos_right = 1,
 } st3m_tripos;
 
-/* Read the state of the left/right button.
- * This ignores user preference and should be used only with good reason.
+/* Configure whether the app button is on the left (default) or on the right.
  */
-st3m_tripos st3m_io_left_button_get();
-st3m_tripos st3m_io_right_button_get();
+void st3m_io_app_button_configure(bool left);
+
+/* Returns true if the app button is on the left (default), false otherwise.
+ */
+bool st3m_io_app_button_is_left(void);
+
+/* Read the state of the application and OS buttons. By default, the application
+ * button is on the left and the OS button is on the right. However, the user
+ * can change that preference - see st3m_io_app_button_configure.
+ */
+st3m_tripos st3m_io_app_button_get();
+st3m_tripos st3m_io_os_button_get();
 
 #define BADGE_LINK_PIN_MASK_LINE_IN_TIP 0b0001
 #define BADGE_LINK_PIN_MASK_LINE_IN_RING 0b0010
@@ -58,3 +67,7 @@ uint8_t st3m_io_badge_link_disable(uint8_t pin_mask);
  * a while. Warn user.
  */
 uint8_t st3m_io_badge_link_enable(uint8_t pin_mask);
+
+/* Returns true if the battery is currently being charged.
+ */
+bool st3m_io_charger_state_get();
\ No newline at end of file

components/st3m/st3m_leds.c

@@ -14,24 +14,43 @@
 #include "flow3r_bsp.h"
 #include "st3m_colors.h"
 
+#define TAU360 0.017453292519943295
+
+#define MAX(a, b) (((a) > (b)) ? (a) : (b))
+
 static const char *TAG = "st3m-leds";
 
 typedef struct {
     uint8_t lut[256];
 } st3m_leds_gamma_table_t;
 
+typedef struct {
+    uint16_t r;
+    uint16_t g;
+    uint16_t b;
+} st3m_u16_rgb_t;
+
+typedef struct {
+    uint8_t r;
+    uint8_t g;
+    uint8_t b;
+} st3m_u8_rgb_t;
+
 typedef struct {
     uint8_t brightness;
     uint8_t slew_rate;
     bool auto_update;
+    bool is_steady;
+    uint8_t timer;
 
     st3m_leds_gamma_table_t gamma_red;
     st3m_leds_gamma_table_t gamma_green;
     st3m_leds_gamma_table_t gamma_blue;
 
-    st3m_rgb_t target[40];
-    st3m_rgb_t target_buffer[40];
-    st3m_rgb_t hardware_value[40];
+    st3m_u8_rgb_t target[40];
+    st3m_u8_rgb_t target_buffer[40];
+    st3m_u16_rgb_t slew_output[40];
+    st3m_u8_rgb_t ret_prev[40];
 } st3m_leds_state_t;
 
 static st3m_leds_state_t state;
@@ -44,12 +63,22 @@ SemaphoreHandle_t mutex_incoming;
 #define LOCK_INCOMING xSemaphoreTake(mutex_incoming, portMAX_DELAY)
 #define UNLOCK_INCOMING xSemaphoreGive(mutex_incoming)
 
-static void set_single_led(uint8_t index, st3m_rgb_t c) {
+static void set_single_led(uint8_t index, st3m_u8_rgb_t c) {
     index = (index + 29) % 40;
     flow3r_bsp_leds_set_pixel(index, c.r, c.g, c.b);
 }
 
-uint8_t led_get_slew(int16_t old, int16_t new, int16_t slew) {
+static uint16_t led_get_slew(uint16_t old, uint16_t new, uint16_t slew,
+                             uint16_t factor) {
+    new = new << 8;
+    if (slew == 255 || (old == new)) return new;
+    int16_t bonus = ((int16_t)slew) - 225;
+    slew = 30 + (slew << 2) + ((slew * slew) >> 3);
+    if (bonus > 0) {
+        slew += 62 * bonus * bonus;
+    }
+    slew = ((uint32_t)slew * factor) >> 8;
+
     if (new > old + slew) {
         return old + slew;
     } else if (new > old) {
@@ -80,29 +109,67 @@ void st3m_leds_update_hardware() {
 
     if (state.auto_update) leds_update_target();
 
+    bool is_different = false;
+    bool is_steady = true;
+
     for (int i = 0; i < 40; i++) {
-        st3m_rgb_t c = state.target[i];
-        c.r = c.r * state.brightness / 255;
-        c.g = c.g * state.brightness / 255;
-        c.b = c.b * state.brightness / 255;
+        st3m_u8_rgb_t ret = state.target[i];
+        st3m_u16_rgb_t prev = state.slew_output[i];
+        st3m_u16_rgb_t c = prev;
+        uint16_t diff_r = abs((int32_t)prev.r - (ret.r << 8u));
+        uint16_t diff_g = abs((int32_t)prev.g - (ret.g << 8u));
+        uint16_t diff_b = abs((int32_t)prev.b - (ret.b << 8u));
+        uint16_t max_diff = MAX(MAX(diff_r, diff_g), diff_b);
+        if (max_diff) {
+            c.r = led_get_slew(prev.r, ret.r, state.slew_rate,
+                               ((uint32_t)diff_r * 256) / max_diff);
+            c.g = led_get_slew(prev.g, ret.g, state.slew_rate,
+                               ((uint32_t)diff_g * 256) / max_diff);
+            c.b = led_get_slew(prev.b, ret.b, state.slew_rate,
+                               ((uint32_t)diff_b * 256) / max_diff);
+            is_steady = false;
+        }
+        state.slew_output[i] = c;
 
-        c.r = state.gamma_red.lut[c.r];
-        c.g = state.gamma_red.lut[c.g];
-        c.b = state.gamma_red.lut[c.b];
+        c.r = ((uint32_t)c.r * state.brightness) >> 8;
+        c.g = ((uint32_t)c.g * state.brightness) >> 8;
+        c.b = ((uint32_t)c.b * state.brightness) >> 8;
 
-        c.r = led_get_slew(state.hardware_value[i].r, c.r, state.slew_rate);
-        c.g = led_get_slew(state.hardware_value[i].g, c.g, state.slew_rate);
-        c.b = led_get_slew(state.hardware_value[i].b, c.b, state.slew_rate);
-        state.hardware_value[i] = c;
+        ret.r = state.gamma_red.lut[c.r >> 8];
+        ret.g = state.gamma_green.lut[c.g >> 8];
+        ret.b = state.gamma_blue.lut[c.b >> 8];
 
-        set_single_led(i, c);
+        if ((ret.r != state.ret_prev[i].r) || (ret.g != state.ret_prev[i].g) ||
+            (ret.b != state.ret_prev[i].b)) {
+            set_single_led(i, ret);
+            state.ret_prev[i] = ret;
+            is_different = true;
+        }
     }
+    state.is_steady = is_steady;
     UNLOCK;
 
+    if (is_different || (state.timer > 10)) {
         esp_err_t ret = flow3r_bsp_leds_refresh(portMAX_DELAY);
         if (ret != ESP_OK) {
             ESP_LOGE(TAG, "LED refresh failed: %s", esp_err_to_name(ret));
         }
+        state.timer = 0;
+    } else {
+        state.timer += 1;
+    }
+}
+
+void st3m_leds_set_single_rgba(uint8_t index, float red, float green,
+                               float blue, float alpha) {
+    float r, g, b;
+    st3m_leds_get_single_rgb(index, &r, &g, &b);
+
+    float nalpha = 1 - alpha;
+    r = alpha * red + nalpha * r;
+    g = alpha * green + nalpha * g;
+    b = alpha * blue + nalpha * blue;
+    st3m_leds_set_single_rgb(index, r, g, b);
 }
 
 void st3m_leds_set_single_rgb(uint8_t index, float red, float green,
@@ -115,18 +182,27 @@ void st3m_leds_set_single_rgb(uint8_t index, float red, float green,
     state.target_buffer[index].r = (uint8_t)(red * 255);
     state.target_buffer[index].g = (uint8_t)(green * 255);
     state.target_buffer[index].b = (uint8_t)(blue * 255);
+    state.is_steady = false;
+    UNLOCK_INCOMING;
+}
+
+void st3m_leds_get_single_rgb(uint8_t index, float *red, float *green,
+                              float *blue) {
+    LOCK_INCOMING;
+    *red = ((float)state.target_buffer[index].r) / 255;
+    *green = ((float)state.target_buffer[index].g) / 255;
+    *blue = ((float)state.target_buffer[index].b) / 255;
     UNLOCK_INCOMING;
 }
 
 void st3m_leds_set_single_hsv(uint8_t index, float hue, float sat, float val) {
     st3m_hsv_t hsv = {
-        .h = hue,
+        .h = hue * TAU360,
         .s = sat,
         .v = val,
     };
-    LOCK_INCOMING;
-    state.target_buffer[index] = st3m_hsv_to_rgb(hsv);
-    UNLOCK_INCOMING;
+    st3m_rgb_t rgb = st3m_hsv_to_rgb(hsv);
+    st3m_leds_set_single_rgb(index, rgb.r, rgb.g, rgb.b);
 }
 
 void st3m_leds_set_all_rgb(float red, float green, float blue) {
@@ -135,9 +211,21 @@ void st3m_leds_set_all_rgb(float red, float green, float blue) {
     }
 }
 
-void st3m_leds_set_all_hsv(float h, float s, float v) {
+void st3m_leds_set_all_rgba(float red, float green, float blue, float alpha) {
+    for (int i = 0; i < 40; i++) {
+        st3m_leds_set_single_rgba(i, red, green, blue, alpha);
+    }
+}
+
+void st3m_leds_set_all_hsv(float hue, float sat, float val) {
+    st3m_hsv_t hsv = {
+        .h = hue * TAU360,
+        .s = sat,
+        .v = val,
+    };
+    st3m_rgb_t rgb = st3m_hsv_to_rgb(hsv);
     for (int i = 0; i < 40; i++) {
-        st3m_leds_set_single_hsv(i, h, s, v);
+        st3m_leds_set_single_rgb(i, rgb.r, rgb.g, rgb.b);
     }
 }
 
@@ -148,7 +236,7 @@ static void _leds_task(void *_data) {
 
     TickType_t last_wake = xTaskGetTickCount();
     while (true) {
-        vTaskDelayUntil(&last_wake, pdMS_TO_TICKS(100));  // 10 Hz
+        vTaskDelayUntil(&last_wake, pdMS_TO_TICKS(20));  // 50 Hz
         st3m_leds_update_hardware();
     }
 }
@@ -160,9 +248,10 @@ void st3m_leds_init() {
     assert(mutex_incoming != NULL);
 
     memset(&state, 0, sizeof(state));
-    state.brightness = 69;
-    state.slew_rate = 255;
+    state.brightness = 70;
+    state.slew_rate = 235;
     state.auto_update = false;
+    state.timer = 255;
 
     for (uint16_t i = 0; i < 256; i++) {
         state.gamma_red.lut[i] = i;
@@ -207,6 +296,13 @@ uint8_t st3m_leds_get_slew_rate() {
     return res;
 }
 
+bool st3m_leds_get_steady() {
+    LOCK;
+    uint8_t res = state.is_steady;
+    UNLOCK;
+    return res;
+}
+
 void st3m_leds_set_auto_update(bool on) {
     LOCK;
     state.auto_update = on;
@@ -222,12 +318,7 @@ bool st3m_leds_get_auto_update() {
 
 void st3m_leds_set_gamma(float red, float green, float blue) {
     LOCK;
-    for (uint16_t i = 0; i < 256; i++) {
-        if (i == 0) {
-            state.gamma_red.lut[i] = 0;
-            state.gamma_green.lut[i] = 0;
-            state.gamma_blue.lut[i] = 0;
-        }
+    for (uint16_t i = 1; i < 256; i++) {
         float step = ((float)i) / 255.;
         state.gamma_red.lut[i] = (uint8_t)(254. * (powf(step, red)) + 1);
         state.gamma_green.lut[i] = (uint8_t)(254. * (powf(step, green)) + 1);

components/st3m/st3m_leds.h

@@ -25,9 +25,14 @@ void st3m_leds_init();
 // autoupdates.
 void st3m_leds_set_single_rgb(uint8_t index, float red, float green,
                               float blue);
+void st3m_leds_set_single_rgba(uint8_t index, float red, float green,
+                               float blue, float alpha);
 void st3m_leds_set_single_hsv(uint8_t index, float hue, float sat, float value);
 void st3m_leds_set_all_rgb(float red, float green, float blue);
+void st3m_leds_set_all_rgba(float red, float green, float blue, float alpha);
 void st3m_leds_set_all_hsv(float hue, float sat, float value);
+void st3m_leds_get_single_rgb(uint8_t index, float* red, float* green,
+                              float* blue);
 
 // Set/get global LED brightness, 0-255. Default 69.
 //
@@ -36,9 +41,10 @@ void st3m_leds_set_brightness(uint8_t brightness);
 uint8_t st3m_leds_get_brightness();
 
 // Set/get maximum change rate of brightness. Set to 1-3 for fade effects, set
-// to 255 to disable. Currently clocks at 10Hz.
+// to 255 to disable. Currently clocks at 50Hz.
 void st3m_leds_set_slew_rate(uint8_t slew_rate);
 uint8_t st3m_leds_get_slew_rate();
+bool st3m_leds_get_steady();
 
 // Update LEDs. Ie., copy the LED state from the first buffer into the second
 // buffer, effectively scheduling the LED state to be presented to the user.

components/st3m/st3m_media.c

+#include "st3m_media.h"
+#include "st3m_audio.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#include "esp_log.h"
+#include "esp_task.h"
+#include "esp_timer.h"
+#include "freertos/FreeRTOS.h"
+#include "freertos/semphr.h"
+#include "freertos/task.h"
+#define ST3M_PCM_BUF_SIZE (16384)
+
+static st3m_media *media_item = NULL;
+#ifdef CONFIG_FLOW3R_CTX_FLAVOUR_FULL
+
+static TaskHandle_t media_task;
+static bool media_pending_destroy = false;
+static SemaphoreHandle_t media_lock;
+
+static void st3m_media_task(void *_arg) {
+    (void)_arg;
+    TickType_t wake_time = xTaskGetTickCount();
+    TickType_t last_think = wake_time;
+    while (!media_pending_destroy) {
+        TickType_t ticks;
+        vTaskDelayUntil(&wake_time, 20 / portTICK_PERIOD_MS);
+        if (media_item->think) {
+            if (xSemaphoreTake(media_lock, portMAX_DELAY)) {
+                bool seeking = media_item->seek != -1;
+                ticks = xTaskGetTickCount();
+                media_item->think(media_item,
+                                  (ticks - last_think) * portTICK_PERIOD_MS);
+                last_think = ticks;
+                if (seeking && media_item->seek == -1) {
+                    // don't count seeking time into delta
+                    last_think = xTaskGetTickCount();
+                }
+                xSemaphoreGive(media_lock);
+            }
+        }
+        // don't starve lower priority tasks if think takes a long time
+        if ((xTaskGetTickCount() - wake_time) * portTICK_PERIOD_MS > 10) {
+            vTaskDelay(10 / portTICK_PERIOD_MS);
+        }
+    }
+    if (media_item->destroy) media_item->destroy(media_item);
+    media_item = NULL;
+    vSemaphoreDelete(media_lock);
+    media_pending_destroy = false;
+    vTaskDelete(NULL);
+}
+
+void st3m_media_stop(void) {
+    if (!media_item) return;
+    media_pending_destroy = true;
+    while (media_pending_destroy) {
+        vTaskDelay(10);
+    }
+}
+
+void st3m_media_pause(void) {
+    if (!media_item) return;
+    media_item->paused = 1;
+}
+
+void st3m_media_play(void) {
+    if (!media_item) return;
+    media_item->paused = 0;
+}
+
+bool st3m_media_is_playing(void) {
+    if (!media_item) return 0;
+    return !media_item->paused;
+}
+
+bool st3m_media_has_video(void) {
+    if (!media_item) return false;
+    return media_item->has_video;
+}
+
+bool st3m_media_has_audio(void) {
+    if (!media_item) return false;
+    return media_item->has_audio;
+}
+
+bool st3m_media_is_visual(void) {
+    if (!media_item) return false;
+    return media_item->is_visual;
+}
+
+float st3m_media_get_duration(void) {
+    if (!media_item) return 0;
+    return media_item->duration;
+}
+
+float st3m_media_get_position(void) {
+    if (!media_item) return 0;
+    return media_item->position;
+}
+
+float st3m_media_get_time(void) {
+    if (!media_item) return 0;
+    if (media_item->time <= 0) return media_item->time;
+    return media_item->time - st3m_pcm_queued() / 48000.0 / 2.0;
+}
+
+void st3m_media_seek(float position) {
+    if (!media_item) return;
+    if (position < 0.0) position = 0.0f;
+    media_item->seek = position;
+}
+
+void st3m_media_seek_relative(float time) {
+    if (!media_item) return;
+    st3m_media_seek((media_item->position * media_item->duration) + time);
+}
+
+void st3m_media_set_volume(float volume) { st3m_pcm_set_volume(volume); }
+
+float st3m_media_get_volume(void) { return st3m_pcm_get_volume(); }
+
+void st3m_media_draw(Ctx *ctx) {
+    if (media_item && media_item->draw) {
+        if (xSemaphoreTake(media_lock, portMAX_DELAY)) {
+            media_item->draw(media_item, ctx);
+            xSemaphoreGive(media_lock);
+        }
+    }
+}
+
+void st3m_media_think(float ms) { (void)ms; }
+
+char *st3m_media_get_string(const char *key) {
+    if (!media_item) return NULL;
+    if (!media_item->get_string) return NULL;
+    return media_item->get_string(media_item, key);
+}
+
+static float _st3m_media_zoom = 1.0f;
+static float _st3m_media_cx = 0.0f;
+static float _st3m_media_cy = 0.0f;
+
+float st3m_media_get(const char *key) {
+    if (!strcmp(key, "zoom")) return _st3m_media_zoom;
+    if (!strcmp(key, "cx")) return _st3m_media_cx;
+    if (!strcmp(key, "cy")) return _st3m_media_cy;
+    if (!media_item || !media_item->get) return -1.0f;
+    return media_item->get(media_item, key);
+}
+
+void st3m_media_set(const char *key, float value) {
+    if (!strcmp(key, "zoom")) {
+        _st3m_media_zoom = value;
+        return;
+    }
+    if (!strcmp(key, "cx")) {
+        _st3m_media_cx = value;
+        return;
+    }
+    if (!strcmp(key, "cy")) {
+        _st3m_media_cy = value;
+        return;
+    }
+    if (!media_item || !media_item->set) return;
+    return media_item->set(media_item, key, value);
+}
+
+st3m_media *st3m_media_load_mpg1(const char *path);
+st3m_media *st3m_media_load_mod(const char *path);
+st3m_media *st3m_media_load_gif(const char *path);
+st3m_media *st3m_media_load_mp3(const char *path);
+st3m_media *st3m_media_load_txt(const char *path);
+st3m_media *st3m_media_load_bin(const char *path);
+
+static int file_get_contents(const char *path, uint8_t **contents,
+                             size_t *length) {
+    FILE *file;
+    long size;
+    long remaining;
+    uint8_t *buffer;
+    file = fopen(path, "rb");
+    if (!file) {
+        return -1;
+    }
+    fseek(file, 0, SEEK_END);
+    size = remaining = ftell(file);
+
+    if (length) {
+        *length = size;
+    }
+    rewind(file);
+    buffer = malloc(size + 2);
+    if (!buffer) {
+        fclose(file);
+        return -1;
+    }
+    remaining -= fread(buffer, 1, remaining, file);
+    if (remaining) {
+        fclose(file);
+        free(buffer);
+        return -1;
+    }
+    fclose(file);
+    *contents = (unsigned char *)buffer;
+    buffer[size] = 0;
+    return 0;
+}
+
+bool st3m_media_load(const char *path, bool paused) {
+    struct stat statbuf;
+    if (!strncmp(path, "http://", 7)) {
+        st3m_media_stop();
+        media_item = st3m_media_load_mp3(path);
+    } else if (stat(path, &statbuf)) {
+        st3m_media_stop();
+        media_item = st3m_media_load_txt(path);
+    } else if (strstr(path, ".mp3") == strrchr(path, '.')) {
+        st3m_media_stop();
+        media_item = st3m_media_load_mp3(path);
+    } else if (strstr(path, ".mpg")) {
+        st3m_media_stop();
+        media_item = st3m_media_load_mpg1(path);
+    } else if (strstr(path, ".gif")) {
+        st3m_media_stop();
+        media_item = st3m_media_load_gif(path);
+    } else if ((strstr(path, ".mod") == strrchr(path, '.'))) {
+        st3m_media_stop();
+        media_item = st3m_media_load_mod(path);
+    } else if ((strstr(path, ".json") == strrchr(path, '.')) ||
+               (strstr(path, ".txt") == strrchr(path, '.')) ||
+               (strstr(path, "/README") == strrchr(path, '/')) ||
+               (strstr(path, ".toml") == strrchr(path, '.')) ||
+               (strstr(path, ".py") == strrchr(path, '.'))) {
+        st3m_media_stop();
+        media_item = st3m_media_load_txt(path);
+    }
+
+    if (!media_item) {
+        media_item = st3m_media_load_txt(path);
+    }
+
+    media_item->volume = 4096;
+    media_item->paused = paused;
+
+    media_lock = xSemaphoreCreateMutex();
+
+    BaseType_t res =
+        xTaskCreatePinnedToCore(st3m_media_task, "media", 16384, NULL,
+                                ESP_TASK_PRIO_MIN + 3, &media_task, 1);
+
+    return res == pdPASS;
+}
+
+typedef struct {
+    st3m_media control;
+    char *data;
+    size_t size;
+    float scroll_pos;
+    char *path;
+} txt_state;
+
+static void txt_destroy(st3m_media *media) {
+    txt_state *self = (void *)media;
+    if (self->data) free(self->data);
+    if (self->path) free(self->path);
+    free(self);
+}
+
+static void txt_draw(st3m_media *media, Ctx *ctx) {
+    txt_state *self = (void *)media;
+    ctx_rectangle(ctx, -120, -120, 240, 240);
+    ctx_gray(ctx, 0);
+    ctx_fill(ctx);
+    ctx_gray(ctx, 1.0);
+    ctx_move_to(ctx, -85, -70);
+    ctx_font(ctx, "mono");
+    ctx_font_size(ctx, 13.0);
+    // ctx_text (ctx, self->path);
+    ctx_text(ctx, self->data);
+}
+
+static void txt_think(st3m_media *media, float ms_elapsed) {
+    // txt_state *self = (void*)media;
+}
+
+st3m_media *st3m_media_load_txt(const char *path) {
+    txt_state *self = (txt_state *)malloc(sizeof(txt_state));
+    memset(self, 0, sizeof(txt_state));
+    self->control.draw = txt_draw;
+    self->control.think = txt_think;
+    self->control.destroy = txt_destroy;
+    file_get_contents(path, (void *)&self->data, &self->size);
+    if (!self->data) {
+        self->data = malloc(strlen(path) + 64);
+        sprintf(self->data, "40x - %s", path);
+        self->size = strlen((char *)self->data);
+    }
+    self->path = strdup(path);
+    return (void *)self;
+}
+
+st3m_media *st3m_media_load_bin(const char *path) {
+    txt_state *self = (txt_state *)malloc(sizeof(txt_state));
+    memset(self, 0, sizeof(txt_state));
+    self->control.draw = txt_draw;
+    self->control.destroy = txt_destroy;
+    file_get_contents(path, (void *)&self->data, &self->size);
+    if (!self->data) {
+        self->data = malloc(strlen(path) + 64);
+        sprintf(self->data, "40x - %s", path);
+        self->size = strlen(self->data);
+    }
+    self->path = strdup(path);
+    return (void *)self;
+}
+
+st3m_media_tags_t *audio_mp3_get_tags(const char *path);
+
+st3m_media_tags_t *st3m_media_get_tags(const char *path) {
+    // only implemented for mp3 for now
+    return audio_mp3_get_tags(path);
+}
+
+void st3m_media_free_tags(st3m_media_tags_t *tags) {
+    if (!tags) return;
+    free(tags->artist);
+    free(tags->title);
+    free(tags->album);
+    free(tags);
+}
+#endif

components/st3m/st3m_media.h

+#pragma once
+
+#include <ctx.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include "st3m_pcm.h"
+
+typedef struct _st3m_media st3m_media;
+
+struct _st3m_media {
+    // set a tunable, to a numeric value - available tunables depends on
+    // decoder
+    void (*set)(st3m_media *media, const char *key, float value);
+    // get a property/or tunable, defaulting to -1 for nonexisting keys
+    float (*get)(st3m_media *media, const char *key);
+    // get a string property/metadata, NULL if not existing or a string
+    // to be freed
+    char *(*get_string)(st3m_media *media, const char *key);
+
+    // free resources used by this media instance
+    void (*destroy)(st3m_media *media);
+
+    // draw the current frame / visualization / metadata screen
+    void (*draw)(st3m_media *media, Ctx *ctx);
+
+    // do decoding work corresponding to passed time
+    void (*think)(st3m_media *media, float ms);
+
+    // Duration of media in seconds or -1 for infinite/streaming media
+    // at worst approximation of some unit, set by decoder.
+    float duration;
+
+    // currently played back position - set by decoder
+    float position;
+
+    // currently played back position, in seconds - set by decoder
+    float time;  // time might be precise even when duration is not
+                 // until first play through of some media, when time
+                 // duration should also be set exact.
+
+    // whether the loaded file contains a video stream - set by decoder
+    bool has_video;
+
+    // whether the loaded file contains an audio stream - set by decoder
+    bool has_audio;
+
+    // whether the loaded file contains visual content - set by decoder
+    bool is_visual;
+
+    // decoder should seek to this relative if not -1, and set it to -1
+    float seek;
+
+    // audio volume
+    int volume;
+
+    // if set to 1 playback is momentarily stopped but can be resumed,
+    // this is toggled by st3m_media_play | st3m_media_pause
+    bool paused;
+};
+
+// stops the currently playing media item
+void st3m_media_stop(void);
+// set a new media item
+bool st3m_media_load(const char *path_or_uri, bool paused);
+
+// decode current media item ms ahead (unless paused)
+void st3m_media_think(float ms);
+
+// draw the codecs own view of itself / its meta data - progress
+// for video or animations formats this should draw the media-content
+// other codecs can find mixes of debug visualizations.
+void st3m_media_draw(Ctx *ctx);
+
+// controls whether we are playing
+void st3m_media_pause(void);
+void st3m_media_play(void);
+bool st3m_media_is_playing(void);
+
+// whether there's a video stream being played
+bool st3m_media_has_video(void);
+
+// whether there's a video stream being played
+bool st3m_media_has_audio(void);
+
+// whether the played file has visual content
+bool st3m_media_is_visual(void);
+
+// get duration in seconds
+float st3m_media_get_duration(void);
+// get current playback time in seconds
+float st3m_media_get_time(void);
+// get current playback position relative to overall duration
+float st3m_media_get_position(void);
+// seek to a position relative to overall duration
+void st3m_media_seek(float position);
+// seek a relative amount of seconds forward or with negative values back
+void st3m_media_seek_relative(float seconds_jump);
+// set audio volume (0.0 - 1.0)
+void st3m_media_set_volume(float volume);
+// get audio volume
+float st3m_media_get_volume(void);
+
+// get decoder specific string or NULL if not existing, free returned value
+//  common values:
+//     "title"  "artist"
+char *st3m_media_get_string(const char *key);
+// get a decoder specific numeric value, defaulting to -1 for nonexisting values
+float st3m_media_get(const char *key);
+// set a decoder specific floating point value
+// example posible/or already used values:
+//
+//    "grayscale"   0 or 1     - drop color bits for performance
+//
+// some keys are global and stored outside codecs, they can be queried
+// and set both by apps and codecs:
+//
+//    "zoom"        0.0 - 1.0  - how large part of display video cover
+//    "cx" "cy"     -120 - 120 - where to put the center of video textures
+void st3m_media_set(const char *key, float value);
+
+typedef struct {
+    char *artist;
+    char *title;
+    char *album;
+    int year;
+    int track_number;
+} st3m_media_tags_t;
+
+// get collection of tags
+st3m_media_tags_t *st3m_media_get_tags(const char *path);
+void st3m_media_free_tags(st3m_media_tags_t *tags);

components/st3m/st3m_mode.c

@@ -13,6 +13,7 @@
 #include "st3m_fs_sd.h"
 #include "st3m_gfx.h"
 #include "st3m_io.h"
+#include "st3m_media.h"
 #include "st3m_usb.h"
 
 static const char *TAG = "st3m-mode";
@@ -87,6 +88,11 @@ void st3m_mode_set(st3m_mode_kind_t kind, const char *message) {
         _mode.message = NULL;
     }
 
+    if (_mode.kind != kind) {
+        st3m_media_stop();
+        st3m_gfx_flush(200);
+    }
+
     _mode.kind = kind;
     _mode.shown = false;
 
@@ -123,10 +129,12 @@ void st3m_mode_update_display(bool *restartable) {
         case st3m_mode_kind_disk_sd:
         case st3m_mode_kind_disk_flash: {
             const char *lines[] = {
-                "Press left shoulder button",
-                "to exit.",
+                (st3m_io_app_button_is_left()) ? "Press right shoulder button"
+                                               : "Press left shoulder button",
+                "to restart.",
                 NULL,
             };
+            // st3m_io_app_button_is_left
             st3m_gfx_textview_t tv = {
                 .title = "Disk Mode",
                 .lines = lines,
@@ -139,7 +147,9 @@ void st3m_mode_update_display(bool *restartable) {
                 _mode.shown = true;
                 const char *lines[] = {
                     "Send Ctrl-D over USB",
-                    "or press left shoulder button",
+                    (st3m_io_app_button_is_left())
+                        ? "or press right shoulder button"
+                        : "or press left shoulder button",
                     "to restart.",
                     NULL,
                 };
@@ -158,7 +168,8 @@ void st3m_mode_update_display(bool *restartable) {
             }
             const char *lines[] = {
                 msg,
-                "Press left shoulder button",
+                (st3m_io_app_button_is_left()) ? "Press right shoulder button"
+                                               : "Press left shoulder button",
                 "to restart.",
                 NULL,
             };
@@ -186,13 +197,13 @@ static void _task(void *arg) {
         st3m_mode_update_display(&restartable);
 
         if (restartable) {
-            st3m_tripos tp = st3m_io_left_button_get();
+            st3m_tripos tp = st3m_io_os_button_get();
             if (tp == st3m_tripos_mid) {
                 st3m_gfx_textview_t tv = {
                     .title = "Restarting...",
                 };
                 st3m_gfx_show_textview(&tv);
-                vTaskDelay(100 / portTICK_PERIOD_MS);
+                vTaskDelay(500 / portTICK_PERIOD_MS);
                 esp_restart();
             }
         }

components/st3m/st3m_pcm.c

+#include "st3m_pcm.h"
+
+#include <unistd.h>
+
+#include "freertos/FreeRTOS.h"
+#define ST3M_PCM_BUF_SIZE (16384)
+
+static int st3m_pcm_gain = 4096;
+int st3m_pcm_queue_length(void) { return ST3M_PCM_BUF_SIZE - 4000; }
+
+EXT_RAM_BSS_ATTR static int16_t st3m_pcm_buffer[ST3M_PCM_BUF_SIZE];
+static int st3m_pcm_w = 1;
+static int st3m_pcm_r = 0;
+
+static inline int16_t apply_gain(int16_t sample, int16_t gain) {
+    if (gain == 4096) return sample;
+    int32_t val = (sample * st3m_pcm_gain) >> 12;
+    if (gain < 4096) return val;
+    if (val > 32767) {
+        return 32767;
+    } else if (val < -32767) {
+        return -32767;
+    }
+    return val;
+}
+
+bool st3m_pcm_audio_render(int16_t *rx, int16_t *tx, uint16_t len) {
+    if (((((st3m_pcm_r + 1) % ST3M_PCM_BUF_SIZE)) == st3m_pcm_w)) return false;
+    for (int i = 0; i < len; i++) {
+        if ((st3m_pcm_r + 1 != st3m_pcm_w) &&
+            (st3m_pcm_r + 1 - ST3M_PCM_BUF_SIZE != st3m_pcm_w)) {
+            tx[i] = st3m_pcm_buffer[st3m_pcm_r++];
+            st3m_pcm_r %= ST3M_PCM_BUF_SIZE;
+        } else {
+            tx[i] = 0;
+        }
+    }
+    return true;
+}
+
+static int phase = 0;
+
+void st3m_pcm_queue_s16(int hz, int ch, int count, int16_t *data) {
+    if (hz == 48000) {
+        if (ch == 2) {
+            if (st3m_pcm_gain == 4096) {
+                for (int i = 0; i < count * 2; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] = data[i];
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+            } else {
+                for (int i = 0; i < count * 2; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] =
+                        apply_gain(data[i], st3m_pcm_gain);
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+            }
+        } else if (ch == 1) {
+            if (st3m_pcm_gain == 4096) {
+                for (int i = 0; i < count; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] = data[i];
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                    st3m_pcm_buffer[st3m_pcm_w++] = data[i];
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+            } else {
+                for (int i = 0; i < count; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] =
+                        apply_gain(data[i], st3m_pcm_gain);
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                    st3m_pcm_buffer[st3m_pcm_w++] =
+                        apply_gain(data[i], st3m_pcm_gain);
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+            }
+        }
+    } else {
+        int fraction = ((48000.0 / hz) - 1.0) * 65536;
+        if (ch == 2) {
+            for (int i = 0; i < count; i++) {
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i * 2], st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i * 2 + 1], st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                if (phase > 65536) {
+                    phase -= 65536;
+                    i--;
+                    continue;
+                }
+                phase += fraction;
+            }
+        } else if (ch == 1) {
+            for (int i = 0; i < count; i++) {
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i], st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i], st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+
+                if (phase > 65536) {
+                    phase -= 65536;
+                    i--;
+                    continue;
+                }
+                phase += fraction;
+            }
+        }
+    }
+}
+
+void st3m_pcm_queue_float(int hz, int ch, int count, float *data) {
+    if (hz == 48000) {
+        if (ch == 2) {
+            if (st3m_pcm_gain == 4096)
+                for (int i = 0; i < count * 2; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] = data[i] * 32767;
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+            else
+                for (int i = 0; i < count * 2; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] =
+                        apply_gain(data[i] * 32767, st3m_pcm_gain);
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+        } else if (ch == 1) {
+            if (st3m_pcm_gain == 4096)
+                for (int i = 0; i < count; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] = data[i] * 32767;
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                    st3m_pcm_buffer[st3m_pcm_w++] = data[i] * 32767;
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+            else
+                for (int i = 0; i < count; i++) {
+                    st3m_pcm_buffer[st3m_pcm_w++] =
+                        apply_gain(data[i] * 32767, st3m_pcm_gain);
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                    st3m_pcm_buffer[st3m_pcm_w++] =
+                        apply_gain(data[i] * 32767, st3m_pcm_gain);
+                    st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                }
+        }
+    } else {
+        int fraction = ((48000.0 / hz) - 1.0) * 65536;
+        if (ch == 2) {
+            for (int i = 0; i < count; i++) {
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i * 2] * 32767, st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i * 2 + 1] * 32767, st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                if (phase >= 65536) {
+                    phase -= 65536;
+                    i--;
+                    continue;
+                }
+                phase += fraction;
+            }
+        } else if (ch == 1) {
+            for (int i = 0; i < count; i++) {
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i] * 32767, st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+                st3m_pcm_buffer[st3m_pcm_w++] =
+                    apply_gain(data[i] * 32767, st3m_pcm_gain);
+                st3m_pcm_w %= ST3M_PCM_BUF_SIZE;
+
+                if (phase >= 65536) {
+                    phase -= 65536;
+                    i--;
+                    continue;
+                }
+                phase += fraction;
+            }
+        }
+    }
+}
+
+int st3m_pcm_queued(void) {
+    if (st3m_pcm_r > st3m_pcm_w)
+        return (ST3M_PCM_BUF_SIZE - st3m_pcm_r) + st3m_pcm_w;
+    return st3m_pcm_w - st3m_pcm_r;
+}
+
+void st3m_pcm_set_volume(float volume) { st3m_pcm_gain = volume * 4096; }
+
+float st3m_pcm_get_volume(void) { return st3m_pcm_gain / 4096.0; }

components/st3m/st3m_pcm.h

+#pragma once
+
+#include <stdbool.h>
+#include <stdint.h>
+
+// audio rendering function used by st3m_audio
+bool st3m_pcm_audio_render(int16_t *rx, int16_t *tx, uint16_t len);
+
+// query how many pcm samples have been queued for output
+int st3m_pcm_queued(void);
+
+// returns the internal PCM buffer length, this return the maximum number
+// of samples that can be queued - the buffer might internally be larger.
+int st3m_pcm_queue_length(void);
+
+// queue signed 16bit samples, supports hz is up to 48000, ch 1 for mono
+// or 2 for stereo
+void st3m_pcm_queue_s16(int hz, int ch, int count, int16_t *data);
+
+// queue 32bit float samples, supports hz is up to 48000 ch 1 for mono or 2
+// for stereo
+void st3m_pcm_queue_float(int hz, int ch, int count, float *data);
+
+// set audio volume (0.0 - 1.0)
+void st3m_pcm_set_volume(float volume);
+
+// get audio volume
+float st3m_pcm_get_volume(void);

components/st3m/st3m_scope.c

@@ -49,7 +49,8 @@ void st3m_scope_init(void) {
     memset(scope.read_buffer, 0, sizeof(int16_t) * scope.buffer_size);
 
     scope.write_head_position = 0;
-    scope.prev_write_attempt = 0;
+    scope.prev_value = 0;
+    scope.zero_crossing_occurred = false;
     ESP_LOGI(TAG, "initialized");
 }
 
@@ -58,39 +59,45 @@ void st3m_scope_write(int16_t value) {
         return;
     }
 
-    int16_t prev_write_attempt = scope.prev_write_attempt;
-    scope.prev_write_attempt = value;
-
-    // If we're about to write the first sample, make sure we do so at a
-    // positive zero crossing.
-    if (scope.write_head_position == 0) {
-        // Calculate 'positivity' sign of this value and previous value.
-        int16_t this = value > 0;
-        int16_t prev = prev_write_attempt > 0;
-
-        if (this != 1 || prev != 0) {
-            return;
-        }
+    if ((!scope.zero_crossing_occurred) && (value > 0) &&
+        (scope.prev_value <= 0)) {
+        scope.write_head_position = 0;
+        scope.zero_crossing_occurred = true;
     }
 
     if (scope.write_head_position >= scope.buffer_size) {
         scope.write_buffer = Atomic_SwapPointers_p32(
             (void *volatile *)&scope.exchange_buffer, scope.write_buffer);
         scope.write_head_position = 0;
+        scope.zero_crossing_occurred = false;
     } else {
         scope.write_buffer[scope.write_head_position] = value;
         scope.write_head_position++;
     }
+
+    scope.prev_value = value;
 }
 
-void st3m_scope_draw(Ctx *ctx) {
+size_t st3m_scope_get_buffer_x(int16_t **buf) {
     if (scope.write_buffer == NULL) {
-        return;
+        return 0;
     }
 
     scope.read_buffer = Atomic_SwapPointers_p32(
         (void *volatile *)&scope.exchange_buffer, scope.read_buffer);
 
+    if (buf) {
+        *buf = scope.read_buffer;
+    }
+
+    return scope.buffer_size;
+}
+
+void st3m_scope_draw(Ctx *ctx) {
+    if (st3m_scope_get_buffer_x(NULL) == 0) {
+        return;
+    }
+
     // How much to divide the values persisted in the buffer to scale to
     // -120+120.
     //
@@ -105,7 +112,7 @@ void st3m_scope_draw(Ctx *ctx) {
     //
     // decimate == 2 -> every second sample is drawn (240/2 == 120 line segments
     // are drawn). Looks good enough.
-    size_t decimate = 2;
+    size_t decimate = 1;
 
     int x = -120;
     int y = scope.read_buffer[0] >> shift;
@@ -116,8 +123,8 @@ void st3m_scope_draw(Ctx *ctx) {
         ctx_line_to(ctx, x, y);
     }
 
-    ctx_line_to(ctx, 130, 0);
-    ctx_line_to(ctx, 130, -130);
-    ctx_line_to(ctx, -130, -130);
-    ctx_line_to(ctx, -130, 0);
+    ctx_save(ctx);
+    ctx_line_width(ctx, -1.0f);
+    ctx_stroke(ctx);
+    ctx_restore(ctx);
 }

components/st3m/st3m_scope.h

 #pragma once
 
-// st3m_scope implements an oscilloscope-style music visualizer.
+// st3m_scope implements a basic scope.
 //
 // The audio subsystem will continuously send the global mixing output result
 // into the oscilloscope. User code can decide when to draw said scope.
@@ -24,9 +24,8 @@ typedef struct {
 
     // Offset where the write handler should write the next sample.
     uint32_t write_head_position;
-    // Previous sample that was attempted to be written. Used for
-    // zero-detection.
-    int16_t prev_write_attempt;
+    int16_t prev_value;
+    bool zero_crossing_occurred;
 } st3m_scope_t;
 
 // Initialize global scope. Must be performed before any other access to scope
@@ -39,12 +38,11 @@ void st3m_scope_init(void);
 // Write a sound sample to the scope.
 void st3m_scope_write(int16_t value);
 
+// Retrieve scope's data buffer. Remains valid until the next
+// st3m_scope_get_buffer_x or st3m_scope_draw call. Returns buffer's length.
+size_t st3m_scope_get_buffer_x(int16_t **buf);
+
 // Draw the scope at bounding box -120/-120 +120/+120.
 //
-// The scope will be drawn as a closable line segment starting at x:-120
-// y:sample[0], through x:120 y:sample[239], then going through x:130 y:130,
-// x:-130 y:130.
-//
-// The user is responsible for setting a color and running a fill/stroke
-// afterwards.
+// The user is responsible for clearing background and setting a color.
 void st3m_scope_draw(Ctx *ctx);

components/st3m/st3m_tar.c

@@ -7,6 +7,8 @@
 #include "esp_log.h"
 #include "miniz.h"
 
+#define READ_BUF_SIZE 4096
+
 static const char *TAG = "st3m-tar";
 
 void st3m_tar_parser_init(st3m_tar_parser_t *parser) {
@@ -201,9 +203,14 @@ static int _mkpath(char *file_path, mode_t mode) {
 static void _extractor_on_file_start(void *user, const char *name, size_t size,
                                      char type) {
     st3m_tar_extractor_t *extractor = (st3m_tar_extractor_t *)user;
-    if (extractor->cur_file != NULL) {
-        fclose(extractor->cur_file);
-        extractor->cur_file = NULL;
+    if (extractor->cur_file_contents != NULL) {
+        free(extractor->cur_file_contents);
+        extractor->cur_file_contents = NULL;
+        extractor->cur_file_offset = NULL;
+    }
+    if (extractor->cur_file_path != NULL) {
+        free(extractor->cur_file_path);
+        extractor->cur_file_path = NULL;
     }
 
     if (size == 0 || type != '0') {
@@ -222,33 +229,88 @@ static void _extractor_on_file_start(void *user, const char *name, size_t size,
         ESP_LOGW(TAG, "Failed to create parent directory for %s: %s", path,
                  strerror(errno));
     }
-    extractor->cur_file = fopen(path, "w");
-    if (extractor->cur_file == NULL) {
-        ESP_LOGW(TAG, "Failed to create %s: %s", path, strerror(errno));
-    }
+
+    extractor->cur_file_path = path;
+
+    extractor->cur_file_contents = malloc(size);
+    extractor->cur_file_offset = extractor->cur_file_contents;
+    extractor->cur_file_size = size;
 
     if (extractor->on_file != NULL) {
         extractor->on_file(path);
     }
-    free(path);
 }
 
 static void _extractor_on_file_data(void *user, const uint8_t *buffer,
                                     size_t bufsize) {
     st3m_tar_extractor_t *extractor = (st3m_tar_extractor_t *)user;
-    if (extractor->cur_file == NULL) {
+    if (extractor->cur_file_contents == NULL) {
         return;
     }
-    fwrite(buffer, 1, bufsize, extractor->cur_file);
+
+    memcpy(extractor->cur_file_offset, buffer, bufsize);
+    extractor->cur_file_offset += bufsize;
+}
+
+static bool _is_write_needed(char *path, uint8_t *contents, size_t size) {
+    struct stat sb;
+    if (stat(path, &sb) != 0 || sb.st_size != size) {
+        // file doesn't exist or size changed
+        return true;
+    }
+
+    // go on to compare file contents
+    FILE *file = fopen(path, "r");
+    if (file == NULL) {
+        return true;
+    }
+
+    // malloc because we don't have that much stack
+    uint8_t *buf = malloc(READ_BUF_SIZE);
+
+    for (size_t off = 0; off < size; off += READ_BUF_SIZE) {
+        size_t leftover = (size - off);
+        size_t read_size =
+            (leftover < READ_BUF_SIZE) ? leftover : READ_BUF_SIZE;
+
+        if ((fread(buf, 1, read_size, file) != read_size) ||
+            (memcmp(buf, contents + off, read_size) != 0)) {
+            fclose(file);
+            free(buf);
+            return true;
+        }
+    }
+
+    fclose(file);
+    free(buf);
+    return false;
 }
 
 static void _extractor_on_file_end(void *user) {
     st3m_tar_extractor_t *extractor = (st3m_tar_extractor_t *)user;
-    if (extractor->cur_file == NULL) {
+    if (extractor->cur_file_contents == NULL) {
         return;
     }
-    fclose(extractor->cur_file);
-    extractor->cur_file = NULL;
+
+    if (_is_write_needed(extractor->cur_file_path, extractor->cur_file_contents,
+                         extractor->cur_file_size)) {
+        FILE *cur_file = fopen(extractor->cur_file_path, "w");
+        if (cur_file == NULL) {
+            ESP_LOGW(TAG, "Failed to create %s: %s", extractor->cur_file_path,
+                     strerror(errno));
+        } else {
+            fwrite(extractor->cur_file_contents, 1, extractor->cur_file_size,
+                   cur_file);
+            fflush(cur_file);
+            fclose(cur_file);
+        }
+    }
+
+    free(extractor->cur_file_contents);
+    free(extractor->cur_file_path);
+    extractor->cur_file_contents = NULL;
+    extractor->cur_file_offset = NULL;
+    extractor->cur_file_path = NULL;
 }
 
 void st3m_tar_extractor_init(st3m_tar_extractor_t *extractor) {

components/st3m/st3m_tar.h

@@ -51,7 +51,11 @@ typedef struct {
     // If set, will be called any file begins extraction. Useful for feedback.
     void (*on_file)(const char *name);
 
-    FILE *cur_file;
+    char *cur_file_path;
+    uint8_t *cur_file_contents;
+    uint8_t *cur_file_offset;
+    size_t cur_file_size;
+
     st3m_tar_parser_t parser;
 } st3m_tar_extractor_t;
 

components/st3m/st3m_usb.c

 #include "st3m_usb.h"
+#ifndef CONFIG_DEBUG_GDB_ENABLED
+#ifndef CONFIG_DEBUG_GDB_DISABLED
+#error "st3m: no debug flags"
+#endif
+#endif
+#ifdef CONFIG_DEBUG_GDB_ENABLED
+#ifdef CONFIG_DEBUG_GDB_DISABLED
+#error "st3m: invalid debug flags"
+#endif
+#endif
 
 static const char *TAG = "st3m-usb";
 
@@ -11,6 +21,8 @@ static const char *TAG = "st3m-usb";
 #include "esp_private/usb_phy.h"
 #include "tusb.h"
 
+#include "st3m_console.h"
+
 static SemaphoreHandle_t _mu = NULL;
 static st3m_usb_mode_kind_t _mode = st3m_usb_mode_kind_disabled;
 static usb_phy_handle_t phy_hdl;
@@ -45,12 +57,16 @@ static void _generate_serial(void) {
 }
 
 void st3m_usb_init(void) {
+#ifdef CONFIG_DEBUG_GDB_ENABLED
+    return;
+#endif
     assert(_mu == NULL);
     _mu = xSemaphoreCreateMutex();
     assert(_mu != NULL);
     _mode = st3m_usb_mode_kind_disabled;
 
     _generate_serial();
+#ifndef CONFIG_DEBUG_GDB_ENABLED
     st3m_usb_cdc_init();
     usb_phy_config_t phy_conf = {
         .controller = USB_PHY_CTRL_OTG,
@@ -68,9 +84,13 @@ void st3m_usb_init(void) {
 
     xTaskCreate(_usb_task, "usb", 4096, NULL, 5, NULL);
     ESP_LOGI(TAG, "USB stack started");
+#endif
 }
 
 void st3m_usb_mode_switch(st3m_usb_mode_t *mode) {
+#ifdef CONFIG_DEBUG_GDB_ENABLED
+    return;
+#endif
     xSemaphoreTake(_mu, portMAX_DELAY);
 
     bool running = false;
@@ -120,6 +140,9 @@ void st3m_usb_mode_switch(st3m_usb_mode_t *mode) {
 }
 
 bool st3m_usb_connected(void) {
+#ifdef CONFIG_DEBUG_GDB_ENABLED
+    return false;
+#endif
     xSemaphoreTake(_mu, portMAX_DELAY);
     bool res = _connected;
     xSemaphoreGive(_mu);
@@ -144,3 +167,24 @@ void tud_suspend_cb(bool remote_wakeup_en) {
         st3m_usb_cdc_detached();
     }
 }
+
+void st3m_usb_startup() {
+#ifdef CONFIG_DEBUG_GDB_ENABLED
+    return;
+#endif
+    st3m_usb_app_conf_t app = {
+        .fn_rx = st3m_console_cdc_on_rx,
+        .fn_txpoll = st3m_console_cdc_on_txpoll,
+        .fn_detach = st3m_console_cdc_on_detach,
+    };
+    st3m_usb_mode_t usb_mode = {
+        .kind = st3m_usb_mode_kind_app,
+        .app = &app,
+    };
+    st3m_usb_mode_switch(&usb_mode);
+    puts(" ___ _           ___     _         _");
+    puts("|  _| |___ _ _ _|_  |___| |_ ___ _| |___ ___");
+    puts("|  _| | . | | | |_  |  _| . | .'| . | . | -_|");
+    puts("|_| |_|___|_____|___|_| |___|__,|___|_  |___|");
+    puts("                                    |___|");
+}

components/st3m/st3m_usb.h

@@ -38,6 +38,7 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
+#include "sdkconfig.h"
 
 typedef enum {
     // Device should not enumerate.
@@ -106,6 +107,8 @@ void st3m_usb_mode_switch(st3m_usb_mode_t *target);
 // Initialize the subsystem. Must be called, bad things will happen otherwise.
 void st3m_usb_init();
 
+void st3m_usb_startup();
+
 // Return true if the badge is connected to a host.
 bool st3m_usb_connected(void);
 

components/st3m/st3m_usb_descriptors.c

@@ -45,8 +45,8 @@ static st3m_usb_descriptor_set_t _descset_app = {
             .bDeviceProtocol = MISC_PROTOCOL_IAD,
             .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE,
 
-            .idVendor = 0x303a,
-            .idProduct = 0x4042,
+            .idVendor = 0x1209,
+            .idProduct = 0xF103,
             .bcdDevice = 0x0100,
 
             .iManufacturer = 0x01,
@@ -94,8 +94,8 @@ static st3m_usb_descriptor_set_t _descset_disk = {
             .bDeviceProtocol = 0x00,
 
             .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE,
-            .idVendor = 0x303a,
-            .idProduct = 0x4023,
+            .idVendor = 0x1209,
+            .idProduct = 0xF10B,
 
             .bcdDevice = 0x0100,
 

components/video_gif/CMakeLists.txt

+idf_component_register(
+    SRCS
+        video_gif.c
+    INCLUDE_DIRS
+        .
+        ../ctx
+        ../ctx/fonts/
+        ../st3m
+)

components/video_gif/video_gif.c

+/*
+ * Copyright 2019, 2021, 2023 Øyvind Kolås <pippin@gimp.org>
+ *
+ *  plays a gif animation on loop using stb_image, originally written as
+ *  a l0dable for card10
+ *
+ */
+
+#include <st3m_gfx.h>
+#include <st3m_media.h>
+
+#include "stb_image.h"
+
+/* we need more defs, copied from internals */
+typedef uint32_t stbi__uint32;
+typedef int16_t stbi__int16;
+
+typedef struct {
+    stbi__uint32 img_x, img_y;
+    int img_n, img_out_n;
+
+    stbi_io_callbacks io;
+    void *io_user_data;
+
+    int read_from_callbacks;
+    int buflen;
+    stbi_uc buffer_start[128];
+    int callback_already_read;
+
+    stbi_uc *img_buffer, *img_buffer_end;
+    stbi_uc *img_buffer_original, *img_buffer_original_end;
+} stbi__context;
+
+typedef struct {
+    stbi__int16 prefix;
+    stbi_uc first;
+    stbi_uc suffix;
+} stbi__gif_lzw;
+
+typedef struct {
+    int w, h;
+    stbi_uc *out;  // output buffer (always 4 components)
+    stbi_uc
+        *background;  // The current "background" as far as a gif is concerned
+    stbi_uc *history;
+    int flags, bgindex, ratio, transparent, eflags;
+    stbi_uc pal[256][4];
+    stbi_uc lpal[256][4];
+    stbi__gif_lzw codes[8192];
+    stbi_uc *color_table;
+    int parse, step;
+    int lflags;
+    int start_x, start_y;
+    int max_x, max_y;
+    int cur_x, cur_y;
+    int line_size;
+    int delay;
+} stbi__gif;
+
+stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp,
+                             int req_comp, stbi_uc *two_back);
+
+void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user);
+
+/////////////////////////////////////////////////////
+
+typedef struct {
+    st3m_media control;
+    char *path;
+    stbi__context s;
+    stbi__gif g;
+
+    int width;
+    int height;
+    int delay;
+    int file_size;
+    FILE *file;
+    uint8_t *pixels;
+} gif_state;
+
+static int read_cb(void *user, char *data, int size) {
+    gif_state *gif = user;
+    return fread(data, 1, size, gif->file);
+}
+static void skip_cb(void *user, int n) {
+    gif_state *gif = user;
+    long pos = ftell(gif->file);
+    fseek(gif->file, pos + n, SEEK_SET);
+}
+
+static int eof_cb(void *user) {
+    gif_state *gif = user;
+    long pos = ftell(gif->file);
+    if (pos >= gif->file_size) return 1;
+    return 0;
+}
+
+static stbi_io_callbacks clbk = { read_cb, skip_cb, eof_cb };
+
+static void gif_stop(gif_state *gif);
+
+static int gif_init(gif_state *gif) {
+    if (gif->file) {
+        gif_stop(gif);
+    }
+    gif->file = fopen(gif->path, "rb");
+    if (!gif->file) return -1;
+    gif->width = -1;
+    gif->height = -1;
+
+    fseek(gif->file, 0, SEEK_END);
+    gif->file_size = ftell(gif->file);
+    fseek(gif->file, 0, SEEK_SET);
+    memset(&gif->s, 0, sizeof(gif->s));
+    memset(&gif->g, 0, sizeof(gif->g));
+    stbi__start_callbacks(&gif->s, &clbk, (void *)gif);
+    return 0;
+}
+
+static int gif_load_frame(gif_state *gif) {
+    int c;
+    if (!gif->file) return -1;
+
+    gif->pixels = stbi__gif_load_next(&gif->s, &gif->g, &c, 4, 0);
+    if (gif->pixels == (uint8_t *)&gif->s) {
+        gif->pixels = NULL;
+        gif_stop(gif);
+        return -1;
+    }
+    if (gif->width < 0) gif->width = gif->g.w;
+    if (gif->height < 0) gif->height = gif->g.h;
+
+    return gif->g.delay;
+}
+
+static void gif_stop(gif_state *gif) {
+    if (!gif->file) return;
+    fclose(gif->file);
+    gif->file = NULL;
+    if (gif->g.out) {
+        free(gif->g.out);
+        gif->g.out = NULL;
+    }
+    if (gif->g.history) {
+        free(gif->g.history);
+        gif->g.history = NULL;
+    }
+    if (gif->g.background) {
+        free(gif->g.background);
+        gif->g.background = NULL;
+    }
+}
+
+static void gif_draw(st3m_media *media, Ctx *ctx) {
+    gif_state *gif = (gif_state *)media;
+    if (!gif->pixels) return;
+    ctx_save(ctx);
+    ctx_rectangle(ctx, -120, -120, 240, 240);
+    ctx_translate(ctx, -120, -120);
+    float scale = ctx_width(ctx) * 1.0 / gif->width;
+    float scaleh = ctx_height(ctx) * 1.0 / gif->height;
+    if (scaleh < scale) scale = scaleh;
+    ctx_translate(ctx, (ctx_width(ctx) - gif->width * scale) / 2.0,
+                  (ctx_height(ctx) - gif->height * scale) / 2.0);
+    ctx_scale(ctx, scale, scale);
+    ctx_image_smoothing(ctx, 0);
+    ctx_define_texture(ctx, "!video", gif->width, gif->height, gif->width * 4,
+                       CTX_FORMAT_RGBA8, gif->pixels, NULL);
+    ctx_compositing_mode(ctx, CTX_COMPOSITE_COPY);
+    ctx_fill(ctx);
+    ctx_restore(ctx);
+}
+
+static void gif_think(st3m_media *media, float ms_elapsed) {
+    gif_state *gif = (gif_state *)media;
+    if (st3m_media_is_playing()) gif->delay -= ms_elapsed;
+
+    if (gif->control.seek == 0) {
+        gif_init(gif);
+        gif->delay = 0;
+        gif->control.seek = -1;
+    }
+
+    if (gif->delay <= 0) {
+        gif->delay = gif_load_frame(gif);
+        if (gif->delay < 0) {
+            gif_init(gif);
+            gif->delay = gif_load_frame(gif);
+        }
+        if (gif->delay == 0) gif->delay = 100;
+    }
+}
+
+static void gif_destroy(st3m_media *media) {
+    gif_state *gif = (gif_state *)media;
+    gif_stop(gif);
+    free(gif->path);
+    free(media);
+}
+
+st3m_media *st3m_media_load_gif(const char *path) {
+    gif_state *gif = (gif_state *)malloc(sizeof(gif_state));
+    memset(gif, 0, sizeof(gif_state));
+    gif->control.duration = -1;
+    gif->control.draw = gif_draw;
+    gif->control.think = gif_think;
+    gif->control.destroy = gif_destroy;
+    gif->control.has_video = true;
+    gif->control.is_visual = true;
+    gif->path = strdup(path);
+    if (gif_init(gif) != 0) {
+        gif_destroy((st3m_media *)gif);
+        return NULL;
+    }
+
+    return (st3m_media *)gif;
+}

components/video_mpeg/.clang-tidy

+Checks: '-clang-analyzer-core.UndefinedBinaryOperatorResult,-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling, -clang-analyzer-optin.portability.UnixAPI'