media: add mp3, mpeg1video and pocketmod protracker playback

components/audio_mod/CMakeLists.txt

+idf_component_register(
+    SRCS
+        audio_mod.c
+    INCLUDE_DIRS
+        .
+        ../ctx
+        ../st3m
+)

components/audio_mod/audio_mod.c

+#ifndef __clang__
+#pragma GCC optimize("O2")
+#endif
+
+#include <fcntl.h>
+#include <st3m_audio.h>
+#include <st3m_media.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "ctx.h"
+
+#define POCKETMOD_IMPLEMENTATION
+#include "pocketmod.h"
+
+typedef struct {
+    st3m_media control;
+    pocketmod_context pocketmod;
+    uint8_t *data;
+    size_t size;
+} mod_state;
+
+static void mod_draw(st3m_media *media, Ctx *ctx) {
+    mod_state *self = (void *)media;
+
+    ctx_rectangle(ctx, -120, -120, 240, 240);
+    ctx_gray(ctx, 0);
+    ctx_fill(ctx);
+    // ctx_arc(ctx, 0, 0, 10, 10);
+    ctx_rgb(ctx, 1.0, 1.0, 1.0);
+    ctx_font_size(ctx, 20);
+    char buf[100];
+    sprintf(buf, "p:%i/%i l:%i lc:%i", self->pocketmod.pattern,
+            self->pocketmod.num_patterns, self->pocketmod.line,
+            self->pocketmod.loop_count);
+    ctx_move_to(ctx, -90, 0);
+    ctx_text(ctx, buf);
+    ctx_fill(ctx);
+}
+
+static void mod_think(st3m_media *media, float ms_elapsed) {
+    int samples_needed = (ms_elapsed / 1000.0) * 48000;
+    if (samples_needed > 1000) samples_needed = 1000;
+
+    float rendered[samples_needed * 2];
+    mod_state *self = (void *)media;
+    int rend = pocketmod_render(&self->pocketmod, rendered, sizeof(rendered));
+    for (int i = 0; i < rend / 4; i++) {
+        self->control.audio_buffer[self->control.audio_w++] =
+            rendered[i] * 20000;
+        if (self->control.audio_w >= AUDIO_BUF_SIZE) self->control.audio_w = 0;
+    }
+}
+
+static void mod_destroy(st3m_media *media) {
+    mod_state *self = (void *)media;
+    if (self->data) free(self->data);
+    free(self);
+}
+
+static int file_get_contents(const char *path, uint8_t **contents,
+                             size_t *length) {
+    FILE *file;
+    long size;
+    long remaining;
+    uint8_t *buffer;
+    file = fopen(path, "rb");
+    if (!file) {
+        return -1;
+    }
+    fseek(file, 0, SEEK_END);
+    size = remaining = ftell(file);
+
+    if (length) {
+        *length = size;
+    }
+    rewind(file);
+    buffer = malloc(size + 2);
+    if (!buffer) {
+        fclose(file);
+        return -1;
+    }
+    remaining -= fread(buffer, 1, remaining, file);
+    if (remaining) {
+        fclose(file);
+        free(buffer);
+        return -1;
+    }
+    fclose(file);
+    *contents = (unsigned char *)buffer;
+    buffer[size] = 0;
+    return 0;
+}
+
+st3m_media *st3m_media_load_mod(const char *path) {
+    mod_state *self = (mod_state *)malloc(sizeof(mod_state));
+    memset(self, 0, sizeof(mod_state));
+    self->control.draw = mod_draw;
+    self->control.think = mod_think;
+    self->control.destroy = mod_destroy;
+    file_get_contents(path, &self->data, &self->size);
+    if (!self->data ||
+        !pocketmod_init(&self->pocketmod, self->data, self->size, 48000)) {
+        printf("BOOO\n");
+        if (self->data) free(self->data);
+        free(self);
+        return NULL;
+    }
+
+    return (st3m_media *)self;
+}

components/audio_mp3/CMakeLists.txt

+idf_component_register(
+    SRCS
+        audio_mp3.c
+    INCLUDE_DIRS
+        .
+        ../ctx
+        ../st3m
+)

components/audio_mp3/LICENSE

+CC0 1.0 Universal
+
+Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of
+authorship and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific
+works ("Commons") that the public can reliably and without fear of later
+claims of infringement build upon, modify, incorporate in other works, reuse
+and redistribute as freely as possible in any form whatsoever and for any
+purposes, including without limitation commercial purposes. These owners may
+contribute to the Commons to promote the ideal of a free culture and the
+further production of creative, cultural and scientific works, or to gain
+reputation or greater distribution for their Work in part through the use and
+efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation
+of additional consideration or compensation, the person associating CC0 with a
+Work (the "Affirmer"), to the extent that he or she is an owner of Copyright
+and Related Rights in the Work, voluntarily elects to apply CC0 to the Work
+and publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+1. Copyright and Related Rights. A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and
+Related Rights"). Copyright and Related Rights include, but are not limited
+to, the following:
+
+  i. the right to reproduce, adapt, distribute, perform, display, communicate,
+  and translate a Work;
+
+  ii. moral rights retained by the original author(s) and/or performer(s);
+
+  iii. publicity and privacy rights pertaining to a person's image or likeness
+  depicted in a Work;
+
+  iv. rights protecting against unfair competition in regards to a Work,
+  subject to the limitations in paragraph 4(a), below;
+
+  v. rights protecting the extraction, dissemination, use and reuse of data in
+  a Work;
+
+  vi. database rights (such as those arising under Directive 96/9/EC of the
+  European Parliament and of the Council of 11 March 1996 on the legal
+  protection of databases, and under any national implementation thereof,
+  including any amended or successor version of such directive); and
+
+  vii. other similar, equivalent or corresponding rights throughout the world
+  based on applicable law or treaty, and any national implementations thereof.
+
+2. Waiver. To the greatest extent permitted by, but not in contravention of,
+applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+unconditionally waives, abandons, and surrenders all of Affirmer's Copyright
+and Related Rights and associated claims and causes of action, whether now
+known or unknown (including existing as well as future claims and causes of
+action), in the Work (i) in all territories worldwide, (ii) for the maximum
+duration provided by applicable law or treaty (including future time
+extensions), (iii) in any current or future medium and for any number of
+copies, and (iv) for any purpose whatsoever, including without limitation
+commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes
+the Waiver for the benefit of each member of the public at large and to the
+detriment of Affirmer's heirs and successors, fully intending that such Waiver
+shall not be subject to revocation, rescission, cancellation, termination, or
+any other legal or equitable action to disrupt the quiet enjoyment of the Work
+by the public as contemplated by Affirmer's express Statement of Purpose.
+
+3. Public License Fallback. Should any part of the Waiver for any reason be
+judged legally invalid or ineffective under applicable law, then the Waiver
+shall be preserved to the maximum extent permitted taking into account
+Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
+is so judged Affirmer hereby grants to each affected person a royalty-free,
+non transferable, non sublicensable, non exclusive, irrevocable and
+unconditional license to exercise Affirmer's Copyright and Related Rights in
+the Work (i) in all territories worldwide, (ii) for the maximum duration
+provided by applicable law or treaty (including future time extensions), (iii)
+in any current or future medium and for any number of copies, and (iv) for any
+purpose whatsoever, including without limitation commercial, advertising or
+promotional purposes (the "License"). The License shall be deemed effective as
+of the date CC0 was applied by Affirmer to the Work. Should any part of the
+License for any reason be judged legally invalid or ineffective under
+applicable law, such partial invalidity or ineffectiveness shall not
+invalidate the remainder of the License, and in such case Affirmer hereby
+affirms that he or she will not (i) exercise any of his or her remaining
+Copyright and Related Rights in the Work or (ii) assert any associated claims
+and causes of action with respect to the Work, in either case contrary to
+Affirmer's express Statement of Purpose.
+
+4. Limitations and Disclaimers.
+
+  a. No trademark or patent rights held by Affirmer are waived, abandoned,
+  surrendered, licensed or otherwise affected by this document.
+
+  b. Affirmer offers the Work as-is and makes no representations or warranties
+  of any kind concerning the Work, express, implied, statutory or otherwise,
+  including without limitation warranties of title, merchantability, fitness
+  for a particular purpose, non infringement, or the absence of latent or
+  other defects, accuracy, or the present or absence of errors, whether or not
+  discoverable, all to the greatest extent permissible under applicable law.
+
+  c. Affirmer disclaims responsibility for clearing rights of other persons
+  that may apply to the Work or any use thereof, including without limitation
+  any person's Copyright and Related Rights in the Work. Further, Affirmer
+  disclaims responsibility for obtaining any necessary consents, permissions
+  or other rights required for any use of the Work.
+
+  d. Affirmer understands and acknowledges that Creative Commons is not a
+  party to this document and has no duty or obligation with respect to this
+  CC0 or use of the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>
+

components/audio_mp3/audio_mp3.c

+#ifndef __clang__
+#pragma GCC optimize("O2")
+#endif
+
+#include <fcntl.h>
+#include <st3m_audio.h>
+#include <st3m_media.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include "lwip/igmp.h"
+#include "lwip/ip4.h"
+#include "lwip/netdb.h"
+#include "lwip/sockets.h"
+
+#include "ctx.h"
+
+#define MINIMP3_NONSTANDARD_BUT_LOGICAL
+#define MINIMP3_NO_SIMD
+#define MINIMP3_IMPLEMENTATION
+#include "minimp3.h"
+
+#define BUFFER_SIZE (32 * 1024)
+
+typedef struct {
+    st3m_media control;
+    mp3dec_t mp3d;
+    char *path;
+    char *artist;
+    char *title;
+    int year;
+    int started;
+    int samplerate;
+    int channels;
+    uint8_t *data;
+    size_t size;
+    size_t count;
+
+    int pos;
+    int offset;
+    int buffer_size;
+
+    int file_size;
+    FILE *file;
+
+    int socket;
+} mp3_state;
+
+static int has_data(mp3_state *mp3) {
+    fd_set rfds;
+    struct timeval tv = { 0, 0 };
+    FD_ZERO(&rfds);
+    FD_SET(mp3->socket, &rfds);
+    if (select(mp3->socket + 1, &rfds, NULL, NULL, &tv) == 1)
+        return FD_ISSET(mp3->socket, &rfds);
+    return 0;
+}
+
+static void mp3_fetch_data(mp3_state *mp3) {
+    if (mp3->pos + (16 * 1024) >= mp3->count) {
+        memmove(mp3->data, &mp3->data[mp3->pos], mp3->count - mp3->pos);
+        mp3->offset += mp3->pos;
+        mp3->count -= mp3->pos;
+        mp3->pos = 0;
+    }
+
+    if ((mp3->size - mp3->count > 0) && has_data(mp3)) {
+        int desire_bytes = (mp3->size - mp3->count);
+        if (desire_bytes) {
+            int read_bytes;
+            if (mp3->file)
+                read_bytes =
+                    fread(mp3->data + mp3->count, 1, desire_bytes, mp3->file);
+            else
+                read_bytes =
+                    read(mp3->socket, mp3->data + mp3->count, desire_bytes);
+            mp3->count += read_bytes;
+        }
+    }
+}
+
+static void mp3_draw(st3m_media *media, Ctx *ctx) {
+    mp3_state *self = (void *)media;
+#if 1
+    ctx_rectangle(ctx, -120, -120, 240, 240);
+    ctx_gray(ctx, 0);
+    ctx_fill(ctx);
+    ctx_rgb(ctx, 1.0, 1.0, 1.0);
+    ctx_rectangle(ctx, -120, 0, 240, 1);
+    ctx_rectangle(ctx, -120 + self->offset * 240.0 / self->file_size, -32, 2,
+                  64);
+    ctx_fill(ctx);
+    ctx_font_size(ctx, 24);
+    ctx_text_align(ctx, CTX_TEXT_ALIGN_CENTER);
+    ctx_move_to(ctx, 0, -40);
+    ctx_text(ctx, self->artist);
+    ctx_move_to(ctx, 0, 64);
+    ctx_text(ctx, self->title);
+#endif
+}
+
+static void mp3_think(st3m_media *media, float ms_elapsed) {
+    mp3_state *self = (void *)media;
+
+    mp3_fetch_data(self);
+
+    if (!self->started) {
+        self->started = 1;
+        mp3_think(media, 100);
+    }
+    int samples_needed =
+        ((AUDIO_BUF_SIZE - st3m_media_samples_queued()) / 2) - 2400;
+
+    int samples;
+    mp3dec_frame_info_t info = {
+        0,
+    };
+    if (samples_needed > 0 &&
+        ((self->offset + 512 < self->file_size) || (!self->file)))
+        do {
+            int16_t rendered[MINIMP3_MAX_SAMPLES_PER_FRAME];
+            samples =
+                mp3dec_decode_frame(&self->mp3d, self->data + self->pos,
+                                    self->count - self->pos, rendered, &info);
+            self->samplerate = info.hz;
+            self->channels = info.channels;
+            self->pos += info.frame_bytes;
+
+            if (self->samplerate != 48000) {
+                int phase = 0;
+                int fraction = ((48000.0 / self->samplerate) - 1.0) * 65536;
+                if (info.channels == 1)
+                    for (int i = 0; i < samples; i++) {
+                    again1:
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+                        phase += fraction;
+                        if (phase > 65536) {
+                            phase -= 65536;
+                            phase -= fraction;
+                            goto again1;
+                        }
+                    }
+                else if (info.channels == 2) {
+                    int phase = 0;
+                    for (int i = 0; i < samples; i++) {
+                    again2:
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i * 2];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i * 2 + 1];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+
+                        phase += fraction;
+                        if (phase > 65536) {
+                            phase -= 65536;
+                            phase -= fraction;
+                            goto again2;
+                        }
+                    }
+                }
+            } else {
+                if (info.channels == 1)
+                    for (int i = 0; i < samples; i++) {
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+                    }
+                else if (info.channels == 2) {
+                    for (int i = 0; i < samples; i++) {
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i * 2];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+                        self->control.audio_buffer[self->control.audio_w++] =
+                            rendered[i * 2 + 1];
+                        if (self->control.audio_w >= AUDIO_BUF_SIZE)
+                            self->control.audio_w = 0;
+                    }
+                }
+            }
+
+            samples_needed -= (samples);
+        } while (samples_needed > 0);
+}
+
+static void mp3_destroy(st3m_media *media) {
+    mp3_state *self = (void *)media;
+    if (self->data) free(self->data);
+    if (self->file) fclose(self->file);
+    if (self->path) free(self->path);
+    if (self->title) free(self->title);
+    if (self->artist) free(self->artist);
+    free(self);
+}
+
+typedef struct {
+    char tag[3];
+    char artist[30];
+    char title[30];
+    char year[4];
+    char pad[128];
+} id3tag_t;
+
+st3m_media *st3m_media_load_mp3(const char *path) {
+    mp3_state *self = (mp3_state *)malloc(sizeof(mp3_state));
+    id3tag_t id3;
+    memset(self, 0, sizeof(mp3_state));
+    self->control.draw = mp3_draw;
+    self->control.think = mp3_think;
+    self->control.destroy = mp3_destroy;
+    self->samplerate = 44100;
+    self->buffer_size = 32 * 1024;
+
+    printf("%i\n", __LINE__);
+    fflush(0);
+    sleep(1);
+    if (!strncmp(path, "http://", 7)) {
+        int port = 80;
+        char *hostname = strdup(path + 7);
+        char *rest = NULL;
+        self->buffer_size = 64 * 1024;
+        rest = strchr(hostname, '/') + 1;
+        strchr(hostname, '/')[0] = 0;
+        if (strchr(hostname, ':')) {
+            port = atoi(strchr(hostname, ':') + 1);
+            strchr(hostname, ':')[0] = 0;
+        }
+
+        fflush(0);
+        sleep(1);
+
+        struct hostent *host;
+        struct sockaddr_in addr;
+        self->socket = socket(PF_INET, SOCK_STREAM, 0);
+        if (self->socket < 0) {
+            free(hostname);
+            free(self);
+            return NULL;
+        }
+        memset(&addr, 0, sizeof(addr));
+        addr.sin_family = AF_INET;
+        addr.sin_port = htons(port);
+        host = gethostbyname(hostname);
+        if (!host) {
+            free(self);
+            free(hostname);
+            return NULL;
+        }
+        addr.sin_addr.s_addr = ((long unsigned int **)host->h_addr_list)[0][0];
+
+        if (connect(self->socket, (struct sockaddr *)&addr, sizeof(addr)) ==
+            0) {
+            char s[1024];
+            sprintf(s, "GET /%s HTTP/1.1\r\n", rest);
+            write(self->socket, s, strlen(s));
+            sprintf(s, "Range: bytes=0-\r\n");
+            write(self->socket, s, strlen(s));
+            if (hostname) {
+                sprintf(s, "Host: %s\r\n", hostname);
+                write(self->socket, s, strlen(s));
+            }
+            sprintf(s, "User-Agent: flow3r\r\n");
+            write(self->socket, s, strlen(s));
+            sprintf(s, "\r\n");
+            write(self->socket, s, strlen(s));
+            fsync(self->socket);
+
+            self->data = malloc(self->buffer_size);
+            self->size = self->buffer_size;
+
+            // prebuffer a full fill - at least we get to play this without
+            // glitches
+            do {
+                int desire = self->size - self->count;
+                if (desire > 4096) desire = 4096;
+                int count =
+                    read(self->socket, self->data + self->count, desire);
+                self->count += count;
+            } while (self->count < self->size);
+
+            mp3dec_init(&self->mp3d);
+            self->control.duration = 1200;
+            free(hostname);
+            return (st3m_media *)self;
+        }
+        free(hostname);
+
+        free(self);
+        return NULL;
+    }
+
+    self->file = fopen(path, "r");
+    fseek(self->file, 0, SEEK_END);
+    self->file_size = ftell(self->file);
+    fseek(self->file, self->file_size - 128, SEEK_SET);
+    fread(&id3, 128, 1, self->file);
+    if (id3.tag[0] == 'T' && id3.tag[1] == 'A' && id3.tag[2] == 'G') {
+        self->title = strndup(id3.title, 30);
+        while (self->title[strlen(self->title) - 1] == ' ')
+            self->title[strlen(self->title) - 1] = 0;
+        self->artist = strndup(id3.artist, 30);
+        while (self->artist[strlen(self->artist) - 1] == ' ')
+            self->artist[strlen(self->artist) - 1] = 0;
+        self->year = atoi(id3.year);
+    } else {
+        self->artist = "Anonymous";
+        self->title = strdup(strrchr(path, '/') + 1);
+    }
+    self->path = strdup(path);
+    rewind(self->file);
+
+    self->data = malloc(self->buffer_size);
+    self->size = self->buffer_size;
+    if (!self->file) {
+        printf("!!!!bo\n");
+        free(self);
+        return NULL;
+    }
+    mp3dec_init(&self->mp3d);
+    self->control.duration = 1200.0;
+    return (st3m_media *)self;
+}

components/micropython/usermodule/micropython.cmake

@@ -16,6 +16,7 @@ target_sources(usermod_badge23 INTERFACE
     ${CMAKE_CURRENT_LIST_DIR}/mp_sys_display.c
     ${CMAKE_CURRENT_LIST_DIR}/mp_sys_kernel.c
     ${CMAKE_CURRENT_LIST_DIR}/mp_uctx.c
+    ${CMAKE_CURRENT_LIST_DIR}/mp_media.c
 )
 
 target_include_directories(usermod_badge23 INTERFACE

components/micropython/usermodule/mp_media.c

+#include <st3m_media.h>
+
+#include "py/builtin.h"
+#include "py/runtime.h"
+
+typedef struct _mp_ctx_obj_t {
+    mp_obj_base_t base;
+    Ctx *ctx;
+    mp_obj_t user_data;
+} mp_ctx_obj_t;
+
+STATIC mp_obj_t mp_load(mp_obj_t path) {
+    return mp_obj_new_int(st3m_media_load(mp_obj_str_get_str(path)));
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_load_obj, mp_load);
+
+STATIC mp_obj_t mp_draw(mp_obj_t uctx_mp) {
+    mp_ctx_obj_t *uctx = MP_OBJ_TO_PTR(uctx_mp);
+    st3m_media_draw(uctx->ctx);
+    return 0;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_draw_obj, mp_draw);
+
+STATIC mp_obj_t mp_think(mp_obj_t ms_in) {
+    st3m_media_think(mp_obj_get_float(ms_in));
+    return 0;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_1(mp_think_obj, mp_think);
+
+STATIC mp_obj_t mp_stop(void) {
+    st3m_media_stop();
+    return 0;
+}
+STATIC MP_DEFINE_CONST_FUN_OBJ_0(mp_stop_obj, mp_stop);
+
+STATIC const mp_rom_map_elem_t globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR_draw), MP_ROM_PTR(&mp_draw_obj) },
+    { MP_ROM_QSTR(MP_QSTR_think), MP_ROM_PTR(&mp_think_obj) },
+    { MP_ROM_QSTR(MP_QSTR_stop), MP_ROM_PTR(&mp_stop_obj) },
+    { MP_ROM_QSTR(MP_QSTR_load), MP_ROM_PTR(&mp_load_obj) },
+};
+
+STATIC MP_DEFINE_CONST_DICT(globals, globals_table);
+
+const mp_obj_module_t mp_module_media = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&globals,
+};
+
+MP_REGISTER_MODULE(MP_QSTR_media, mp_module_media);

components/st3m/CMakeLists.txt

@@ -19,6 +19,7 @@ idf_component_register(
         st3m_usb_msc.c
         st3m_usb.c
         st3m_console.c
+        st3m_media.c
         st3m_mode.c
         st3m_captouch.c
         st3m_ringbuffer.c
@@ -40,6 +41,9 @@ idf_component_register(
         esp_timer
         esp_netif
         usb
+        video_mpeg
+        audio_mod
+        audio_mp3
 )
 
 idf_component_get_property(tusb_lib tinyusb COMPONENT_LIB)

components/st3m/st3m_media.c

+#include "st3m_media.h"
+#include "st3m_audio.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+
+#include "esp_log.h"
+#include "freertos/FreeRTOS.h"
+
+#ifdef CONFIG_FLOW3R_CTX_FLAVOUR_FULL
+static st3m_media *audio_media = NULL;
+
+// XXX should be refactored to either be temporary SPIRAM allocation
+// or a static shared global pcm queuing API
+//
+static int16_t *audio_buffer = NULL;
+
+void st3m_media_audio_out(int16_t *rx, int16_t *tx, uint16_t len) {
+    if (!audio_media) return;
+    for (int i = 0; i < len; i++) {
+        if ((audio_media->audio_r + 1 != audio_media->audio_w) &&
+            (audio_media->audio_r + 1 - AUDIO_BUF_SIZE !=
+             audio_media->audio_w)) {
+            tx[i] = audio_media->audio_buffer[audio_media->audio_r++];
+            if (audio_media->audio_r >= AUDIO_BUF_SIZE)
+                audio_media->audio_r = 0;
+        } else
+            tx[i] = 0;
+    }
+}
+int st3m_media_samples_queued(void) {
+    if (!audio_media) return 0;
+    if (audio_media->audio_r > audio_media->audio_w)
+        return (AUDIO_BUF_SIZE - audio_media->audio_r) + audio_media->audio_w;
+    return audio_media->audio_w - audio_media->audio_r;
+}
+
+void st3m_media_stop(void) {
+    if (audio_media && audio_media->destroy) audio_media->destroy(audio_media);
+    audio_media = 0;
+    st3m_audio_set_player_function(st3m_audio_player_function_dummy);
+    if (audio_buffer) {
+        free(audio_buffer);
+        audio_buffer = NULL;
+    }
+}
+
+void st3m_media_pause(void) {
+    if (!audio_media) return;
+    audio_media->paused = 1;
+}
+
+void st3m_media_play(void) {
+    if (!audio_media) return;
+    audio_media->paused = 0;
+}
+
+int st3m_media_is_playing(void) {
+    if (!audio_media) return 0;
+    return !audio_media->paused;
+}
+
+float st3m_media_get_duration(void) {
+    if (!audio_media) return 0;
+    return audio_media->duration;
+}
+
+float st3m_media_get_position(void) {
+    if (!audio_media) return 0;
+    return audio_media->position;
+}
+
+float st3m_media_get_time(void) {
+    if (!audio_media) return 0;
+    return audio_media->time;
+}
+
+void st3m_media_seek(float position) {
+    if (!audio_media) return;
+    audio_media->seek = position;
+}
+
+void st3m_media_seek_relative(float time) {
+    if (!audio_media) return;
+    st3m_media_seek((audio_media->position * audio_media->duration) + time);
+}
+
+void st3m_media_draw(Ctx *ctx) {
+    if (audio_media && audio_media->draw) audio_media->draw(audio_media, ctx);
+}
+
+void st3m_media_think(float ms) {
+    if (audio_media && audio_media->think) audio_media->think(audio_media, ms);
+}
+
+char *st3m_media_get_string(const char *key) {
+    if (!audio_media) return NULL;
+    if (!audio_media->get_string) return NULL;
+    return audio_media->get_string(audio_media, key);
+}
+
+float st3m_media_get(const char *key) {
+    if (!audio_media || !audio_media->get_string) return -1.0f;
+    return audio_media->get(audio_media, key);
+}
+
+void st3m_media_set(const char *key, float value) {
+    if (!audio_media || !audio_media->set) return;
+    return audio_media->set(audio_media, key, value);
+}
+
+st3m_media *st3m_media_load_mpg1(const char *path);
+st3m_media *st3m_media_load_mod(const char *path);
+st3m_media *st3m_media_load_mp3(const char *path);
+
+int st3m_media_load(const char *path) {
+    if (strstr(path, ".mpg")) {
+        st3m_media_stop();
+        audio_media = st3m_media_load_mpg1(path);
+    } else if (strstr(path, ".mod")) {
+        st3m_media_stop();
+        audio_media = st3m_media_load_mod(path);
+    } else if (strstr(path, "mp3") || !strncmp(path, "http://", 7)) {
+        st3m_media_stop();
+        audio_media = st3m_media_load_mp3(path);
+    }
+
+    if (!audio_media) return 0;
+
+    if (!audio_buffer)
+        audio_buffer = heap_caps_malloc(AUDIO_BUF_SIZE * 2, MALLOC_CAP_DMA);
+    st3m_audio_set_player_function(st3m_media_audio_out);
+    audio_media->audio_buffer = audio_buffer;
+    audio_media->audio_r = 0;
+    audio_media->audio_w = 1;
+
+    return 1;
+}
+
+#endif

components/st3m/st3m_media.h

+#pragma once
+
+#include <ctx.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+#define AUDIO_BUF_SIZE (8192)
+
+typedef struct _st3m_media st3m_media;
+
+struct _st3m_media {
+    // set a tunable, to a numeric value - available tunables depends on
+    // decoder
+    void (*set)(st3m_media *media, const char *key, float value);
+    // get a property/or tunable, defaulting to -1 for nonexisting keys
+    float (*get)(st3m_media *media, const char *key);
+    // get a string property/metadata, NULL if not existing or a string
+    // to be freed
+    char *(*get_string)(st3m_media *media, const char *key);
+
+    // free resources used by this media instance
+    void (*destroy)(st3m_media *media);
+
+    // draw the current frame / visualization / metadata screen
+    void (*draw)(st3m_media *media, Ctx *ctx);
+
+    // do decoding work corresponding to passed time
+    void (*think)(st3m_media *media, float ms);
+
+    // pointer to global pcm output buffer
+    int16_t *audio_buffer;
+    // playback head
+    int audio_r;
+    // queuing/writing head
+    int audio_w;
+
+    // Duration of media in seconds or -1 for infinite/streaming media
+    // at worst approximation of some unit, set by decoder.
+    float duration;
+
+    // currently played back position - set by decoder
+    float position;
+
+    // currently played back position, in seconds - set by decoder
+    float time;  // time might be precise even when duration is not
+                 // until first play through of some media, when time
+                 // duration should also be set exact.
+
+    // decoder should seek to this relative if not -1, and set it to -1
+    float seek;
+
+    // if set to 1 playback is momentarily stopped but can be resumed,
+    // this is toggled by st3m_media_play | st3m_media_pause
+    int paused;
+};
+
+// stops the currently playing media item
+void st3m_media_stop(void);
+// set a new media item
+int st3m_media_load(const char *path_or_uri);
+
+// decode current media item ms ahead (unless paused)
+void st3m_media_think(float ms);
+
+// draw the codecs own view of itself / its meta data - progress
+// for video or animations formats this should draw the media-content
+// other codecs can find mixes of debug visualizations.
+void st3m_media_draw(Ctx *ctx);
+
+// controls whether we are playing
+void st3m_media_pause(void);
+void st3m_media_play(void);
+int st3m_media_is_playing(void);
+
+// get duration in seconds
+float st3m_media_get_duration(void);
+// get current playback time in seconds
+float st3m_media_get_time(void);
+// get current playback position relative to overall duration
+float st3m_media_get_position(void);
+// seek to a position relative to overall duration
+void st3m_media_seek(float position);
+// seek a relative amount of seconds forward or with negative values back
+void st3m_media_seek_relative(float seconds_jump);
+
+// get decoder specific string or NULL if not existing, free returned value
+//  common values:
+//     "title"  "artist"
+char *st3m_media_get_string(const char *key);
+// get a decoder specific numeric value, defaulting to -1 for nonexisting values
+float st3m_media_get(const char *key);
+// set a decoder specific floating point value
+// example posible/or already used values:
+//    "scale"       0.0 - 1.0  - how large part of the screen to take up
+//    "grayscale"   0 or 1     - drop color bits for performance
+//    "smoothing"   0 or 1     - enable smooth texture scaling
+void st3m_media_set(const char *key, float value);
+
+// API for use in implementations
+// query how manu audio samples have been queued in the pcm output buffer
+int st3m_media_samples_queued(void);

components/video_mpeg/CMakeLists.txt

+idf_component_register(
+    SRCS
+        video_mpeg.c
+    INCLUDE_DIRS
+        .
+        ../ctx
+        ../st3m
+)

components/video_mpeg/video_mpeg.c

+#ifndef __clang__
+#pragma GCC optimize("O3")
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+#include <st3m_audio.h>
+#include <st3m_media.h>
+
+#include "ctx.h"
+
+#define PL_MPEG_IMPLEMENTATION
+#include "pl_mpeg.h"
+
+typedef struct {
+    st3m_media control;
+
+    plm_t *plm;
+    uint8_t *frame_data;
+    int width;
+    int height;
+    int frame_drop;
+    int sample_rate;
+    int frame_no;
+    int prev_frame_no;
+    int prev_prev_frame_no;
+    // last decoded frame contained chroma samples
+    // this allows us to take a grayscale fast-path
+    unsigned last_frame_chroma : 1;
+    unsigned color : 1;
+    // whether we smooth the video when scaling it up
+    unsigned smoothing : 1;
+    unsigned video : 1;
+    unsigned audio : 1;
+    unsigned loop : 1;
+    float scale;
+} mpg1_state;
+
+static void mpg1_on_video(plm_t *player, plm_frame_t *frame, void *user);
+static void mpg1_on_audio(plm_t *player, plm_samples_t *samples, void *user);
+
+static void mpg1_think(mpg1_state *self, float ms_elapsed) {
+    float elapsed_time = ms_elapsed / 1000.0;
+    double seek_to = -1;
+
+    if (self->control.seek >= 0.0) {
+        seek_to = self->control.seek * self->control.duration;
+        self->control.seek = -1;
+    }
+
+    if (elapsed_time > 1.0 / 25.0) {
+        elapsed_time = 1.0 / 25.0;
+    }
+
+    if (self->control.paused) elapsed_time = 0;
+
+    // Seek or advance decode
+    if (seek_to != -1) {
+        // XXX : clear queued audio
+        plm_seek(self->plm, seek_to, FALSE);
+    } else {
+        plm_decode(self->plm, elapsed_time);
+    }
+
+    if (plm_has_ended(self->plm)) {
+    }
+}
+
+static inline int memcpy_chroma(uint8_t *restrict target, uint8_t *restrict src,
+                                int count) {
+    int ret = 0;
+    for (int i = 0; i < count; i++) {
+        uint8_t val = src[i];
+        target[i] = val;
+        ret = (ret | (val != 128));
+    }
+    return ret;
+}
+
+static void mpg1_on_video(plm_t *mpeg, plm_frame_t *frame, void *user) {
+    mpg1_state *self = (mpg1_state *)user;
+
+    self->frame_no++;
+
+    self->width = frame->y.width;
+    self->height = frame->y.height;
+    memcpy(self->frame_data, frame->y.data, frame->y.width * frame->y.height);
+
+    if (self->color) {
+        /* copy u and v components */
+        self->last_frame_chroma = memcpy_chroma(
+            self->frame_data + frame->y.width * frame->y.height, frame->cb.data,
+            (frame->y.width / 2) * (frame->y.height / 2));
+        self->last_frame_chroma = memcpy_chroma(
+            self->frame_data + frame->y.width * frame->y.height +
+                (frame->y.width / 2) * (frame->y.height / 2),
+            frame->cr.data, (frame->y.width / 2) * (frame->y.height / 2));
+    }
+}
+
+static void mpg1_on_audio(plm_t *mpeg, plm_samples_t *samples, void *user) {
+    mpg1_state *mpg1 = user;
+    // if (self->control.paused) return;
+    if (mpg1->sample_rate == 44100) {
+        int phase = 0;
+        for (int i = 0; i < samples->count; i++) {
+        again:
+            mpg1->control.audio_buffer[mpg1->control.audio_w++] =
+                samples->interleaved[i * 2] * 20000;
+            if (mpg1->control.audio_w >= AUDIO_BUF_SIZE)
+                mpg1->control.audio_w = 0;
+            mpg1->control.audio_buffer[mpg1->control.audio_w++] =
+                samples->interleaved[i * 2 + 1] * 20000;
+            if (mpg1->control.audio_w >= AUDIO_BUF_SIZE)
+                mpg1->control.audio_w = 0;
+            phase += ((48000 / 44100.0) - 1.0) * 65536;
+            if (phase > 65536) {
+                phase -= 65536;
+                phase -= ((48000 / 44100.0) - 1.0) * 65536;
+                goto again;
+            }
+        }
+    } else
+        for (int i = 0; i < samples->count; i++) {
+            mpg1->control.audio_buffer[mpg1->control.audio_w++] =
+                samples->interleaved[i * 2] * 20000;
+            if (mpg1->control.audio_w >= AUDIO_BUF_SIZE)
+                mpg1->control.audio_w = 0;
+            mpg1->control.audio_buffer[mpg1->control.audio_w++] =
+                samples->interleaved[i * 2 + 1] * 20000;
+            if (mpg1->control.audio_w >= AUDIO_BUF_SIZE)
+                mpg1->control.audio_w = 0;
+        }
+}
+
+static void mpg1_draw(st3m_media *media, Ctx *ctx) {
+    mpg1_state *mpg1 = (mpg1_state *)media;
+
+    {
+        float dim = 240 * mpg1->scale;
+
+        if (mpg1->video) {
+            float scale = dim / mpg1->width;
+            float scaleh = dim / mpg1->height;
+            if (scaleh < scale) scale = scaleh;
+            char eid[16];
+            sprintf(eid, "%i", mpg1->frame_no);
+            if (mpg1->frame_no != mpg1->prev_frame_no) {
+                if (mpg1->frame_no <
+                    10) {  // ensure we've filled at least some complete frames
+                    ctx_rectangle(ctx, -120, -120, 240, 240);
+                    ctx_gray(ctx, 0.0);
+                    ctx_fill(ctx);
+                }
+                ctx_translate(ctx, -dim / 2, -dim / 2);
+                ctx_translate(ctx, (dim - mpg1->width * scale) / 2.0,
+                              (dim - mpg1->height * scale) / 2.0);
+                ctx_scale(ctx, scale, scale);
+                ctx_rectangle(ctx, 0, 0, dim, dim);
+                ctx_define_texture(ctx, eid, mpg1->width, mpg1->height,
+                                   mpg1->width,
+                                   mpg1->last_frame_chroma ? CTX_FORMAT_YUV420
+                                                           : CTX_FORMAT_GRAY8,
+                                   mpg1->frame_data, NULL);
+                ctx_image_smoothing(ctx, mpg1->smoothing);
+                ctx_compositing_mode(ctx, CTX_COMPOSITE_COPY);
+                ctx_fill(ctx);
+                char eid[16];
+                sprintf(eid, "%i", mpg1->prev_prev_frame_no);
+                ctx_drop_eid(ctx, eid);
+                mpg1->prev_prev_frame_no = mpg1->prev_frame_no;
+                mpg1->prev_frame_no = mpg1->frame_no;
+            } else {
+                // do nothing, keep display contents
+            }
+        } else {
+            ctx_rgb(ctx, 0.2, 0.3, 0.4);
+            ctx_fill(ctx);
+        }
+    }
+}
+
+static void mpg1_destroy(st3m_media *media) {
+    mpg1_state *self = (void *)media;
+    plm_destroy(self->plm);
+    free(self->frame_data);
+    free(self);
+}
+
+st3m_media *st3m_media_load_mpg1(const char *path) {
+    mpg1_state *self = (mpg1_state *)malloc(sizeof(mpg1_state));
+    memset(self, 0, sizeof(mpg1_state));
+    self->control.draw = mpg1_draw;
+    self->control.think = mpg1_think;
+    self->control.destroy = mpg1_destroy;
+
+    self->plm = plm_create_with_filename(path);
+    self->color = 1;
+    self->last_frame_chroma = 0;
+    self->prev_frame_no = 255;  // anything but 0
+    self->scale = 0.75;
+    self->audio = 1;
+    self->video = 1;
+    self->loop = 0;
+    self->smoothing = 0;
+    self->frame_drop = 1;
+    if (!self->plm) {
+        printf("Couldn't open %s", path);
+        free(self);
+        return NULL;
+    }
+
+    self->sample_rate = plm_get_samplerate(self->plm);
+    self->control.duration = plm_get_duration(self->plm);
+
+    plm_set_video_decode_callback(self->plm, mpg1_on_video, self);
+    plm_set_audio_decode_callback(self->plm, mpg1_on_audio, self);
+    plm_set_video_enabled(self->plm, self->video);
+
+    plm_set_loop(self->plm, self->loop);
+    plm_set_audio_enabled(self->plm, self->audio);
+    plm_set_audio_stream(self->plm, 0);
+
+    if (plm_get_num_audio_streams(self->plm) > 0) {
+        plm_set_audio_lead_time(self->plm, 0.05);
+    }
+
+    self->frame_data =
+        (uint8_t *)malloc(plm_get_width(self->plm) * plm_get_height(self->plm) *
+                          2);  // XXX : this is not quite riught
+
+    mpg1_think(self, 0);  // the frame is constructed in think
+    return (st3m_media *)self;
+}

tools/encode-mpg.sh

+#!/usr/bin/env bash
+set -e -x
+if (( $# != 2 )); then
+    >&2 echo "Usage: mpg1-encode <video-file> <output-file>"
+    exit
+fi
+
+ffmpeg -i $1 -vf scale=128:96 -c:v mpeg1video -b:v 96k -ac 1 -c:a mp2 -format mpeg -b:a 64k  -ar 48000  -r 24 $2 
+
+