st3m: casually implement shitty tar library

components/st3m/CMakeLists.txt

@@ -17,6 +17,7 @@ idf_component_register(
         st3m_mode.c
         st3m_captouch.c
         st3m_ringbuffer.c
+        st3m_tar.c
     INCLUDE_DIRS
         .
     REQUIRES
@@ -30,4 +31,4 @@ idf_component_register(
 )
 
 idf_component_get_property(tusb_lib tinyusb COMPONENT_LIB)
-target_include_directories(${tusb_lib} PRIVATE .)
\ No newline at end of file
+target_include_directories(${tusb_lib} PRIVATE .)

components/st3m/st3m_tar.c

+#include "st3m_tar.h"
+
+#include <errno.h>
+#include <string.h>
+#include <sys/stat.h>
+
+#include "esp_log.h"
+#include "miniz.h"
+
+static const char *TAG = "st3m-tar";
+
+void st3m_tar_parser_init(st3m_tar_parser_t *parser) {
+    memset(parser, 0, sizeof(st3m_tar_parser_t));
+    parser->in_header = true;
+}
+
+static void _header_done(st3m_tar_parser_t *parser) {
+    parser->in_header = false;
+    parser->in_file = true;
+    parser->file_offset = 0;
+    parser->file_size = 0;
+
+    parser->ustar = memmem(parser->header_current, 512, "ustar", 5) != NULL;
+
+    size_t size = 0;
+    sscanf((char *)parser->header_current + 124, "%o", &size);
+    parser->file_size = size;
+
+    size_t name_size = 100;
+    if (parser->ustar) {
+        name_size += 155;
+    }
+    // One extra byte for null termination.
+    name_size += 1;
+    char *name = calloc(1, name_size);
+    assert(name != NULL);
+
+    if (parser->ustar) {
+        const size_t prefix_offset = 345;
+
+        // Get size of right-padded name prefix.
+        size_t prefix_size =
+            strlen((char *)parser->header_current + prefix_offset);
+        if (prefix_size > 155) {
+            prefix_size = 155;
+        }
+        memcpy(name, parser->header_current + prefix_offset, prefix_size);
+    }
+    size_t suffix_size = strlen((char *)parser->header_current);
+    if (suffix_size > 100) {
+        suffix_size = 100;
+    }
+    memcpy(name + strlen(name), parser->header_current, suffix_size);
+
+    if (parser->ustar) {
+        parser->type = parser->header_current[156];
+    } else {
+        parser->type = '0';
+    }
+
+    if (parser->on_file_start != NULL) {
+        parser->on_file_start(parser->user, name, parser->file_size,
+                              parser->type);
+    }
+
+    free(name);
+}
+
+void st3m_tar_parser_feed_bytes(st3m_tar_parser_t *parser,
+                                const uint8_t *buffer, size_t bufsize) {
+    size_t skip = 1;
+    for (size_t i = 0; i < bufsize; i += skip) {
+        skip = 1;
+
+        if (parser->in_header) {
+            parser->header_current[parser->header_offset] = buffer[i];
+            parser->header_offset++;
+            if (parser->header_offset == 512) {
+                _header_done(parser);
+            }
+            continue;
+        }
+
+        if (parser->in_file) {
+            size_t nbytes = bufsize - i;
+            size_t file_left = parser->file_size - parser->file_offset;
+            if (nbytes > file_left) {
+                nbytes = file_left;
+            }
+            skip = nbytes;
+            if (parser->on_file_data != NULL) {
+                parser->on_file_data(parser->user, buffer + i, nbytes);
+            }
+
+            parser->file_offset += nbytes;
+
+            if (parser->file_offset >= parser->file_size) {
+                parser->in_file = false;
+                parser->padding_left = 512 - (parser->file_size % 512);
+                if (parser->padding_left == 512) {
+                    parser->in_header = true;
+                    parser->header_offset = 0;
+                } else {
+                    parser->in_padding = true;
+                }
+                if (parser->on_file_end != NULL) {
+                    parser->on_file_end(parser->user);
+                }
+            }
+            continue;
+        }
+
+        if (parser->in_padding) {
+            size_t nbytes = bufsize - i;
+            skip = parser->padding_left;
+            if (skip > nbytes) {
+                skip = nbytes;
+            }
+            parser->padding_left -= skip;
+            if (parser->padding_left <= 0) {
+                parser->in_padding = false;
+                parser->in_header = true;
+                parser->header_offset = 0;
+            }
+        }
+    }
+}
+
+static int _decompress_cb(const void *buf, int len, void *arg) {
+    st3m_tar_parser_t *parser = (st3m_tar_parser_t *)arg;
+    st3m_tar_parser_feed_bytes(parser, (const uint8_t *)buf, len);
+    return 1;
+}
+
+// From miniz, which otherwise lives in ROM, but this function doesn't.
+static int _tinfl_decompress_mem_to_callback(
+    const void *pIn_buf, size_t *pIn_buf_size,
+    tinfl_put_buf_func_ptr pPut_buf_func, void *pPut_buf_user, int flags) {
+    int result = 0;
+    tinfl_decompressor *decomp = calloc(1, sizeof(tinfl_decompressor));
+    if (decomp == NULL) {
+        return TINFL_STATUS_FAILED;
+    }
+
+    mz_uint8 *pDict = (mz_uint8 *)malloc(TINFL_LZ_DICT_SIZE);
+    if (pDict == 0) {
+        free(decomp);
+        return TINFL_STATUS_FAILED;
+    }
+
+    size_t in_buf_ofs = 0, dict_ofs = 0;
+    memset(pDict, 0, TINFL_LZ_DICT_SIZE);
+    tinfl_init(decomp);
+    for (;;) {
+        size_t in_buf_size = *pIn_buf_size - in_buf_ofs,
+               dst_buf_size = TINFL_LZ_DICT_SIZE - dict_ofs;
+        tinfl_status status = tinfl_decompress(
+            decomp, (const mz_uint8 *)pIn_buf + in_buf_ofs, &in_buf_size, pDict,
+            pDict + dict_ofs, &dst_buf_size,
+            (flags & ~(TINFL_FLAG_HAS_MORE_INPUT |
+                       TINFL_FLAG_USING_NON_WRAPPING_OUTPUT_BUF)));
+        in_buf_ofs += in_buf_size;
+        if ((dst_buf_size) &&
+            (!(*pPut_buf_func)(pDict + dict_ofs, (int)dst_buf_size,
+                               pPut_buf_user)))
+            break;
+        if (status != TINFL_STATUS_HAS_MORE_OUTPUT) {
+            result = (status == TINFL_STATUS_DONE);
+            break;
+        }
+        dict_ofs = (dict_ofs + dst_buf_size) & (TINFL_LZ_DICT_SIZE - 1);
+    }
+    free(decomp);
+    free(pDict);
+    *pIn_buf_size = in_buf_ofs;
+    return result;
+}
+
+bool st3m_tar_parser_run_zlib(st3m_tar_parser_t *parser, const uint8_t *buffer,
+                              size_t bufsize) {
+    int ret = _tinfl_decompress_mem_to_callback(
+        buffer, &bufsize, _decompress_cb, parser, TINFL_FLAG_PARSE_ZLIB_HEADER);
+    return ret == 1;
+}
+
+static int _mkpath(char *file_path, mode_t mode) {
+    assert(file_path && *file_path);
+    for (char *p = strchr(file_path + 1, '/'); p; p = strchr(p + 1, '/')) {
+        *p = '\0';
+        if (mkdir(file_path, mode) == -1) {
+            if (errno != EEXIST) {
+                *p = '/';
+                return -1;
+            }
+        }
+        *p = '/';
+    }
+    return 0;
+}
+
+static void _extractor_on_file_start(void *user, const char *name, size_t size,
+                                     char type) {
+    st3m_tar_extractor_t *extractor = (st3m_tar_extractor_t *)user;
+    if (extractor->cur_file != NULL) {
+        fclose(extractor->cur_file);
+        extractor->cur_file = NULL;
+    }
+
+    if (size == 0 || type != '0') {
+        return;
+    }
+
+    char *path = malloc(256);
+    assert(path != NULL);
+    const char *root = "";
+    if (extractor->root != NULL) {
+        root = extractor->root;
+    }
+    snprintf(path, 256, "%s%s", root, name);
+    ESP_LOGI(TAG, "Writing %s...", path);
+    if (_mkpath(path, 0755) != 0) {
+        ESP_LOGW(TAG, "Failed to create parent directory for %s: %s", path,
+                 strerror(errno));
+    }
+    extractor->cur_file = fopen(path, "w");
+    if (extractor->cur_file == NULL) {
+        ESP_LOGW(TAG, "Failed to create %s: %s", path, strerror(errno));
+    }
+
+    if (extractor->on_file != NULL) {
+        extractor->on_file(path);
+    }
+    free(path);
+}
+
+static void _extractor_on_file_data(void *user, const uint8_t *buffer,
+                                    size_t bufsize) {
+    st3m_tar_extractor_t *extractor = (st3m_tar_extractor_t *)user;
+    if (extractor->cur_file == NULL) {
+        return;
+    }
+    fwrite(buffer, 1, bufsize, extractor->cur_file);
+}
+
+static void _extractor_on_file_end(void *user) {
+    st3m_tar_extractor_t *extractor = (st3m_tar_extractor_t *)user;
+    if (extractor->cur_file == NULL) {
+        return;
+    }
+    fclose(extractor->cur_file);
+    extractor->cur_file = NULL;
+}
+
+void st3m_tar_extractor_init(st3m_tar_extractor_t *extractor) {
+    memset(extractor, 0, sizeof(st3m_tar_extractor_t));
+    st3m_tar_parser_init(&extractor->parser);
+    extractor->parser.on_file_start = _extractor_on_file_start;
+    extractor->parser.on_file_data = _extractor_on_file_data;
+    extractor->parser.on_file_end = _extractor_on_file_end;
+    extractor->parser.user = extractor;
+}

components/st3m/st3m_tar.h

+#pragma once
+
+// Trivial and likely very buggy tar (Tape ARchive) extraction library.
+//
+// Supports plain tar and tars compressed with zlib (which isn't the same as a
+// .tar.gz!). Can do minimal ustar detection and avoidance.
+//
+// Stream-based for minimal RAM usage.
+
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+// Main tar parser/extractor structure. Must be initialized with
+// st3m_tar_parser_init, then callbacks may be set before feeding it data.
+typedef struct {
+    // Callback functions for when the tar parser makes progress.
+    void *user;
+    void (*on_file_start)(void *user, const char *name, size_t size, char type);
+    void (*on_file_data)(void *user, const uint8_t *buffer, size_t bufsize);
+    void (*on_file_end)(void *user);
+
+    // Private fields follow.
+
+    // FSM state: in header.
+    bool in_header;
+    size_t header_offset;
+    uint8_t header_current[512];
+    bool ustar;
+    char type;
+
+    // FSM state: in file.
+    bool in_file;
+    size_t file_offset;
+    size_t file_size;
+
+    // FSM state: in padding.
+    bool in_padding;
+    size_t padding_left;
+} st3m_tar_parser_t;
+
+// Tar extractor which writes into a filesystem. Not secure.
+//
+// Must be initialized using st3m_tar_extractor_init. Then, optional fields may
+// be set and the parser may be fed data.
+typedef struct {
+    // If set, file paths will be prefixed with this root before extraction.
+    const char *root;
+
+    // If set, will be called any file begins extraction. Useful for feedback.
+    void (*on_file)(const char *name);
+
+    FILE *cur_file;
+    st3m_tar_parser_t parser;
+} st3m_tar_extractor_t;
+
+// Initialize parser object.
+void st3m_tar_parser_init(st3m_tar_parser_t *parser);
+
+// Feed bytes from packed tar into parser.
+void st3m_tar_parser_feed_bytes(st3m_tar_parser_t *parser,
+                                const uint8_t *buffer, size_t bufsize);
+
+// Feed a zlib'd tar package from a continuous memory into parser.
+bool st3m_tar_parser_run_zlib(st3m_tar_parser_t *parser, const uint8_t *buffer,
+                              size_t bufsize);
+
+// Initialize extractor object.
+void st3m_tar_extractor_init(st3m_tar_extractor_t *extractor);
\ No newline at end of file