Added memzip filesystem support for teensy

You can now append a zipfile (containining uncomressed python sources)
to the micropython.hex file.

Use MEMZIP_DIR=directory when you call make, or set that in your
environment to include a different tree of source files.

Added sample /boot.py, /src/main.py, /test.py and /src/test.py files.

Added run command so that you can execute scripts from REPL (until import is implemented).

Added build directory to .gitignore

.gitignore

@@ -19,3 +19,6 @@
 ######################
 *.swp
 
+# Build directory
+######################
+build/

stm/printf.c

@@ -221,10 +221,12 @@ void stdout_print_strn(void *data, const char *str, unsigned int len) {
     bool any = false;
 
     // TODO should have a setting for which USART port to send to
+#if 0 // if 0'd out so that we're not calling functions with the wrong arguments
     if (usart_is_enabled()) {
         usart_tx_strn_cooked(str, len);
         any = true;
     }
+#endif
 
     if (usb_vcp_is_enabled()) {
         usb_vcp_send_strn_cooked(str, len);

teensy/Makefile

@@ -33,7 +33,8 @@ SRC_C = \
 	lcd.c \
 	led.c \
 	lexerfatfs.c \
-	usart.c \
+	lexermemzip.c \
+	memzip.c \
 	usb.c \
 
 STM_SRC_C = \
@@ -60,9 +61,9 @@ OBJ = $(addprefix $(BUILD)/, $(SRC_C:.c=.o) $(STM_SRC_C:.c=.o) $(STM_SRC_S:.s=.o
 #LIB += -ltermcap
 
 all2: $(BUILD) hex
-hex: $(BUILD)/flash.hex
+hex: $(BUILD)/micropython-mz.hex
 
-post_compile: $(BUILD)/flash.hex
+post_compile: $(BUILD)/micropython-mz.hex
 	$(ECHO) "Preparing $@ for upload"
 	$(Q)$(TOOLS_PATH)/teensy_post_compile -file="$(basename $(<F))" -path="$(<D)" -tools="$(TOOLS_PATH)"
 
@@ -72,11 +73,19 @@ reboot:
 
 upload: post_compile reboot
 
-$(BUILD)/flash.elf: $(OBJ)
+$(BUILD)/micropython.elf: $(OBJ)
 	$(ECHO) "LINK $<"
 	$(Q)$(CC) $(LDFLAGS) -o "$@" -Wl,-Map,$(@:.elf=.map) $(OBJ) $(LIBS)
 	$(Q)$(SIZE) $@
 
+ifeq ($(MEMZIP_DIR),)
+MEMZIP_DIR = memzip_files
+endif
+
+$(BUILD)/micropython-mz.hex: $(BUILD)/micropython.hex $(shell find ${MEMZIP_DIR} -type f)
+	@$(ECHO) "Creating $@"
+	$(Q)./add-memzip.sh $< $@ ${MEMZIP_DIR}
+
 $(BUILD)/%.hex: $(BUILD)/%.elf
 	$(ECHO) "HEX $<"
 	$(Q)$(OBJCOPY) -O ihex -R .eeprom "$<" "$@"

teensy/add-memzip.sh

+#!/bin/bash
+
+if [ "$#" != 3 ]; then
+    echo "Usage: add-memzip.sh input.hex output.hex file-directory"
+    exit 1
+fi
+
+#set -x
+
+input_hex=$1
+output_hex=$2
+memzip_src_dir=$3
+
+input_bin=${input_hex}.bin
+output_bin=${output_hex}.bin
+zip_file=${output_hex}.zip
+zip_base=$(basename ${zip_file})
+zip_dir=$(dirname ${zip_file})
+abs_zip_dir=$(realpath ${zip_dir})
+
+rm -f ${zip_file}
+(cd ${memzip_src_dir}; zip -0 -r -D ${abs_zip_dir}/${zip_base} .)
+objcopy -I ihex -O binary ${input_hex} ${input_bin}
+cat ${input_bin} ${zip_file} > ${output_bin}
+objcopy -I binary -O ihex ${output_bin} ${output_hex}
+echo "Added ${memzip_src_dir} to ${input_hex} creating ${output_hex}"
+

teensy/lexermemzip.c

+#include <stdint.h>
+#include <stdlib.h>
+
+#include "misc.h"
+#include "lexer.h"
+#include "memzip.h"
+
+mp_lexer_t *mp_lexer_new_from_memzip_file(const char *filename)
+{
+    void *data;
+    size_t len;
+
+    if (memzip_locate(filename, &data, &len) != MZ_OK) {
+        return NULL;
+    }
+    return mp_lexer_new_from_str_len(filename, (const char *)data, (uint)len, 0);
+}
+

teensy/lexermemzip.h

+mp_lexer_t *mp_lexer_new_from_memzip_file(const char *filename);
+

teensy/main.c

@@ -8,6 +8,7 @@
 #include "mpconfig.h"
 #include "mpqstr.h"
 #include "lexer.h"
+#include "lexermemzip.h"
 #include "parse.h"
 #include "obj.h"
 #include "compile.h"
@@ -22,52 +23,17 @@
 
 extern uint32_t _heap_start;
 
-#ifdef USE_READLINE
-#include <readline/readline.h>
-#include <readline/history.h>
-#endif
+bool do_file(const char *filename);
 
-#if 0
-static char *str_join(const char *s1, int sep_char, const char *s2) {
-    int l1 = strlen(s1);
-    int l2 = strlen(s2);
-    char *s = m_new(char, l1 + l2 + 2);
-    memcpy(s, s1, l1);
-    if (sep_char != 0) {
-        s[l1] = sep_char;
-        l1 += 1;
+void flash_error(int n) {
+    for (int i = 0; i < n; i++) {
+        led_state(PYB_LED_BUILTIN, 1);
+        delay(250);
+        led_state(PYB_LED_BUILTIN, 0);
+        delay(250);
     }
-    memcpy(s + l1, s2, l2);
-    s[l1 + l2] = 0;
-    return s;
 }
 
-static char *prompt(char *p) {
-#ifdef USE_READLINE
-    char *line = readline(p);
-    if (line) {
-        add_history(line);
-    }
-#else
-    static char buf[256];
-    fputs(p, stdout);
-    char *s = fgets(buf, sizeof(buf), stdin);
-    if (!s) {
-        return NULL;
-    }
-    int l = strlen(buf);
-    if (buf[l - 1] == '\n') {
-        buf[l - 1] = 0;
-    } else {
-        l++;
-    }
-    char *line = m_new(char, l);
-    memcpy(line, buf, l);
-#endif
-    return line;
-}
-#endif
-
 static const char *help_text =
 "Welcome to Micro Python!\n\n"
 "This is a *very* early version of Micro Python and has minimal functionality.\n\n"
@@ -215,6 +181,19 @@ mp_obj_t pyb_hid_send_report(mp_obj_t arg) {
 }
 #endif
 
+static qstr pyb_config_source_dir = 0;
+static qstr pyb_config_main = 0;
+
+mp_obj_t pyb_source_dir(mp_obj_t source_dir) {
+    pyb_config_source_dir = mp_obj_get_qstr(source_dir);
+    return mp_const_none;
+}
+
+mp_obj_t pyb_main(mp_obj_t main) {
+    pyb_config_main = mp_obj_get_qstr(main);
+    return mp_const_none;
+}
+
 mp_obj_t pyb_delay(mp_obj_t count) {
     delay(mp_obj_get_int(count));
     return mp_const_none;
@@ -225,6 +204,12 @@ mp_obj_t pyb_led(mp_obj_t state) {
     return state;
 }
 
+mp_obj_t pyb_run(mp_obj_t filename_obj) {
+    const char *filename = qstr_str(mp_obj_get_qstr(filename_obj));
+    do_file(filename);
+    return mp_const_none;
+}
+
 char *strdup(const char *str) {
     uint32_t len = strlen(str);
     char *s2 = m_new(char, len + 1);
@@ -316,6 +301,39 @@ int readline(vstr_t *line, const char *prompt) {
     }
 }
 
+bool do_file(const char *filename) {
+    mp_lexer_t *lex = mp_lexer_new_from_memzip_file(filename);
+
+    if (lex == NULL) {
+        printf("could not open file '%s' for reading\n", filename);
+        return false;
+    }
+
+    mp_parse_node_t pn = mp_parse(lex, MP_PARSE_FILE_INPUT);
+    mp_lexer_free(lex);
+
+    if (pn == MP_PARSE_NODE_NULL) {
+        return false;
+    }
+
+    mp_obj_t module_fun = mp_compile(pn, false);
+    if (module_fun == mp_const_none) {
+        return false;
+    }
+
+    nlr_buf_t nlr;
+    if (nlr_push(&nlr) == 0) {
+        rt_call_function_0(module_fun);
+        nlr_pop();
+        return true;
+    } else {
+        // uncaught exception
+        mp_obj_print((mp_obj_t)nlr.ret_val);
+        printf("\n");
+        return false;
+    }
+}
+
 void do_repl(void) {
     stdout_tx_str("Micro Python for Teensy 3.1\r\n");
     stdout_tx_str("Type \"help()\" for more information.\r\n");
@@ -397,24 +415,53 @@ soft_reset:
     rt_init();
 
 #if 1
-    printf("About to add functions()\n");
     // add some functions to the python namespace
     {
         rt_store_name(qstr_from_str_static("help"), rt_make_function_0(pyb_help));
         mp_obj_t m = mp_obj_new_module(qstr_from_str_static("pyb"));
         rt_store_attr(m, qstr_from_str_static("info"), rt_make_function_0(pyb_info));
+        rt_store_attr(m, qstr_from_str_static("source_dir"), rt_make_function_1(pyb_source_dir));
+        rt_store_attr(m, qstr_from_str_static("main"), rt_make_function_1(pyb_main));
         rt_store_attr(m, qstr_from_str_static("gc"), rt_make_function_0(pyb_gc));
         rt_store_attr(m, qstr_from_str_static("delay"), rt_make_function_1(pyb_delay));
         rt_store_attr(m, qstr_from_str_static("led"), rt_make_function_1(pyb_led));
         rt_store_attr(m, qstr_from_str_static("Led"), rt_make_function_1(pyb_Led));
         rt_store_attr(m, qstr_from_str_static("gpio"), (mp_obj_t)&pyb_gpio_obj);
         rt_store_name(qstr_from_str_static("pyb"), m);
+        rt_store_name(qstr_from_str_static("run"), rt_make_function_1(pyb_run));
     }
 #endif
 
+    if (!do_file("/boot.py")) {
+        printf("Unable to open '/boot.py'\n");
+        flash_error(4);
+    }
+
     // Turn bootup LED off
     led_state(PYB_LED_BUILTIN, 0);
 
+    // run main script
+    {
+        vstr_t *vstr = vstr_new();
+        vstr_add_str(vstr, "/");
+        if (pyb_config_source_dir == 0) {
+            vstr_add_str(vstr, "src");
+        } else {
+            vstr_add_str(vstr, qstr_str(pyb_config_source_dir));
+        }
+        vstr_add_char(vstr, '/');
+        if (pyb_config_main == 0) {
+            vstr_add_str(vstr, "main.py");
+        } else {
+            vstr_add_str(vstr, qstr_str(pyb_config_main));
+        }
+        if (!do_file(vstr_str(vstr))) {
+            printf("Unable to open '%s'\n", vstr_str(vstr));
+            flash_error(3);
+        }
+        vstr_free(vstr);
+    }
+
     do_repl();
 
     printf("PYB: soft reboot\n");

teensy/memzip.c

+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include "memzip.h"
+
+extern uint8_t _staticfs[];
+
+MEMZIP_RESULT memzip_locate(const char *filename, void **data, size_t *len)
+{
+    const MEMZIP_FILE_HDR *file_hdr = (const MEMZIP_FILE_HDR *)_staticfs;
+    uint8_t *mem_data;
+
+    /* Zip file filenames don't have a leading /, so we strip it off */
+
+    if (*filename == '/') {
+        filename++;
+    }
+    while (file_hdr->signature == MEMZIP_FILE_HEADER_SIGNATURE) {
+        const char *file_hdr_filename = (const char *)&file_hdr[1];
+        mem_data = (uint8_t *)file_hdr_filename;
+        mem_data += file_hdr->filename_len;
+        mem_data += file_hdr->extra_len;
+        if (!strncmp(file_hdr_filename, filename, file_hdr->filename_len)) {
+            /* We found a match */
+            if (file_hdr->compression_method != 0) {
+                return MZ_FILE_COMPRESSED;
+            }
+
+            *data = mem_data;
+            *len = file_hdr->uncompressed_size;
+            return MZ_OK;
+        }
+        mem_data += file_hdr->uncompressed_size;
+        file_hdr = (const MEMZIP_FILE_HDR *)mem_data;
+    }
+    return MZ_NO_FILE;
+}

teensy/memzip.h

+#pragma pack(push, 1)
+
+#define MEMZIP_FILE_HEADER_SIGNATURE 0x04034b50
+typedef struct
+{
+    uint32_t    signature;
+    uint16_t    version;
+    uint16_t    flags;
+    uint16_t    compression_method;
+    uint16_t    last_mod_time;
+    uint16_t    last_mod_date;
+    uint32_t    crc32;
+    uint32_t    compressed_size;
+    uint32_t    uncompressed_size;
+    uint16_t    filename_len;
+    uint16_t    extra_len;
+
+    /* char     filename[filename_len]  */
+    /* uint8_t  extra[extra_len]        */
+
+} MEMZIP_FILE_HDR;
+
+#define MEMZIP_CENTRAL_DIRECTORY_SIGNATURE 0x02014b50
+typedef struct
+{
+    uint32_t    signature;
+    uint16_t    version_made_by;
+    uint16_t    version_read_with;
+    uint16_t    flags;
+    uint16_t    compression_method;
+    uint16_t    last_mod_time;
+    uint16_t    last_mod_date;
+    uint32_t    crc32;
+    uint32_t    compressed_size;
+    uint32_t    uncompressed_size;
+    uint16_t    filename_len;
+    uint16_t    extra_len;
+    uint16_t    disk_num;
+    uint16_t    internal_file_attributes;
+    uint32_t    external_file_attributes;
+    uint32_t    file_header_offset;
+
+    /* char     filename[filename_len]  */
+    /* uint8_t  extra[extra_len]        */
+
+} MEMZIP_CENTRAL_DIRECTORY_HDR;
+
+#define MEMZIP_END_OF_CENTRAL_DIRECTORY_SIGNATURE 0x06054b50
+typedef struct
+{
+    uint32_t    signature;
+    uint16_t    disk_num;
+    uint16_t    central_directory_disk;
+    uint16_t    num_central_directories_this_disk;
+    uint16_t    total_central_directories;
+    uint32_t    central_directory_size;
+    uint32_t    central_directory_offset;
+    uint16_t    comment_len;
+
+    /* char     comment[comment_len]    */
+
+} MEMZIP_END_OF_CENTRAL_DIRECTORY;
+
+#pragma pack(pop)
+
+typedef enum {
+    MZ_OK = 0,          /* (0) Succeeded */
+    MZ_NO_FILE,         /* (1) Could not find the file. */
+    MZ_FILE_COMPRESSED, /* (2) File is compressed (expecting uncompressed) */
+
+} MEMZIP_RESULT;
+
+MEMZIP_RESULT memzip_locate(const char *filename, void **data, size_t *len);

teensy/memzip_files/boot.py

+print("Executing boot.py")

teensy/memzip_files/src/main.py

+print("Executing main.py")
+
+x=pyb.led(1)
+pyb.delay(100)
+x=pyb.led(0)
+pyb.delay(100)
+x=pyb.led(1)
+pyb.delay(100)
+x=pyb.led(0)
+
+

teensy/memzip_files/src/test.py

+print("Executing /src/test.py")

teensy/memzip_files/test.py

+print("Executing /test.py")

teensy/mk20dx256.ld

@@ -126,6 +126,12 @@ SECTIONS
 		_edata = .; 
 	} > RAM
 
+	/*
+	 * _staticfs is the place in flash where the static filesystem which
+	 * is concatenated to the .hex file will wind up.
+	 */
+	_staticfs = LOADADDR(.data) + SIZEOF(.data);
+
 	.noinit (NOLOAD) : {
 		*(.noinit*)
 	} > RAM