py: Implement very simple frozen modules support.

Only modules (not packages) supported now. Source modules can be converted
to frozen module structures using tools/make-frozen.py script.

minimal/mpconfigport.h

@@ -32,6 +32,7 @@
 #define MICROPY_PY_IO               (0)
 #define MICROPY_PY_STRUCT           (0)
 #define MICROPY_PY_SYS              (0)
+#define MICROPY_MODULE_FROZEN       (0)
 #define MICROPY_CPYTHON_COMPAT      (0)
 #define MICROPY_LONGINT_IMPL        (MICROPY_LONGINT_IMPL_NONE)
 #define MICROPY_FLOAT_IMPL          (MICROPY_FLOAT_IMPL_NONE)

py/builtinimport.c

@@ -34,6 +34,7 @@
 #include "py/objmodule.h"
 #include "py/runtime.h"
 #include "py/builtin.h"
+#include "py/frozenmod.h"
 
 #if 0 // print debugging info
 #define DEBUG_PRINT (1)
@@ -109,9 +110,7 @@ STATIC mp_import_stat_t find_file(const char *file_str, uint file_len, vstr_t *d
 #endif
 }
 
-STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
-    // create the lexer
-    mp_lexer_t *lex = mp_lexer_new_from_file(vstr_str(file));
+STATIC void do_load_from_lexer(mp_obj_t module_obj, mp_lexer_t *lex, const char *fname) {
 
     if (lex == NULL) {
         // we verified the file exists using stat, but lexer could still fail
@@ -119,7 +118,7 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
             nlr_raise(mp_obj_new_exception_msg(&mp_type_ImportError, "module not found"));
         } else {
             nlr_raise(mp_obj_new_exception_msg_varg(&mp_type_ImportError,
-                "no module named '%s'", vstr_str(file)));
+                "no module named '%s'", fname));
         }
     }
 
@@ -133,6 +132,12 @@ STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
     mp_parse_compile_execute(lex, MP_PARSE_FILE_INPUT, mod_globals, mod_globals);
 }
 
+STATIC void do_load(mp_obj_t module_obj, vstr_t *file) {
+    // create the lexer
+    mp_lexer_t *lex = mp_lexer_new_from_file(vstr_str(file));
+    do_load_from_lexer(module_obj, lex, vstr_str(file));
+}
+
 mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
 #if DEBUG_PRINT
     DEBUG_printf("__import__:\n");
@@ -237,6 +242,15 @@ mp_obj_t mp_builtin___import__(mp_uint_t n_args, const mp_obj_t *args) {
     }
     DEBUG_printf("Module not yet loaded\n");
 
+    #if MICROPY_MODULE_FROZEN
+    mp_lexer_t *lex = mp_find_frozen_module(mod_str, mod_len);
+    if (lex != NULL) {
+        module_obj = mp_obj_new_module(MP_OBJ_QSTR_VALUE(module_name));
+        do_load_from_lexer(module_obj, lex, mod_str);
+        return module_obj;
+    }
+    #endif
+
     uint last = 0;
     VSTR_FIXED(path, MICROPY_ALLOC_PATH_MAX)
     module_obj = MP_OBJ_NULL;

py/frozenmod.c

+/*
+ * This file is part of the Micro Python project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2015 Paul Sokolovsky
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+#include <string.h>
+#include <stdint.h>
+
+#include "py/lexer.h"
+
+#if MICROPY_MODULE_FROZEN
+
+extern const uint16_t mp_frozen_sizes[];
+extern const char mp_frozen_content[];
+
+mp_lexer_t *mp_find_frozen_module(const char *str, int len) {
+    const uint16_t *sz_ptr = mp_frozen_sizes;
+    const char *s = mp_frozen_content;
+
+    while (*sz_ptr) {
+        int l = strlen(s);
+        if (l == len && !memcmp(str, s, l)) {
+            s += l + 1;
+            mp_lexer_t *lex = mp_lexer_new_from_str_len(MP_QSTR_, s, *sz_ptr, 0);
+            return lex;
+        }
+        s += l + 1 + *sz_ptr++;
+    }
+    return NULL;
+}
+
+#endif // MICROPY_MODULE_FROZEN

py/frozenmod.h

+/*
+ * This file is part of the Micro Python project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2014 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ */
+
+mp_lexer_t *mp_find_frozen_module(const char *str, int len);

py/mpconfig.h

@@ -328,6 +328,11 @@ typedef double mp_float_t;
 #define MICROPY_MODULE_WEAK_LINKS (0)
 #endif
 
+// Whether frozen modules are supported
+#ifndef MICROPY_MODULE_FROZEN
+#define MICROPY_MODULE_FROZEN (0)
+#endif
+
 // Whether you can override builtins in the builtins module
 #ifndef MICROPY_CAN_OVERRIDE_BUILTINS
 #define MICROPY_CAN_OVERRIDE_BUILTINS (0)

py/py.mk

@@ -112,6 +112,7 @@ PY_O_BASENAME = \
 	smallint.o \
 	pfenv.o \
 	pfenv_printf.o \
+	frozenmod.o \
 	../extmod/moductypes.o \
 	../extmod/modujson.o \
 	../extmod/modure.o \

tools/make-frozen.py

+#!/usr/bin/env python3
+#
+# Create frozen modules structure for MicroPython.
+#
+# Usage:
+#
+# Have a directory with modules to be frozen (only modules, not packages
+# supported so far):
+#
+# frozen/foo.py
+# frozen/bar.py
+#
+# Run script, passing path to the directory above:
+#
+# ./make-frozen.py frozen > frozen.c
+#
+# Include frozen.c in your build, having defined MICROPY_MODULE_FROZEN in
+# config.
+#
+import sys
+import os
+
+
+def module_name(f):
+    return f[:-len(".py")]
+
+modules = []
+
+for dirpath, dirnames, filenames in os.walk(sys.argv[1]):
+    for f in filenames:
+        st = os.stat(dirpath + "/" + f)
+        modules.append((f, st))
+
+print("#include <stdint.h>")
+print("const uint16_t mp_frozen_sizes[] = {")
+
+for f, st in modules:
+    print("%d," % st.st_size)
+
+print("0};")
+
+print("const char mp_frozen_content[] = {")
+for f, st in modules:
+    m = module_name(f)
+    print('"%s\\0"' % m)
+    data = open(sys.argv[1] + "/" + f).read()
+    data = repr(data)[1:-1]
+    data = data.replace('"', '\\"')
+    print('"%s"' % data)
+print("};")