diff --git a/py/makeqstrdefs.py b/py/makeqstrdefs.py
index de0fefe5b505659eb8d14b0e53fcd8f3b1738d15..35d7658e902943e500d662a85a0c57176315bb60 100644
--- a/py/makeqstrdefs.py
+++ b/py/makeqstrdefs.py
@@ -20,39 +20,85 @@ def debug(message):
pass
+def write_out(fname, output):
+ if output:
+ fname = fname.replace("/", "__").replace("..", "@@")
+ with open(args.output_dir + "/" + fname + ".qstr", "w") as f:
+ f.write("\n".join(output) + "\n")
+
def process_file(f):
output = []
+ last_fname = None
+ outf = None
for line in f:
+ if line and line[0] == "#":
+ comp = line.split()
+ fname = comp[2]
+ assert fname[0] == '"' and fname[-1] == '"'
+ fname = fname[1:-1]
+ if fname[0] == "/" or not fname.endswith(".c"):
+ continue
+ if fname != last_fname:
+ write_out(last_fname, output)
+ output = []
+ last_fname = fname
+ continue
for match in re.findall(r'MP_QSTR_[_a-zA-Z0-9]+', line):
name = match.replace('MP_QSTR_', '')
if name not in QSTRING_BLACK_LIST:
output.append('Q(' + name + ')')
- # make sure there is a newline at the end of the output
- output.append('')
-
- return '\n'.join(output)
+ write_out(last_fname, output)
+ return ""
+
+
+def cat_together():
+ import glob
+ import hashlib
+ hasher = hashlib.md5()
+ all_lines = []
+ outf = open(args.output_dir + "/out", "wb")
+ for fname in glob.glob(args.output_dir + "/*.qstr"):
+ with open(fname, "rb") as f:
+ lines = f.readlines()
+ all_lines += lines
+ all_lines.sort()
+ all_lines = b"\n".join(all_lines)
+ outf.write(all_lines)
+ outf.close()
+ hasher.update(all_lines)
+ new_hash = hasher.hexdigest()
+ #print(new_hash)
+ old_hash = None
+ try:
+ with open(args.output_file + ".hash") as f:
+ old_hash = f.read()
+ except IOError:
+ pass
+ if old_hash != new_hash:
+ print("QSTR updated")
+ os.rename(args.output_dir + "/out", args.output_file)
+ with open(args.output_file + ".hash", "w") as f:
+ f.write(new_hash)
+ else:
+ print("QSTR not updated")
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Generates qstr definitions from a specified source')
- parser.add_argument('-o', '--output-file', dest='output_filename',
- help='Output filename (defaults to stdout)')
- parser.add_argument('input_filename', nargs='?',
- help='Name of the input file (when not specified, the script reads standard input')
- parser.add_argument('-s', '--skip-write-when-same', dest='skip_write_when_same',
- action='store_true', default=False,
- help="Don't write the output file if it already exists and the contents have not changed (disabled by default)")
+ parser.add_argument('input_filename',
+ help='Name of the input file (when not specified, the script reads standard input)')
+ parser.add_argument('output_dir',
+ help='Output directory to store individual qstr files')
+ parser.add_argument('output_file',
+ help='Name of the output file with collected qstrs')
args = parser.parse_args()
-
- # Check if the file contents changed from last time
- write_file = True
-
- # By default write into STDOUT
- outfile = sys.stdout
- real_output_filename = 'STDOUT'
+ try:
+ os.makedirs(args.output_dir)
+ except OSError:
+ pass
if args.input_filename:
infile = open(args.input_filename, 'r')
@@ -61,24 +107,4 @@ if __name__ == "__main__":
file_data = process_file(infile)
infile.close()
-
- # Detect custom output file name
- if args.output_filename:
- real_output_filename = args.output_filename
- if os.path.isfile(args.output_filename) and args.skip_write_when_same:
- with open(args.output_filename, 'r') as f:
- existing_data = f.read()
- if existing_data == file_data:
- debug("Skip regeneration of: %s\n" % real_output_filename)
- write_file = False
- else:
- debug("File HAS changed, overwriting\n")
- outfile = open(args.output_filename, 'w')
- else:
- outfile = open(args.output_filename, 'w')
-
- # Only write the file if we the data has changed
- if write_file:
- sys.stderr.write("QSTR %s\n" % real_output_filename)
- outfile.write(file_data)
- outfile.close()
+ cat_together()
diff --git a/py/mkrules.mk b/py/mkrules.mk
index 8ab06cfdc49036c5078bc5bd4054ef1bf84c410a..9a739b8b7bea74af5a826e31f83fc49c6e4c9b6e 100644
--- a/py/mkrules.mk
+++ b/py/mkrules.mk
@@ -52,7 +52,7 @@ EMPTY_QSTRDEFS_GENERATED_H = $(BUILD)/tmp/genhdr/qstrdefs.generated.h
# List all native flags since the current build system doesn't have
# the micropython configuration available. However, these flags are
# needed to extract all qstrings
-QSTR_GEN_EXTRA_CFLAGS += -P -DN_X64 -DN_X86 -DN_THUMB -DN_ARM
+QSTR_GEN_EXTRA_CFLAGS += -D__QSTR_EXTRACT -DN_X64 -DN_X86 -DN_THUMB -DN_ARM
QSTR_GEN_EXTRA_CFLAGS += -I$(BUILD)/tmp
vpath %.c . $(TOP)
@@ -81,13 +81,13 @@ $(EMPTY_QSTRDEFS_GENERATED_H):
# to get built before we try to compile any of them.
$(OBJ): | $(HEADER_BUILD)/qstrdefs.generated.h $(HEADER_BUILD)/mpversion.h
-# This rule joins all generated qstr files
-$(QSTR_DEFS_COLLECTED): $(addprefix $(HEADER_BUILD)/,$(addsuffix .qstr,$(SRC_QSTR)))
+$(HEADER_BUILD)/qstr.i.last: $(SRC_QSTR) | $(HEADER_BUILD)/mpversion.h
$(ECHO) "GEN $@"
- $(Q)cat $^ > $@
+ $(Q)$(CPP) $(QSTR_GEN_EXTRA_CFLAGS) $(CFLAGS) $? >$(HEADER_BUILD)/qstr.i.last
-
-#
+$(QSTR_DEFS_COLLECTED): $(HEADER_BUILD)/qstr.i.last
+ $(ECHO) "GEN $@"
+ $(Q)$(PYTHON) $(PY_SRC)/makeqstrdefs.py $(HEADER_BUILD)/qstr.i.last $(HEADER_BUILD)/qstr $(QSTR_DEFS_COLLECTED)
# $(sort $(var)) removes duplicates
#
diff --git a/py/qstr.c b/py/qstr.c
index ef31a682ee8e43e53a3f7a9e58a2f3ffe583bb82..24793ff8a345dea85532ea65879a53b20e8ecc67 100644
--- a/py/qstr.c
+++ b/py/qstr.c
@@ -93,9 +93,11 @@ const qstr_pool_t mp_qstr_const_pool = {
10, // set so that the first dynamically allocated pool is twice this size; must be <= the len (just below)
MP_QSTRnumber_of, // corresponds to number of strings in array just below
{
+#ifndef __QSTR_EXTRACT
#define QDEF(id, str) str,
#include "genhdr/qstrdefs.generated.h"
#undef QDEF
+#endif
},
};
diff --git a/py/qstr.h b/py/qstr.h
index b5c261f0d57ea4284815390fd1340ba6a1f7b149..1989ebe962f74f64869afac98257718deb767522 100644
--- a/py/qstr.h
+++ b/py/qstr.h
@@ -37,9 +37,11 @@
// first entry in enum will be MP_QSTR_NULL=0, which indicates invalid/no qstr
enum {
+#ifndef __QSTR_EXTRACT
#define QDEF(id, str) id,
#include "genhdr/qstrdefs.generated.h"
#undef QDEF
+#endif
MP_QSTRnumber_of, // no underscore so it can't clash with any of the above
};