lexer, vstr: Add unicode support.

py/lexer.c

@@ -502,19 +502,32 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
                             case 'v': c = 0x0b; break;
                             case 'f': c = 0x0c; break;
                             case 'r': c = 0x0d; break;
+                            case 'u':
+                            case 'U':
+                                if (is_bytes) {
+                                    // b'\u1234' == b'\\u1234'
+                                    vstr_add_char(&lex->vstr, '\\');
+                                    break;
+                                }
+                                // Otherwise fall through.
                             case 'x':
                             {
                                 uint num = 0;
-                                if (!get_hex(lex, 2, &num)) {
+                                if (!get_hex(lex, (c == 'x' ? 2 : c == 'u' ? 4 : 8), &num)) {
                                     // TODO error message
                                     assert(0);
                                 }
                                 c = num;
                                 break;
                             }
-                            case 'N': break; // TODO \N{name} only in strings
-                            case 'u': break; // TODO \uxxxx only in strings
-                            case 'U': break; // TODO \Uxxxxxxxx only in strings
+                            case 'N':
+                                // Supporting '\N{LATIN SMALL LETTER A}' == 'a' would require keeping the
+                                // entire Unicode name table in the core. As of Unicode 6.3.0, that's nearly
+                                // 3MB of text; even gzip-compressed and with minimal structure, it'll take
+                                // roughly half a meg of storage. This form of Unicode escape may be added
+                                // later on, but it's definitely not a priority right now. -- CJA 20140607
+                                assert(!"Unicode name escapes not supported");
+                                break;
                             default:
                                 if (c >= '0' && c <= '7') {
                                     // Octal sequence, 1-3 chars
@@ -533,7 +546,13 @@ STATIC void mp_lexer_next_token_into(mp_lexer_t *lex, mp_token_t *tok, bool firs
                         }
                     }
                     if (c != MP_LEXER_CHAR_EOF) {
-                        vstr_add_char(&lex->vstr, c);
+                        if (c < 0x110000 && !is_bytes) {
+                            vstr_add_char(&lex->vstr, c);
+                        } else if (c < 0x100 && is_bytes) {
+                            vstr_add_byte(&lex->vstr, c);
+                        } else {
+                            assert(!"TODO: Throw an error, invalid escape code probably");
+                        }
                     }
                 } else {
                     vstr_add_char(&lex->vstr, CUR_CHAR(lex));

py/vstr.c

@@ -199,12 +199,40 @@ void vstr_add_byte(vstr_t *vstr, byte b) {
 }
 
 void vstr_add_char(vstr_t *vstr, unichar c) {
-    // TODO UNICODE
-    byte *buf = (byte*)vstr_add_len(vstr, 1);
-    if (buf == NULL) {
-        return;
+    // TODO: Can this be simplified and deduplicated?
+    // Is it worth just calling vstr_add_len(vstr, 4)?
+    if (c < 0x80) {
+        byte *buf = (byte*)vstr_add_len(vstr, 1);
+        if (buf == NULL) {
+            return;
+        }
+        *buf = (byte)c;
+    } else if (c < 0x800) {
+        byte *buf = (byte*)vstr_add_len(vstr, 2);
+        if (buf == NULL) {
+            return;
+        }
+        buf[0] = (c >> 6) | 0xC0;
+        buf[1] = (c & 0x3F) | 0x80;
+    } else if (c < 0x10000) {
+        byte *buf = (byte*)vstr_add_len(vstr, 3);
+        if (buf == NULL) {
+            return;
+        }
+        buf[0] = (c >> 12) | 0xE0;
+        buf[1] = ((c >> 6) & 0x3F) | 0x80;
+        buf[2] = (c & 0x3F) | 0x80;
+    } else {
+        assert(c < 0x110000);
+        byte *buf = (byte*)vstr_add_len(vstr, 4);
+        if (buf == NULL) {
+            return;
+        }
+        buf[0] = (c >> 18) | 0xF0;
+        buf[1] = ((c >> 12) & 0x3F) | 0x80;
+        buf[2] = ((c >> 6) & 0x3F) | 0x80;
+        buf[3] = (c & 0x3F) | 0x80;
     }
-    buf[0] = c;
 }
 
 void vstr_add_str(vstr_t *vstr, const char *str) {