Merge pull request #383 from pfalcon/yield-from

Implement "yield from"

py/vm.c

@@ -9,6 +9,7 @@
 #include "runtime.h"
 #include "bc0.h"
 #include "bc.h"
+#include "objgenerator.h"
 
 // Value stack grows up (this makes it incompatible with native C stack, but
 // makes sure that arguments to functions are in natural order arg1..argN
@@ -146,7 +147,9 @@ outer_dispatch_loop:
             // If we have exception to inject, now that we finish setting up
             // execution context, raise it. This works as if RAISE_VARARGS
             // bytecode was executed.
-            if (inject_exc != MP_OBJ_NULL) {
+            // Injecting exc into yield from generator is a special case,
+            // handled by MP_BC_YIELD_FROM itself
+            if (inject_exc != MP_OBJ_NULL && *ip != MP_BC_YIELD_FROM) {
                 mp_obj_t t = inject_exc;
                 inject_exc = MP_OBJ_NULL;
                 nlr_jump(rt_make_raise_obj(t));
@@ -634,12 +637,65 @@ unwind_return:
                         nlr_jump(rt_make_raise_obj(obj1));
 
                     case MP_BC_YIELD_VALUE:
+yield:
                         nlr_pop();
                         *ip_in_out = ip;
                         *sp_in_out = sp;
                         *exc_sp_in_out = MP_TAGPTR_MAKE(exc_sp, currently_in_except_block);
                         return MP_VM_RETURN_YIELD;
 
+                    case MP_BC_YIELD_FROM:
+                    {
+//#define EXC_MATCH(exc, type) MP_OBJ_IS_TYPE(exc, type)
+#define EXC_MATCH(exc, type) mp_obj_exception_match(exc, type)
+#define GENERATOR_EXIT_IF_NEEDED(t) if (t != MP_OBJ_NULL && EXC_MATCH(t, &mp_type_GeneratorExit)) { nlr_jump(t); }
+                        mp_vm_return_kind_t ret_kind;
+                        obj1 = POP();
+                        mp_obj_t t_exc = MP_OBJ_NULL;
+                        if (inject_exc != MP_OBJ_NULL) {
+                            t_exc = inject_exc;
+                            inject_exc = MP_OBJ_NULL;
+                            ret_kind = mp_obj_gen_resume(TOP(), mp_const_none, t_exc, &obj2);
+                        } else {
+                            ret_kind = mp_obj_gen_resume(TOP(), obj1, MP_OBJ_NULL, &obj2);
+                        }
+
+                        if (ret_kind == MP_VM_RETURN_YIELD) {
+                            ip--;
+                            PUSH(obj2);
+                            goto yield;
+                        }
+                        if (ret_kind == MP_VM_RETURN_NORMAL) {
+                            // Pop exhausted gen
+                            sp--;
+                            if (obj2 == MP_OBJ_NULL) {
+                                // Optimize StopIteration
+                                // TODO: get StopIteration's value
+                                PUSH(mp_const_none);
+                            } else {
+                                PUSH(obj2);
+                            }
+
+                            // If we injected GeneratorExit downstream, then even
+                            // if it was swallowed, we re-raise GeneratorExit
+                            GENERATOR_EXIT_IF_NEEDED(t_exc);
+                            break;
+                        }
+                        if (ret_kind == MP_VM_RETURN_EXCEPTION) {
+                            // Pop exhausted gen
+                            sp--;
+                            if (EXC_MATCH(obj2, &mp_type_StopIteration)) {
+                                PUSH(mp_obj_exception_get_value(obj2));
+                                // If we injected GeneratorExit downstream, then even
+                                // if it was swallowed, we re-raise GeneratorExit
+                                GENERATOR_EXIT_IF_NEEDED(t_exc);
+                                break;
+                            } else {
+                                nlr_jump(obj2);
+                            }
+                        }
+                    }
+
                     case MP_BC_IMPORT_NAME:
                         DECODE_QSTR;
                         obj1 = POP();

tests/basics/gen-yield-from-close.py

+def gen():
+    yield 1
+    yield 2
+    yield 3
+    yield 4
+
+def gen2():
+    yield -1
+    print((yield from gen()))
+    yield 10
+    yield 11
+
+g = gen2()
+print(next(g))
+print(next(g))
+g.close()
+try:
+    print(next(g))
+except StopIteration:
+    print("StopIteration")
+
+
+# Now variation of same test, but with leaf generator
+# swallowing GeneratorExit exception - its upstream gen
+# generator should still receive one.
+def gen3():
+    yield 1
+    try:
+        yield 2
+    except GeneratorExit:
+        print("leaf caught GeneratorExit and swallowed it")
+        return
+    yield 3
+    yield 4
+
+def gen4():
+    yield -1
+    try:
+        print((yield from gen3()))
+    except GeneratorExit:
+        print("delegating caught GeneratorExit")
+        raise
+    yield 10
+    yield 11
+
+g = gen4()
+print(next(g))
+print(next(g))
+print(next(g))
+g.close()
+try:
+    print(next(g))
+except StopIteration:
+    print("StopIteration")
+
+
+# Yet another variation - leaf generator gets GeneratorExit,
+# but raises StopIteration instead. This still should close chain properly.
+def gen5():
+    yield 1
+    try:
+        yield 2
+    except GeneratorExit:
+        print("leaf caught GeneratorExit and raised StopIteration instead")
+        raise StopIteration(123)
+    yield 3
+    yield 4
+
+def gen6():
+    yield -1
+    try:
+        print((yield from gen5()))
+    except GeneratorExit:
+        print("delegating caught GeneratorExit")
+        raise
+    yield 10
+    yield 11
+
+g = gen6()
+print(next(g))
+print(next(g))
+print(next(g))
+g.close()
+try:
+    print(next(g))
+except StopIteration:
+    print("StopIteration")

tests/basics/gen-yield-from-exc.py

+def gen():
+    yield 1
+    yield 2
+    raise ValueError
+
+def gen2():
+    try:
+        print((yield from gen()))
+    except ValueError:
+        print("caught ValueError from downstream")
+
+g = gen2()
+print(list(g))

tests/basics/gen-yield-from-send.py

+def gen():
+    print("sent:", (yield 1))
+    yield 2
+
+def gen2():
+    print((yield from gen()))
+
+g = gen2()
+next(g)
+print("yielded:", g.send("val"))
+try:
+    next(g)
+except StopIteration:
+    print("StopIteration")

tests/basics/gen-yield-from-throw.py

+def gen():
+    try:
+        yield 1
+    except ValueError:
+        print("got ValueError from upstream!")
+    yield "str1"
+    raise TypeError
+
+def gen2():
+    print((yield from gen()))
+
+g = gen2()
+print(next(g))
+print(g.throw(ValueError))
+try:
+    print(next(g))
+except TypeError:
+    print("got TypeError from downstream!")
+

tests/basics/gen-yield-from.py

+# Case of terminating subgen using return with value
+def gen():
+    yield 1
+    yield 2
+    return 3
+
+def gen2():
+    print("here1")
+    print((yield from gen()))
+    print("here2")
+
+g = gen2()
+print(list(g))
+
+
+# Like above, but terminate subgen using StopIteration
+def gen3():
+    yield 1
+    yield 2
+    raise StopIteration
+
+def gen4():
+    print("here1")
+    print((yield from gen3()))
+    print("here2")
+
+g = gen4()
+print(list(g))
+
+# Like above, but terminate subgen using StopIteration with value
+def gen5():
+    yield 1
+    yield 2
+    raise StopIteration(123)
+
+def gen6():
+    print("here1")
+    print((yield from gen5()))
+    print("here2")
+
+g = gen6()
+print(list(g))