diff --git a/tests/io/file-with.py b/tests/io/file-with.py
index 235072164159bf568994ac6141af8b480a7c15a6..ee1e702422b3a1fa8aab5f679c8a54f34dd6389b 100644
--- a/tests/io/file-with.py
+++ b/tests/io/file-with.py
@@ -10,3 +10,12 @@ except:
     # Note: CPython and us throw different exception trying to read from
     # close file.
     print("can't read file after with")
+
+
+# Regression test: test that exception in with initialization properly
+# thrown and doesn't crash.
+try:
+    with open('__non_existent', 'r'):
+        pass
+except OSError:
+    print("OSError")