diff --git a/pyin.py b/pyin.py
index 1caae67..8214286 100644
--- a/pyin.py
+++ b/pyin.py
@@ -524,13 +524,7 @@ def compiled_expression(self, mode):
 
         """Compile a Python expression using the builtin ``compile()``."""
 
-        try:
-            return builtins.compile(self.expression, '<string>', mode)
-        except SyntaxError as e:
-            raise SyntaxError(
-                f"expression {repr(self.expression)} contains a syntax error:"
-                f" {e.text}"
-            )
+        return builtins.compile(self.expression, '<string>', mode)
 
 
 class OpEval(OpBaseExpression, directives=('%eval', '%stream', '%exec')):
@@ -1246,14 +1240,8 @@ def main(
         # Probably possible to use 'OpEval(%exec)' here, but not immediately
         # clear how to manifest the scope changes.
         for statement in setup:
-            try:
-                code = builtins.compile(statement, '<string>', 'exec')
-            except SyntaxError as e:
-                raise SyntaxError(
-                    f"setup statement contains a syntax error:"
-                    f" {e.text.strip()}"
-                )
-            exec(code, scope, local_scope)
+            code_object = builtins.compile(statement, '<string>', 'exec')
+            exec(code_object, scope, local_scope)
             scope.update(local_scope)
 
         del local_scope
@@ -1339,6 +1327,20 @@ def _cli_entrypoint(rawargs=None):
     try:
         exit_code = main(**vars(args))
 
+    except SyntaxError as e:
+
+        exit_code = 1
+
+        # Reformat the exception information to provide clarity that this is
+        # something the user did wrong, and not something 'pyin' did wrong.
+        lines = [
+            f'ERROR: expression contains a syntax error: {e.msg}',
+            '',
+            f'    {e.text}',
+            f'    {" " * (e.offset - 1)}^',
+        ]
+        print(os.linesep.join(lines), file=sys.stderr)
+
     # User interrupted with '^C' most likely, but technically this is just
     # a SIGINT. Somehow this shows up in the coverage report generated by
     # '$ pytest --cov'. No idea how that works!!
diff --git a/tests/test_operations.py b/tests/test_operations.py
index f362293..3cbd828 100644
--- a/tests/test_operations.py
+++ b/tests/test_operations.py
@@ -150,7 +150,7 @@ def test_simple_stream(directive, args, stream, expected):
     assert list(pyin.eval(expressions, [])) == []
 
 
-def test_Eval_syntax_error():
+def test_eval_syntax_error():
 
     """Produce a helpful error when encountering :obj:`SyntaxError`.
 
@@ -163,8 +163,8 @@ def test_Eval_syntax_error():
     with pytest.raises(SyntaxError) as e:
         list(pyin.eval(expr, range(1)))
 
-    assert 'contains a syntax error' in str(e.value)
-    assert expr in str(e.value)
+    assert 'invalid syntax' in str(e.value)
+    assert expr == e.value.text
 
 
 def test_OpCSVDict(csv_with_header):
diff --git a/tests/test_pyin.py b/tests/test_pyin.py
index 43c02ea..3600aa7 100644
--- a/tests/test_pyin.py
+++ b/tests/test_pyin.py
@@ -314,7 +314,6 @@ def test_setup_syntax_error(runner):
     """``SyntaxError`` in a setup statement."""
 
     statement = '1 invalid syntax'
-    expected = f'ERROR: setup statement contains a syntax error: {statement}'
 
     result = runner.invoke(_cli_entrypoint, [
         '--gen', 'range(1)',
@@ -323,7 +322,8 @@ def test_setup_syntax_error(runner):
 
     assert result.exit_code == 1
     assert not result.output
-    assert result.err == expected + os.linesep
+    assert 'expression contains a syntax error: invalid syntax' in result.err
+    assert statement in result.err
 
 
 @mock.patch.dict(os.environ, {'PYIN_FULL_TRACEBACK': ''})