Merge pull request #395 from kevinmehall/custom-expr

Add `#{|input, pos| ... }` custom expr syntax to replace undocumented `##method()`

peg-macros/analysis.rs

@@ -164,7 +164,12 @@ impl<'a> LeftRecursionVisitor<'a> {
                nullable
             }
 
-            LiteralExpr(_) | PatternExpr(_) | MethodExpr(_, _) | FailExpr(_) | MarkerExpr(_) => false,
+            | LiteralExpr(_)
+            | PatternExpr(_)
+            | MethodExpr(_, _)
+            | CustomExpr(_)
+            | FailExpr(_)
+            | MarkerExpr(_) => false,
 
             PositionExpr => true,
         }
@@ -220,7 +225,7 @@ impl<'a> LoopNullabilityVisitor<'a> {
                 let name = rule_ident.to_string();
                 *self.rule_nullability.get(&name).unwrap_or(&false)
             }
-            
+
             ActionExpr(ref elems, ..) => {
                 let mut nullable = true;
                 for elem in elems {
@@ -250,7 +255,7 @@ impl<'a> LoopNullabilityVisitor<'a> {
                 if inner_nullable && sep_nullable && !bound.has_upper_bound() {
                     self.errors.push(LoopNullabilityError { span: this_expr.span });
                 }
-                
+
                 inner_nullable | !bound.has_lower_bound()
             }
 
@@ -269,10 +274,16 @@ impl<'a> LoopNullabilityVisitor<'a> {
                     }
                 }
 
-                nullable 
+                nullable
             }
 
-            LiteralExpr(_) | PatternExpr(_) | MethodExpr(_, _) | FailExpr(_) | MarkerExpr(_) => false,
+            | LiteralExpr(_)
+            | PatternExpr(_)
+            | MethodExpr(_, _)
+            | CustomExpr(_)
+            | FailExpr(_)
+            | MarkerExpr(_) => false,
+
             PositionExpr => true,
         }
     }

peg-macros/ast.rs

@@ -76,6 +76,7 @@ pub enum Expr {
     PatternExpr(Group),
     RuleExpr(Ident, Vec<RuleArg>),
     MethodExpr(Ident, TokenStream),
+    CustomExpr(Group),
     ChoiceExpr(Vec<SpannedExpr>),
     OptionalExpr(Box<SpannedExpr>),
     Repeat { inner: Box<SpannedExpr>, bound: BoundedRepeat, sep: Option<Box<SpannedExpr>> },

peg-macros/grammar.rs

@@ -3440,7 +3440,7 @@ pub mod peg {
                                                                         );
                                                                         match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { { let __seq_res = __parse_BRACKET_GROUP (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , p) => { :: peg :: RuleResult :: Matched (__pos , (|| { PatternExpr (p) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , }
                                                                     };
-                                                                    match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "(") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "@") { :: peg :: RuleResult :: Matched (__pos , __val) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , ")") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { MarkerExpr (true) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\")\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"@\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"(\"") ; :: peg :: RuleResult :: Failed } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "@") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { MarkerExpr (false) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"@\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "##") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_IDENT (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , method) => { { let __seq_res = __parse_PAREN_GROUP (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , args) => { :: peg :: RuleResult :: Matched (__pos , (|| { MethodExpr (method , args . stream ()) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"##\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "(") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_expression (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , expression) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , ")") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { expression }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\")\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"(\"") ; :: peg :: RuleResult :: Failed } } } } } } } } }
+                                                                    match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "(") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "@") { :: peg :: RuleResult :: Matched (__pos , __val) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , ")") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { MarkerExpr (true) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\")\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"@\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"(\"") ; :: peg :: RuleResult :: Failed } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "@") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { MarkerExpr (false) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"@\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "##") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_IDENT (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , method) => { { let __seq_res = __parse_PAREN_GROUP (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , args) => { :: peg :: RuleResult :: Matched (__pos , (|| { MethodExpr (method , args . stream ()) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"##\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => { let __choice_res = { let __seq_res = __parse_sp (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , sp) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "#") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_BRACE_GROUP (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , code) => { :: peg :: RuleResult :: Matched (__pos , (|| { CustomExpr (code) . at (sp) }) ()) } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"#\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } ; match __choice_res { :: peg :: RuleResult :: Matched (__pos , __value) => :: peg :: RuleResult :: Matched (__pos , __value) , :: peg :: RuleResult :: Failed => match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , "(") { :: peg :: RuleResult :: Matched (__pos , __val) => { { let __seq_res = __parse_expression (__input , __state , __err_state , __pos) ; match __seq_res { :: peg :: RuleResult :: Matched (__pos , expression) => { match :: peg :: ParseLiteral :: parse_string_literal (__input , __pos , ")") { :: peg :: RuleResult :: Matched (__pos , __val) => { :: peg :: RuleResult :: Matched (__pos , (|| { expression }) ()) } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\")\"") ; :: peg :: RuleResult :: Failed } } } :: peg :: RuleResult :: Failed => :: peg :: RuleResult :: Failed , } } } :: peg :: RuleResult :: Failed => { __err_state . mark_failure (__pos , "\"(\"") ; :: peg :: RuleResult :: Failed } } } } } } } } } } }
                                                                 }
                                                             }
                                                         }
@@ -3503,7 +3503,11 @@ pub mod peg {
                             let mut __repeat_value = vec![];
                             loop {
                                 let __pos = __repeat_pos;
-                                let __step_res = __input.eat_until(__pos, ',');
+                                let __step_res = ::peg::call_custom_closure(
+                                    (|input, pos| input.eat_until(pos, ',')),
+                                    __input,
+                                    __pos,
+                                );
                                 match __step_res {
                                     ::peg::RuleResult::Matched(__newpos, __value) => {
                                         __repeat_pos = __newpos;
@@ -3639,7 +3643,7 @@ pub mod peg {
         __pos: usize,
     ) -> ::peg::RuleResult<Span> {
         #![allow(non_snake_case, unused, clippy::redundant_closure_call)]
-        __input.next_span(__pos)
+        ::peg::call_custom_closure((|input, pos| input.next_span(pos)), __input, __pos)
     }
     fn __parse_KEYWORD<'input>(
         __input: &'input Input,
@@ -3755,7 +3759,8 @@ pub mod peg {
             };
             match __seq_res {
                 ::peg::RuleResult::Matched(__pos, _) => {
-                    let __seq_res = __input.ident(__pos);
+                    let __seq_res =
+                        ::peg::call_custom_closure((|input, pos| input.ident(pos)), __input, __pos);
                     match __seq_res {
                         ::peg::RuleResult::Matched(__pos, i) => {
                             ::peg::RuleResult::Matched(__pos, (|| i)())
@@ -3774,7 +3779,7 @@ pub mod peg {
         __pos: usize,
     ) -> ::peg::RuleResult<Literal> {
         #![allow(non_snake_case, unused, clippy::redundant_closure_call)]
-        __input.literal(__pos)
+        ::peg::call_custom_closure((|input, pos| input.literal(pos)), __input, __pos)
     }
     fn __parse_PAREN_GROUP<'input>(
         __input: &'input Input,
@@ -3783,7 +3788,11 @@ pub mod peg {
         __pos: usize,
     ) -> ::peg::RuleResult<Group> {
         #![allow(non_snake_case, unused, clippy::redundant_closure_call)]
-        __input.group(__pos, Delimiter::Parenthesis)
+        ::peg::call_custom_closure(
+            (|input, pos| input.group(pos, Delimiter::Parenthesis)),
+            __input,
+            __pos,
+        )
     }
     fn __parse_BRACE_GROUP<'input>(
         __input: &'input Input,
@@ -3792,7 +3801,11 @@ pub mod peg {
         __pos: usize,
     ) -> ::peg::RuleResult<Group> {
         #![allow(non_snake_case, unused, clippy::redundant_closure_call)]
-        __input.group(__pos, Delimiter::Brace)
+        ::peg::call_custom_closure(
+            (|input, pos| input.group(pos, Delimiter::Brace)),
+            __input,
+            __pos,
+        )
     }
     fn __parse_BRACKET_GROUP<'input>(
         __input: &'input Input,
@@ -3801,7 +3814,11 @@ pub mod peg {
         __pos: usize,
     ) -> ::peg::RuleResult<Group> {
         #![allow(non_snake_case, unused, clippy::redundant_closure_call)]
-        __input.group(__pos, Delimiter::Bracket)
+        ::peg::call_custom_closure(
+            (|input, pos| input.group(pos, Delimiter::Bracket)),
+            __input,
+            __pos,
+        )
     }
     fn __parse_LIFETIME<'input>(
         __input: &'input Input,

peg-macros/grammar.rustpeg

@@ -139,11 +139,12 @@ rule primary() -> SpannedExpr
   / "(" sp:sp() "@" ")" { MarkerExpr(true).at(sp) }
   / sp:sp() "@" { MarkerExpr(false).at(sp) }
   / sp:sp() "##" method:IDENT() args:PAREN_GROUP() { MethodExpr(method, args.stream()).at(sp) }
+  / sp:sp() "#" code:BRACE_GROUP() { CustomExpr(code).at(sp) }
   / "(" expression:expression() ")" { expression }
 
     rule rule_arg() -> RuleArg
         = "<" e:expression() ">" { RuleArg::Peg(e) }
-        / tt:$( ##eat_until(',')+ ) { RuleArg::Rust(tt) }
+        / tt:$( #{|input, pos| input.eat_until(pos, ',')}+ ) { RuleArg::Rust(tt) }
 
 rule precedence_level() -> PrecedenceLevel
   = operators:precedence_op()+
@@ -153,13 +154,13 @@ rule precedence_op() -> PrecedenceOperator
   = span:sp() elements:labeled()* action:BRACE_GROUP()
   { PrecedenceOperator{ span, elements, action } }
 
-rule sp() -> Span = ##next_span()
+rule sp() -> Span = #{|input, pos| input.next_span(pos)}
 rule KEYWORD() = "pub" / "rule" / "use" / "type" / "where"
-rule IDENT() -> Ident = !KEYWORD() i:##ident() {i}
-rule LITERAL() -> Literal = ##literal()
-rule PAREN_GROUP()   -> Group = ##group(Delimiter::Parenthesis)
-rule BRACE_GROUP()   -> Group = ##group(Delimiter::Brace)
-rule BRACKET_GROUP() -> Group = ##group(Delimiter::Bracket)
+rule IDENT() -> Ident = !KEYWORD() i:#{|input, pos| input.ident(pos)} {i}
+rule LITERAL() -> Literal = #{|input, pos| input.literal(pos)}
+rule PAREN_GROUP()   -> Group = #{|input, pos| input.group(pos, Delimiter::Parenthesis)}
+rule BRACE_GROUP()   -> Group = #{|input, pos| input.group(pos, Delimiter::Brace)}
+rule BRACKET_GROUP() -> Group = #{|input, pos| input.group(pos, Delimiter::Bracket)}
 rule LIFETIME() = "'" IDENT()
 rule INTEGER() = LITERAL()
 

peg-macros/translate.rs

@@ -626,6 +626,11 @@ fn compile_expr(context: &Context, e: &SpannedExpr, result_used: bool) -> TokenS
             quote_spanned! { span=> __input.#method(__pos, #args) }
         }
 
+        CustomExpr(ref code) => {
+            let code = code.stream();
+            quote_spanned! { span=> ::peg::call_custom_closure((#code), __input, __pos) }
+        }
+
         ChoiceExpr(ref exprs) => ordered_choice(
             span,
             exprs

peg-runtime/lib.rs

@@ -10,7 +10,9 @@ pub mod str;
 /// The result type used internally in the parser.
 ///
 /// You'll only need this if implementing the `Parse*` traits for a custom input
-/// type. The public API of a parser adapts errors to `std::result::Result`.
+/// type, or using the `#{}` syntax to embed a custom Rust snippet within the parser.
+///
+/// The public API of a parser adapts errors to `std::result::Result` instead of using this type.
 #[derive(Clone, PartialEq, PartialOrd, Eq, Ord, Debug, Hash)]
 pub enum RuleResult<T> {
     /// Success, with final location
@@ -58,3 +60,10 @@ pub trait ParseSlice<'input>: Parse {
 extern crate alloc;
 #[cfg(not(feature = "std"))]
 extern crate core as std;
+
+// needed for type inference on the `#{|input, pos| ..}` closure, since there
+// are different type inference rules on closures in function args.
+#[doc(hidden)]
+pub fn call_custom_closure<I, T>(f: impl FnOnce(I, usize) -> RuleResult<T>, input: I, pos: usize) -> RuleResult<T> {
+    f(input, pos)
+}

src/lib.rs

@@ -125,6 +125,8 @@
 //!     at the current location.
 //!   * `precedence!{ ... }` - Parse infix, prefix, or postfix expressions by precedence climbing.
 //!     [(details)](#precedence-climbing)
+//!   * `#{|input, pos| ... }` - _Custom:_ The provided closure is passed the full input and current
+//!      parse position, and returns a [`RuleResult`].
 //!
 //! ## Expression details
 //!

tests/run-pass/custom_expr.rs

@@ -0,0 +1,25 @@
+use peg::RuleResult;
+
+peg::parser!( grammar test() for str {
+    rule position() -> usize = #{|input, pos| RuleResult::Matched(pos, pos)}
+    pub rule test1() -> usize = ['a']* p1:position() ['b']* { p1 }
+
+    pub rule fail() -> usize = #{|input, pos| RuleResult::Failed}
+
+    rule custom_literal(literal: &str) = #{|input, pos| {
+        let l = literal.len();
+        if input.len() >= pos + l && &input.as_bytes()[pos..pos + l] == literal.as_bytes() {
+            RuleResult::Matched(pos + l, ())
+        } else {
+            RuleResult::Failed
+        }
+    }}
+    pub rule test2() = custom_literal("foo") "_" custom_literal("bar")
+});
+
+fn main() {
+    assert_eq!(test::test1("aaaabb"), Ok(4));
+    assert_eq!(test::fail("aaaabb").unwrap_err().location.offset, 0);
+
+    assert_eq!(test::test2("foo_bar"), Ok(()));
+}