From 36f48df2c61c760475ca3073d19ed297a9442bbd Mon Sep 17 00:00:00 2001 From: Matthew Paras <34500476+mattwparas@users.noreply.github.com> Date: Mon, 4 Dec 2023 20:05:20 -0800 Subject: [PATCH] Move blocking web requests and web sockets entirely to dylibs (#107) Move web sockets and blocking web requests to dylibs, move parser into parser crate --- Cargo.lock | 106 +- Cargo.toml | 2 +- crates/steel-core/Cargo.toml | 14 +- .../steel-core/src/compiled_macros/test.macro | Bin 4437 -> 0 bytes .../steel-core/src/compiled_macros/test.rkt | 34 - crates/steel-core/src/compiler/modules.rs | 4 +- .../src/compiler/passes/analysis.rs | 35 +- crates/steel-core/src/compiler/program.rs | 152 - crates/steel-core/src/parser/ast.rs | 2290 +------------- .../steel-core/src/parser/expand_visitor.rs | 4 +- crates/steel-core/src/parser/expander.rs | 9 - crates/steel-core/src/parser/interner.rs | 165 +- crates/steel-core/src/parser/kernel.rs | 3 +- crates/steel-core/src/parser/parser.rs | 2628 +---------------- crates/steel-core/src/primitives.rs | 3 - .../src/primitives/blocking_requests.rs | 78 - crates/steel-core/src/primitives/strings.rs | 19 + crates/steel-core/src/primitives/web/mod.rs | 1 - .../src/primitives/web/websockets.rs | 71 - crates/steel-core/src/rvals.rs | 118 - crates/steel-core/src/steel_vm/contracts.rs | 742 ----- crates/steel-core/src/steel_vm/mod.rs | 1 - crates/steel-core/src/steel_vm/primitives.rs | 16 +- crates/steel-core/src/steel_vm/tests.rs | 326 +- crates/steel-core/src/values/structs.rs | 158 - crates/steel-parser/Cargo.toml | 4 +- crates/steel-parser/src/ast.rs | 2298 ++++++++++++++ crates/steel-parser/src/interner.rs | 151 + crates/steel-parser/src/lib.rs | 2 + crates/steel-parser/src/parser.rs | 2406 +++++++++++++++ crates/steel-values/Cargo.toml | 8 + crates/steel-values/src/lib.rs | 14 + crates/steel-webrequests/src/lib.rs | 11 +- crates/steel-websockets/src/lib.rs | 14 +- 34 files changed, 5346 insertions(+), 6541 deletions(-) delete mode 100644 crates/steel-core/src/compiled_macros/test.macro delete mode 100644 crates/steel-core/src/compiled_macros/test.rkt delete mode 100644 crates/steel-core/src/primitives/blocking_requests.rs delete mode 100644 crates/steel-core/src/primitives/web/websockets.rs delete mode 100644 crates/steel-core/src/steel_vm/contracts.rs create mode 100644 crates/steel-parser/src/ast.rs create mode 100644 crates/steel-parser/src/interner.rs create mode 100644 crates/steel-values/Cargo.toml create mode 100644 crates/steel-values/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index e7d211d78..34cf0f8a6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,18 @@ dependencies = [ "version_check", ] +[[package]] +name = "ahash" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" +dependencies = [ + "cfg-if 1.0.0", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "0.7.20" @@ -864,12 +876,15 @@ dependencies = [ [[package]] name = "dashmap" -version = "4.0.2" +version = "5.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +checksum = "907076dfda823b0b36d2a1bb5f90c96660a5bbcd7729e10727f07858f22c4edc" dependencies = [ "cfg-if 1.0.0", - "num_cpus", + "hashbrown 0.12.3", + "lock_api", + "once_cell", + "parking_lot_core", ] [[package]] @@ -1178,7 +1193,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.39", ] [[package]] @@ -1283,21 +1298,21 @@ checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7" [[package]] name = "hashbrown" -version = "0.11.2" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash", - "serde", + "ahash 0.7.6", ] [[package]] name = "hashbrown" -version = "0.12.3" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" dependencies = [ - "ahash", + "ahash 0.8.6", + "serde", ] [[package]] @@ -1569,12 +1584,12 @@ checksum = "bf36173d4167ed999940f804952e6b08197cae5ad5d572eb4db150ce8ad5d58f" [[package]] name = "lasso" -version = "0.6.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aeb7b21a526375c5ca55f1a6dfd4e1fad9fa4edd750f530252a718a44b2608f0" +checksum = "4644821e1c3d7a560fe13d842d13f587c07348a1a05d3a797152d41c90c56df2" dependencies = [ "dashmap", - "hashbrown 0.11.2", + "hashbrown 0.13.2", "serde", ] @@ -2061,7 +2076,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.39", ] [[package]] @@ -2124,7 +2139,7 @@ dependencies = [ "phf_shared 0.11.2", "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.39", ] [[package]] @@ -2275,9 +2290,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.60" +version = "1.0.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec2b086b7a862cf4de201096214fa870344cf922b2b30c167badb3af3195406" +checksum = "39278fbbf5fb4f646ce651690877f89d1c5811a3d4acb27700c1cb3cdb78fd3b" dependencies = [ "unicode-ident", ] @@ -2790,29 +2805,29 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "25dd9975e68d0cb5aa1120c288333fc98731bd1dd12f561e468ea4728c042b89" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.193" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "43576ca501357b9b071ac53cdc7da8ef0cbd9493d8df094cd821777ea6e894d3" dependencies = [ "proc-macro2", "quote", - "syn 1.0.107", + "syn 2.0.39", ] [[package]] name = "serde_json" -version = "1.0.92" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -3074,8 +3089,6 @@ dependencies = [ "steel-gen", "steel-parser", "termimad", - "tungstenite", - "ureq", "url", "weak-table", "which", @@ -3087,7 +3100,7 @@ version = "0.4.0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.39", ] [[package]] @@ -3125,7 +3138,10 @@ dependencies = [ name = "steel-parser" version = "0.4.0" dependencies = [ + "lasso", "num-bigint", + "once_cell", + "pretty", "serde", "serde_derive", ] @@ -3173,6 +3189,14 @@ dependencies = [ "toml", ] +[[package]] +name = "steel-values" +version = "0.1.0" +dependencies = [ + "im-lists", + "im-rc", +] + [[package]] name = "steel-webrequests" version = "0.1.0" @@ -3237,9 +3261,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.18" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32d41677bcbe24c20c52e7c70b0d8db04134c5d1066bf98662e2871ad200ea3e" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -3450,7 +3474,7 @@ checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.18", + "syn 2.0.39", ] [[package]] @@ -4148,3 +4172,23 @@ checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" dependencies = [ "winapi", ] + +[[package]] +name = "zerocopy" +version = "0.7.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d6f15f7ade05d2a4935e34a457b936c23dc70a05cc1d97133dc99e7a3fe0f0e" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbbad221e3f78500350ecbd7dfa4e63ef945c05f4c61cb7f4d3f84cd0bba649b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] diff --git a/Cargo.toml b/Cargo.toml index 2e9af5b02..7c8196aeb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,7 +19,7 @@ path = "src/main.rs" [workspace.dependencies] # This has to line up with the workspace version above -steel-core = { path = "./crates/steel-core", version = "0.5.0", features = ["web", "sqlite", "blocking_requests", "dylibs", "markdown"] } +steel-core = { path = "./crates/steel-core", version = "0.5.0", features = ["web", "sqlite", "dylibs", "markdown"] } [dependencies] once_cell = "1.17.0" diff --git a/crates/steel-core/Cargo.toml b/crates/steel-core/Cargo.toml index 44ee6dd61..cfb07bde5 100644 --- a/crates/steel-core/Cargo.toml +++ b/crates/steel-core/Cargo.toml @@ -16,20 +16,19 @@ name = "steel" [dependencies] im-rc = "15.1.0" codespan-reporting = "0.11.1" -# logos = "0.12.1" log = "0.4.17" futures-util = "0.3.28" futures-task = "0.3.28" -serde_json = "1.0.92" -serde = { version = "1.0.152", features = ["derive", "rc"] } -serde_derive = "1.0.152" +serde_json = "1.0.108" +serde = { version = "1.0.193", features = ["derive", "rc"] } +serde_derive = "1.0.193" bincode = "1.3.3" pretty = "0.12.1" im-lists = "0.6.0" quickscope = "0.2.0" -lasso = { version = "0.6.0", features = ["multi-threaded", "serialize"] } +lasso = { version = "0.7.2", features = ["multi-threaded", "serialize"] } once_cell = "1.18.0" fxhash = "0.2.1" # lazy_static = "1.4.0" @@ -63,9 +62,7 @@ cranelift-jit = { version = "0.84.0", optional = true } rusqlite = { version = "0.28.0", features = ["bundled"], optional = true } reqwest = { version = "0.11.14", features = ["blocking", "json"], optional = true } url = { version = "2.3.1", optional = true } -tungstenite = { version = "0.18.0", features = ["rustls-tls-native-roots"], optional = true } anyhow = { version = "1", optional = true } -ureq = { version = "2.6.2", features = ["json"], optional = true } [dev-dependencies] proptest = "1.1.0" @@ -81,12 +78,11 @@ modules = [] jit = ["dep:cranelift", "dep:cranelift-module", "dep:cranelift-jit"] dynamic = [] profiling = [] -web = ["dep:reqwest", "dep:url", "dep:tungstenite"] +web = ["dep:reqwest", "dep:url"] sqlite = ["dep:rusqlite"] unsafe-internals = [] anyhow = ["dep:anyhow"] dylibs = ["dep:abi_stable", "dep:async-ffi"] -blocking_requests = ["dep:ureq"] markdown = ["dep:termimad"] smallvec = ["dep:smallvec"] without-drop-protection = [] diff --git a/crates/steel-core/src/compiled_macros/test.macro b/crates/steel-core/src/compiled_macros/test.macro deleted file mode 100644 index 848640a98371e60a02762774c99ec2823f36ff1b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4437 zcmcguO-K}B7}m6`)W8mbbg-IZZ5E|;$v`9QP=}6PV%r^cA=hwstq3|*5Mdo63bI31 z1f9yj6e21$T-3-y=!Yx{GUyPV`hl`}W}e5Lota))V^D}=7)^<2CS9Nt@!;WW&{)rHXK{YPRssmHhhVQraF0mwh=_NDq546FBTu9g&|A+dXK{WjtmQQo zlLD}b+c})6a8Bc_(83loez^5e3Nn0UXl{T-m=CjWTAXTDvGZy!|CUi>B`9LHZGuz|W8Tu`y z;Abb#&dBzV+9wkg$1}(M#2!y}a~5>j>AoX=p_Q z6*~lEZ^cehc8Q8sG*S?VFnwCFo3uT${c|fc&aVF+@SD20t$BqfdBdsvm&+7>&=U=c z^=c;p9iFs*5F_06g?0zSef?rqUJ`sG5?twB2I|rNNHkm>dPn%9mzH_?Pm%IPR6p

T69e&!N`dxwXMoD2oN7z|>-2xOKl5NC zJ`Dq6&U-i_tVM@vq9RMia2%p=R7$`0r4Pq3Qjg0-elv&T1Sy@O`gvDZH5_M2z91v{ zSEmfaI&&9^UJ^@gGKb?bDOW_raCA`+Xb(q_%4RVUueMbfss=w)r1r@~G@r@oCuh*h z)fRSULqvzYEN$$3(vpP7+%(*yPpB9dfa_>4g?&cA2+Ps+TtV0Bt9wQB(|20|kNh098_1C8pN-Eb~0x5$73S zx8hW(G#+T{Rf0D}%BKl8N$!!6JWIBlaEDT_wAq``N6I}>(S*Ac1lmpLr1G4Yxli#Z ysn2AhXnjucb8;rVT*-E3Zz;W(RvuGF27lW=5T0_=aEd=s@mT;)@kt7&rTh)Gc$5eL diff --git a/crates/steel-core/src/compiled_macros/test.rkt b/crates/steel-core/src/compiled_macros/test.rkt deleted file mode 100644 index 6daed4459..000000000 --- a/crates/steel-core/src/compiled_macros/test.rkt +++ /dev/null @@ -1,34 +0,0 @@ -(define-syntax module - (syntax-rules (provide gen-defines contract/out) - [(module name (provide ids ...) funcs ...) - (begin - (define (datum->syntax name) - ((lambda () funcs ... - (module provide ids ...)))) - (module gen-defines name ids ...))] - - ;; in the contract case, ignore the contract in the hash - [(module provide (contract/out name contract)) (hash 'name name)] - ;; Normal case - [(module provide name) (hash 'name name)] - - ;; in the contract case, ignore the contract in the hash - [(module provide (contract/out name contract) rest ...) - (hash-insert (module provide rest ...) 'name name)] - - ;; Normal case - [(module provide name rest ...) - (hash-insert (module provide rest ...) 'name name)] - - ;; Module contract provides - [(module gen-defines mod (contract/out name contract)) - (define (datum->syntax name) (bind/c contract (hash-get mod 'name)))] - [(module gen-defines mod (contract/out name contract) rest ...) - (begin (define (datum->syntax name) (bind/c contract (hash-get mod 'name))) - (module gen-defines mod rest ...))] - - ;; Normal provides - [(module gen-defines mod name) (define (datum->syntax name) (hash-get mod 'name))] - [(module gen-defines mod name rest ...) - (begin (define (datum->syntax name) (hash-get mod 'name)) - (module gen-defines mod rest ...))])) \ No newline at end of file diff --git a/crates/steel-core/src/compiler/modules.rs b/crates/steel-core/src/compiler/modules.rs index 4098a3d32..8d67e3c5f 100644 --- a/crates/steel-core/src/compiler/modules.rs +++ b/crates/steel-core/src/compiler/modules.rs @@ -7,7 +7,6 @@ use crate::{ }, program::PROVIDE, }, - expr_list, parser::{ ast::{AstTools, Atom, Begin, Define, ExprKind, List, Quote}, expand_visitor::expand_kernel, @@ -23,6 +22,8 @@ use crate::{ }; use crate::{parser::expand_visitor::Expander, rvals::Result}; +use steel_parser::expr_list; + use std::{ borrow::Cow, collections::{HashMap, HashSet}, @@ -37,7 +38,6 @@ use std::time::SystemTime; use crate::parser::expand_visitor::{expand, extract_macro_defs}; -// use itertools::Itertools; use log::{debug, info, log_enabled}; use crate::parser::ast::IteratorExtensions; diff --git a/crates/steel-core/src/compiler/passes/analysis.rs b/crates/steel-core/src/compiler/passes/analysis.rs index 3d09b89e9..66c709882 100644 --- a/crates/steel-core/src/compiler/passes/analysis.rs +++ b/crates/steel-core/src/compiler/passes/analysis.rs @@ -9,13 +9,17 @@ use quickscope::ScopeMap; use crate::{ compiler::modules::{ModuleManager, MANGLER_SEPARATOR}, parser::{ - ast::{Atom, Define, ExprKind, LambdaFunction, Let, List, Quote}, + ast::{ + Atom, Define, ExprKind, LambdaFunction, Let, List, Quote, STANDARD_MODULE_GET, + UNREADABLE_MODULE_GET, + }, expander::SteelMacro, interner::InternedString, parser::{RawSyntaxObject, SyntaxObject, SyntaxObjectId}, span::Span, tokens::TokenType, }, + steel_vm::primitives::MODULE_IDENTIFIERS, throw, SteelErr, SteelVal, }; @@ -313,9 +317,7 @@ impl Analysis { define.name.atom_syntax_object().unwrap().span, ); - if define.is_a_builtin_definition() { - println!("FOUND A BUILTIN: {}", name); - + if is_a_builtin_definition(define) { semantic_info.mark_builtin(); } @@ -603,7 +605,7 @@ impl<'a> AnalysisPass<'a> { } } - if define.is_a_builtin_definition() { + if is_a_builtin_definition(define) { // println!("FOUND A BUILTIN: {}", name); semantic_info.mark_builtin(); @@ -648,7 +650,7 @@ impl<'a> AnalysisPass<'a> { define.name.atom_syntax_object().unwrap().span, ); - if define.is_a_builtin_definition() { + if is_a_builtin_definition(define) { semantic_info.mark_builtin(); } @@ -2087,6 +2089,25 @@ impl<'a> RemoveUnusedDefineImports<'a> { } } +// This should just be a function on the define, not a method - so that it can be moved +// into a different crate +pub(crate) fn is_a_builtin_definition(def: &Define) -> bool { + if let ExprKind::List(l) = &def.body { + match l.first_ident() { + Some(func) if *func == *UNREADABLE_MODULE_GET || *func == *STANDARD_MODULE_GET => { + // return true + + if let Some(module) = l.second_ident() { + return MODULE_IDENTIFIERS.contains(module); + } + } + _ => {} + } + } + + false +} + impl<'a> VisitorMutRefUnit for RemoveUnusedDefineImports<'a> { fn visit_lambda_function(&mut self, lambda_function: &mut LambdaFunction) { self.depth += 1; @@ -2101,7 +2122,7 @@ impl<'a> VisitorMutRefUnit for RemoveUnusedDefineImports<'a> { for (idx, expr) in begin.exprs.iter().enumerate() { if let ExprKind::Define(d) = expr { - if d.is_a_builtin_definition() { + if is_a_builtin_definition(&d) { if let Some(analysis) = self.analysis.get(d.name.atom_syntax_object().unwrap()) { diff --git a/crates/steel-core/src/compiler/program.rs b/crates/steel-core/src/compiler/program.rs index a96fff7c2..8b3d859ae 100644 --- a/crates/steel-core/src/compiler/program.rs +++ b/crates/steel-core/src/compiler/program.rs @@ -49,52 +49,6 @@ fn eval_atom(t: &SyntaxObject) -> Result { } } -// pub fn move_read_local_call_global(instructions: &mut [Instruction]) { -// for i in 0..instructions.len() { -// let move_read_local = instructions.get(i); -// let call_global = instructions.get(i + 1); - -// match (move_read_local, call_global) { -// ( -// Some(Instruction { -// op_code: OpCode::MOVEREADLOCAL, -// .. -// }), -// Some(Instruction { -// op_code: OpCode::CALLGLOBAL, -// .. -// }), -// ) => { -// if let Some(x) = instructions.get_mut(i) { -// x.op_code = OpCode::MOVEREADLOCALCALLGLOBAL; -// } -// } -// _ => {} -// } -// } -// } - -// pub fn specialize_exit_jmp(instructions: &mut [Instruction]) { -// for i in 0..instructions.len() { -// if let Some(Instruction { -// op_code: OpCode::JMP, -// payload_size, -// .. -// }) = instructions.get(i) -// { -// if let Some((OpCode::POPPURE, payload_size_pop)) = instructions -// .get(i + *payload_size) -// .map(|x| (x.op_code, x.payload_size)) -// { -// let guard = instructions.get_mut(i).unwrap(); -// guard.op_code = OpCode::POPPURE; -// guard.payload_size = payload_size_pop; -// continue; -// } -// } -// } -// } - pub fn specialize_read_local(instructions: &mut [Instruction]) { for i in 0..instructions.len() { let read_local = instructions.get(i); @@ -142,112 +96,6 @@ pub fn specialize_read_local(instructions: &mut [Instruction]) { } } -// Often, there may be a loop condition with something like (= x 10000) -// this identifies these and lazily applies the function, only pushing on to the stack -// until it absolutely needs to -// pub fn loop_condition_local_const_arity_two(instructions: &mut [Instruction]) { -// for i in 0..instructions.len() { -// let read_local = instructions.get(i); -// let push_const = instructions.get(i + 1); -// let call_global = instructions.get(i + 2); -// let pass = instructions.get(i + 3); - -// match (read_local, push_const, call_global, pass) { -// ( -// Some(Instruction { -// op_code: OpCode::READLOCAL, -// payload_size: local_idx, -// .. -// }), -// Some(Instruction { -// op_code: OpCode::PUSHCONST, -// payload_size: const_idx, -// .. -// }), -// Some(Instruction { -// op_code: OpCode::CALLGLOBAL, -// payload_size: ident, -// contents: identifier, -// .. -// }), -// // HAS to be arity 2 in this case -// Some(Instruction { -// op_code: OpCode::PASS, -// payload_size: 2, -// .. -// }), -// ) => { -// let local_idx = *local_idx; -// let const_idx = *const_idx; -// let ident = *ident; -// let identifier = identifier.clone(); - -// if let Some(x) = instructions.get_mut(i) { -// x.op_code = OpCode::CGLOCALCONST; -// x.payload_size = ident; -// x.contents = identifier; -// } - -// if let Some(x) = instructions.get_mut(i + 1) { -// x.op_code = OpCode::READLOCAL; -// x.payload_size = local_idx; -// } - -// if let Some(x) = instructions.get_mut(i + 2) { -// x.op_code = OpCode::PUSHCONST; -// x.payload_size = const_idx; -// } -// } -// ( -// Some(Instruction { -// op_code: OpCode::MOVEREADLOCAL, -// payload_size: local_idx, -// .. -// }), -// Some(Instruction { -// op_code: OpCode::PUSHCONST, -// payload_size: const_idx, -// .. -// }), -// Some(Instruction { -// op_code: OpCode::CALLGLOBAL, -// payload_size: ident, -// contents: identifier, -// .. -// }), -// // HAS to be arity 2 in this case -// Some(Instruction { -// op_code: OpCode::PASS, -// payload_size: 2, -// .. -// }), -// ) => { -// let local_idx = *local_idx; -// let const_idx = *const_idx; -// let ident = *ident; -// let identifier = identifier.clone(); - -// if let Some(x) = instructions.get_mut(i) { -// x.op_code = OpCode::MOVECGLOCALCONST; -// x.payload_size = ident; -// x.contents = identifier; -// } - -// if let Some(x) = instructions.get_mut(i + 1) { -// x.op_code = OpCode::MOVEREADLOCAL; -// x.payload_size = local_idx; -// } - -// if let Some(x) = instructions.get_mut(i + 2) { -// x.op_code = OpCode::PUSHCONST; -// x.payload_size = const_idx; -// } -// } -// _ => {} -// } -// } -// } - pub fn specialize_constants(instructions: &mut [Instruction]) -> Result<()> { if instructions.is_empty() { return Ok(()); diff --git a/crates/steel-core/src/parser/ast.rs b/crates/steel-core/src/parser/ast.rs index a8700a075..d2435c5fe 100644 --- a/crates/steel-core/src/parser/ast.rs +++ b/crates/steel-core/src/parser/ast.rs @@ -1,23 +1,13 @@ -use crate::{ - compiler::program::{ - BEGIN, DATUM_SYNTAX, DEFINE, IF, LAMBDA, LAMBDA_FN, LAMBDA_SYMBOL, LET, PLAIN_LET, QUOTE, - REQUIRE, RETURN, SET, STANDARD_MODULE_GET, UNREADABLE_MODULE_GET, - }, - parser::{ - parser::{ParseError, SyntaxObject}, - tokens::TokenType::{self, *}, - tryfrom_visitor::TryFromExprKindForSteelVal, - }, - steel_vm::primitives::MODULE_IDENTIFIERS, +use crate::parser::{ + parser::SyntaxObject, tokens::TokenType::*, tryfrom_visitor::TryFromExprKindForSteelVal, }; -use std::{convert::TryFrom, fmt::Write, sync::atomic::Ordering}; +// use std::{convert::TryFrom, fmt::Write, sync::atomic::Ordering}; -// use itertools::Itertools; -use pretty::RcDoc; -use serde::{Deserialize, Serialize}; -use std::fmt; -use std::ops::Deref; +// use pretty::RcDoc; +// use serde::{Deserialize, Serialize}; +// use std::fmt; +// use std::ops::Deref; use steel_parser::tokens::MaybeBigInt; use crate::{ @@ -25,309 +15,12 @@ use crate::{ rvals::SteelVal::{self, *}, }; -use super::{ - interner::InternedString, - parser::{SyntaxObjectId, SYNTAX_OBJECT_ID}, - span::Span, +pub use steel_parser::ast::{ + AstTools, Atom, Begin, Define, ExprKind, If, IteratorExtensions, LambdaFunction, Let, List, + Macro, PatternPair, Quote, Require, Return, Set, SyntaxRules, STANDARD_MODULE_GET, + UNREADABLE_MODULE_GET, }; -pub(crate) trait AstTools { - fn pretty_print(&self); -} - -impl AstTools for Vec { - fn pretty_print(&self) { - println!("{}", self.iter().map(|x| x.to_pretty(60)).join("\n\n")) - } -} - -impl AstTools for Vec<&ExprKind> { - fn pretty_print(&self) { - println!("{}", self.iter().map(|x| x.to_pretty(60)).join("\n\n")) - } -} - -impl AstTools for &mut Vec { - fn pretty_print(&self) { - println!("{}", self.iter().map(|x| x.to_pretty(60)).join("\n\n")) - } -} - -pub trait IteratorExtensions: Iterator { - fn join(&mut self, sep: &str) -> String - where - Self::Item: std::fmt::Display, - { - match self.next() { - None => String::new(), - Some(first_elt) => { - // estimate lower bound of capacity needed - let (lower, _) = self.size_hint(); - let mut result = String::with_capacity(sep.len() * lower); - write!(&mut result, "{}", first_elt).unwrap(); - self.for_each(|elt| { - result.push_str(sep); - write!(&mut result, "{}", elt).unwrap(); - }); - result - } - } - } -} - -impl IteratorExtensions for T where T: Iterator {} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub enum ExprKind { - Atom(Atom), - If(Box), - Let(Box), - Define(Box), - LambdaFunction(Box), - Begin(Begin), - Return(Box), - Quote(Box), - Macro(Macro), - SyntaxRules(SyntaxRules), - List(List), - Set(Box), - Require(Require), -} - -#[macro_export] -macro_rules! expr_list { - () => { $crate::parser::ast::ExprKind::List($crate::parser::ast::List::new(vec![])) }; - - ( $($x:expr),* ) => {{ - $crate::parser::ast::ExprKind::List($crate::parser::ast::List::new(vec![$( - $x, - ) *])) - }}; - - ( $($x:expr ,)* ) => {{ - $crate::parser::ast::ExprKind::List($crate::parser::ast::List::new(vec![$($x, )*])) - }}; -} - -impl ExprKind { - pub fn into_lambda_function(self) -> Option> { - if let ExprKind::LambdaFunction(func) = self { - Some(func) - } else { - None - } - } - - pub fn quoted_list() -> ExprKind { - ExprKind::Quote(Box::new(Quote::new( - Self::empty(), - SyntaxObject::default(TokenType::QuoteTick), - ))) - } - - pub fn empty() -> ExprKind { - ExprKind::List(List::new(Vec::new())) - } - - pub fn integer_literal(value: isize, span: Span) -> ExprKind { - ExprKind::Atom(crate::parser::ast::Atom::new(SyntaxObject::new( - TokenType::IntegerLiteral(MaybeBigInt::Small(value)), - span, - ))) - } - - pub fn atom>(name: T) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::Identifier( - name.into(), - )))) - } - - pub fn ident(name: &str) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::Identifier( - name.into(), - )))) - } - - pub fn string_lit(input: String) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::StringLiteral( - input, - )))) - } - - pub fn bool_lit(b: bool) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::BooleanLiteral( - b, - )))) - } - - pub fn default_if(test: ExprKind, then: ExprKind, els: ExprKind) -> ExprKind { - ExprKind::If(Box::new(If::new( - test, - then, - els, - SyntaxObject::default(TokenType::If), - ))) - } - - pub fn into_atom_syntax_object(self) -> Option { - match self { - Self::Atom(Atom { syn }) => Some(syn), - _ => None, - } - } - - pub fn atom_syntax_object(&self) -> Option<&SyntaxObject> { - match self { - Self::Atom(Atom { syn }) => Some(syn), - _ => None, - } - } - - pub fn define_syntax_ident(&self) -> bool { - match self { - Self::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::DefineSyntax, - .. - }, - }) => true, - _ => false, - } - } - - pub fn atom_identifier_mut(&mut self) -> Option<&mut InternedString> { - match self { - Self::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(s), - .. - }, - }) => Some(s), - _ => None, - } - } - - pub fn lambda_function(&self) -> Option<&LambdaFunction> { - match self { - Self::LambdaFunction(l) => Some(l), - _ => None, - } - } - - pub fn atom_identifier_or_else E>( - &self, - err: F, - ) -> std::result::Result<&InternedString, E> { - match self { - Self::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(s), - .. - }, - }) => Ok(s), - _ => Err(err()), - } - } - - pub fn atom_identifier(&self) -> Option<&InternedString> { - match self { - Self::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(s), - .. - }, - }) => Some(s), - _ => None, - } - } - - pub fn string_literal(&self) -> Option<&str> { - match self { - Self::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::StringLiteral(s), - .. - }, - }) => Some(s), - _ => None, - } - } - - pub fn list(&self) -> Option<&List> { - if let ExprKind::List(l) = self { - Some(l) - } else { - None - } - } - - pub fn list_or_else E>(&self, err: F) -> std::result::Result<&List, E> { - match self { - Self::List(l) => Ok(l), - _ => Err(err()), - } - } - - pub fn list_mut_or_else E>( - &mut self, - err: F, - ) -> std::result::Result<&mut List, E> { - match self { - Self::List(l) => Ok(l), - _ => Err(err()), - } - } - - pub fn into_list_or_else E>(self, err: F) -> std::result::Result { - match self { - Self::List(l) => Ok(l), - _ => Err(err()), - } - } - - pub fn into_list(self) -> List { - if let ExprKind::List(l) = self { - l - } else { - panic!("Attempted to coerce a non list to a list"); - } - } - - pub fn unwrap_function(self) -> Option> { - if let ExprKind::LambdaFunction(l) = self { - Some(l) - } else { - None - } - } - - pub fn get_list(&self) -> Option<&List> { - if let ExprKind::List(l) = self { - Some(l) - } else { - None - } - } - - pub fn update_string_in_atom(&mut self, ident: InternedString) { - if let ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(ref mut s), - .. - }, - }) = self - { - *s = ident; - } - } -} - impl TryFrom for SteelVal { type Error = SteelErr; @@ -562,1964 +255,3 @@ impl TryFrom<&SteelVal> for ExprKind { inner_try_from(r, 0) } } - -pub trait ToDoc { - fn to_doc(&self) -> RcDoc<()>; -} - -impl ToDoc for ExprKind { - fn to_doc(&self) -> RcDoc<()> { - match self { - ExprKind::Atom(a) => a.to_doc(), - ExprKind::If(i) => i.to_doc(), - ExprKind::Define(d) => d.to_doc(), - ExprKind::LambdaFunction(l) => l.to_doc(), - ExprKind::Begin(b) => b.to_doc(), - ExprKind::Return(r) => r.to_doc(), - ExprKind::Let(l) => l.to_doc(), - ExprKind::Quote(q) => q.to_doc(), - ExprKind::Macro(m) => m.to_doc(), - ExprKind::SyntaxRules(s) => s.to_doc(), - ExprKind::List(l) => l.to_doc(), - ExprKind::Set(s) => s.to_doc(), - ExprKind::Require(r) => r.to_doc(), - } - } -} - -impl ExprKind { - pub fn to_pretty(&self, width: usize) -> String { - let mut w = Vec::new(); - self.to_doc().render(width, &mut w).unwrap(); - String::from_utf8(w).unwrap() - } -} - -impl fmt::Display for ExprKind { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - ExprKind::Atom(a) => write!(f, "{a}"), - ExprKind::If(i) => write!(f, "{i}"), - ExprKind::Define(d) => write!(f, "{d}"), - ExprKind::LambdaFunction(l) => write!(f, "{l}"), - ExprKind::Begin(b) => write!(f, "{b}"), - ExprKind::Return(r) => write!(f, "{r}"), - ExprKind::Let(l) => write!(f, "{l}"), - ExprKind::Quote(q) => write!(f, "{q}"), - ExprKind::Macro(m) => write!(f, "{m}"), - ExprKind::SyntaxRules(s) => write!(f, "{s}"), - ExprKind::List(l) => write!(f, "{l}"), - ExprKind::Set(s) => write!(f, "{s}"), - ExprKind::Require(r) => write!(f, "{r}"), - } - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Atom { - pub syn: SyntaxObject, -} - -impl Atom { - pub fn new(syn: SyntaxObject) -> Self { - Atom { syn } - } - - pub fn ident(&self) -> Option<&InternedString> { - if let TokenType::Identifier(ref ident) = self.syn.ty { - Some(ident) - } else { - None - } - } - - pub fn ident_mut(&mut self) -> Option<&mut InternedString> { - if let TokenType::Identifier(ref mut ident) = self.syn.ty { - Some(ident) - } else { - None - } - } -} - -impl fmt::Display for Atom { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "{}", self.syn.ty) - } -} - -impl ToDoc for Atom { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text(self.syn.ty.to_string()) - } -} - -impl From for ExprKind { - fn from(val: Atom) -> Self { - ExprKind::Atom(val) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Let { - pub bindings: Vec<(ExprKind, ExprKind)>, - pub body_expr: ExprKind, - pub location: SyntaxObject, - pub syntax_object_id: usize, -} - -impl Let { - pub fn new( - bindings: Vec<(ExprKind, ExprKind)>, - body_expr: ExprKind, - location: SyntaxObject, - ) -> Self { - Let { - bindings, - body_expr, - location, - syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), - } - } - - pub fn local_bindings(&self) -> impl Iterator { - self.bindings.iter().map(|x| &x.0) - } - - pub fn expression_arguments(&self) -> impl Iterator { - self.bindings.iter().map(|x| &x.1) - } -} - -impl fmt::Display for Let { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "(%plain-let ({}) {})", - self.bindings - .iter() - .map(|x| format!("({} {})", x.0, x.1)) - .join(" "), - self.body_expr - ) - } -} - -impl ToDoc for Let { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(%plain-let") - .append(RcDoc::space()) - .append(RcDoc::text("(")) - .append( - RcDoc::intersperse( - self.bindings.iter().map(|x| { - RcDoc::text("(") - .append(x.0.to_doc()) - .append(RcDoc::space()) - .append(x.1.to_doc()) - .append(RcDoc::text(")")) - }), - RcDoc::line(), - ) - .nest(2) - .group(), - ) - .append(RcDoc::text(")")) - .append(RcDoc::line()) - .append(self.body_expr.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } -} - -impl From for ExprKind { - fn from(val: Let) -> Self { - ExprKind::Let(Box::new(val)) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Set { - pub variable: ExprKind, - pub expr: ExprKind, - pub location: SyntaxObject, -} - -impl Set { - pub fn new(variable: ExprKind, expr: ExprKind, location: SyntaxObject) -> Self { - Set { - variable, - expr, - location, - } - } -} - -impl fmt::Display for Set { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(set! {} {})", self.variable, self.expr) - } -} - -impl ToDoc for Set { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(set!") - .append(RcDoc::space()) - .append(self.variable.to_doc()) - .append(RcDoc::line()) - .append(self.expr.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - .group() - } -} - -impl From for ExprKind { - fn from(val: Set) -> Self { - ExprKind::Set(Box::new(val)) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct If { - pub test_expr: ExprKind, - pub then_expr: ExprKind, - pub else_expr: ExprKind, - pub location: SyntaxObject, -} - -impl ToDoc for If { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(if") - .append(RcDoc::space()) - .append(self.test_expr.to_doc()) - .append(RcDoc::line()) - .append(self.then_expr.to_doc()) - .append(RcDoc::line()) - .append(self.else_expr.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - .group() - } -} - -impl fmt::Display for If { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "(if {} {} {})", - self.test_expr, self.then_expr, self.else_expr - ) - } -} - -impl If { - pub fn new( - test_expr: ExprKind, - then_expr: ExprKind, - else_expr: ExprKind, - location: SyntaxObject, - ) -> Self { - If { - test_expr, - then_expr, - else_expr, - location, - } - } -} - -impl From for ExprKind { - fn from(val: If) -> Self { - ExprKind::If(Box::new(val)) - } -} - -// Define normal -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Define { - // This could either be name + args - pub name: ExprKind, - pub body: ExprKind, - pub location: SyntaxObject, -} - -impl fmt::Display for Define { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(define {} {})", self.name, self.body) - } -} - -impl ToDoc for Define { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(define") - .append(RcDoc::space()) - .append(self.name.to_doc()) - .append(RcDoc::line()) - .append(self.body.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } -} - -impl Define { - pub fn new(name: ExprKind, body: ExprKind, location: SyntaxObject) -> Self { - Define { - name, - body, - location, - } - } - - pub(crate) fn is_an_alias_definition(&self) -> Option { - if let Some(atom) = self.body.atom_syntax_object() { - if let TokenType::Identifier(_) = atom.ty { - return Some(atom.syntax_object_id); - } - } - - None - } - - // TODO: address this later - pub(crate) fn is_a_builtin_definition(&self) -> bool { - if let ExprKind::List(l) = &self.body { - match l.first_ident() { - Some(func) if *func == *UNREADABLE_MODULE_GET || *func == *STANDARD_MODULE_GET => { - // return true - - if let Some(module) = l.second_ident() { - return MODULE_IDENTIFIERS.contains(module); - } - } - _ => {} - } - } - - false - } - - pub(crate) fn name_id(&self) -> Option { - self.name.atom_syntax_object().map(|x| x.syntax_object_id) - } -} - -impl From for ExprKind { - fn from(val: Define) -> Self { - ExprKind::Define(Box::new(val)) - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct LambdaFunction { - pub args: Vec, - pub body: ExprKind, - pub location: SyntaxObject, - pub rest: bool, - pub syntax_object_id: usize, -} - -impl Clone for LambdaFunction { - fn clone(&self) -> Self { - Self { - args: self.args.clone(), - body: self.body.clone(), - location: self.location.clone(), - rest: self.rest, - syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), - } - } -} - -impl PartialEq for LambdaFunction { - fn eq(&self, other: &Self) -> bool { - self.args == other.args - && self.body == other.body - && self.location == other.location - && self.rest == other.rest - } -} - -impl fmt::Display for LambdaFunction { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "(lambda ({}) {})", - self.args.iter().map(|x| x.to_string()).join(" "), - self.body - ) - } -} - -impl ToDoc for LambdaFunction { - fn to_doc(&self) -> RcDoc<()> { - if self.rest && self.args.len() == 1 { - RcDoc::text("(λ") - .append(RcDoc::space()) - .append(self.args.first().unwrap().to_doc()) - .append(RcDoc::line()) - .append(self.body.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } else { - RcDoc::text("(λ") - .append(RcDoc::space()) - .append(RcDoc::text("(")) - .append( - RcDoc::intersperse(self.args.iter().map(|x| x.to_doc()), RcDoc::line()) - .nest(2) - .group(), - ) - .append(RcDoc::text(")")) - .append(RcDoc::line()) - .append(self.body.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } - } -} - -impl LambdaFunction { - pub fn new(args: Vec, body: ExprKind, location: SyntaxObject) -> Self { - LambdaFunction { - args, - body, - location, - rest: false, - syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), - } - } - - pub fn new_with_rest_arg(args: Vec, body: ExprKind, location: SyntaxObject) -> Self { - LambdaFunction { - args, - body, - location, - rest: true, - syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), - } - } - - pub fn new_maybe_rest( - args: Vec, - body: ExprKind, - location: SyntaxObject, - rest: bool, - ) -> Self { - LambdaFunction { - args, - body, - location, - rest, - syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), - } - } - - pub fn arguments(&self) -> Option> { - self.args.iter().map(|x| x.atom_identifier()).collect() - } - - pub fn arguments_mut(&mut self) -> impl Iterator { - self.args.iter_mut().filter_map(|x| x.atom_identifier_mut()) - } -} - -impl From for ExprKind { - fn from(val: LambdaFunction) -> Self { - ExprKind::LambdaFunction(Box::new(val)) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Begin { - pub exprs: Vec, - pub location: SyntaxObject, -} - -impl fmt::Display for Begin { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(begin {})", self.exprs.iter().join(" ")) - } -} - -impl ToDoc for Begin { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(begin") - .append(RcDoc::line()) - .nest(5) - .append( - RcDoc::intersperse(self.exprs.iter().map(|x| x.to_doc()), RcDoc::line()) - .nest(5) - .group(), - ) - .append(RcDoc::text(")")) - .nest(1) - .group() - } -} - -impl Begin { - pub fn new(exprs: Vec, location: SyntaxObject) -> Self { - Begin { exprs, location } - } -} - -impl From for ExprKind { - fn from(val: Begin) -> Self { - ExprKind::Begin(val) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Return { - pub expr: ExprKind, - pub location: SyntaxObject, -} - -impl Return { - pub fn new(expr: ExprKind, location: SyntaxObject) -> Self { - Return { expr, location } - } -} - -impl ToDoc for Return { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(return") - .append(RcDoc::line()) - .append(self.expr.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } -} - -impl fmt::Display for Return { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(return! {})", self.expr) - } -} - -impl From for ExprKind { - fn from(val: Return) -> Self { - ExprKind::Return(Box::new(val)) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Require { - pub modules: Vec, - pub location: SyntaxObject, -} - -impl Require { - pub fn new(modules: Vec, location: SyntaxObject) -> Self { - Require { modules, location } - } -} - -impl ToDoc for Require { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(require") - .append(RcDoc::line()) - .append( - RcDoc::intersperse(self.modules.iter().map(|x| x.to_doc()), RcDoc::line()) - .nest(2) - .group(), - ) - .append(RcDoc::text(")")) - .nest(2) - } -} - -impl fmt::Display for Require { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(require {})", self.modules.iter().join(" ")) - } -} - -impl From for ExprKind { - fn from(val: Require) -> Self { - ExprKind::Require(val) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct List { - pub args: Vec, - pub(crate) syntax_object_id: usize, -} - -impl PartialEq for List { - fn eq(&self, other: &Self) -> bool { - self.args == other.args - } -} - -impl List { - pub fn new(args: Vec) -> Self { - List { - args, - syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), - } - } - - pub fn is_empty(&self) -> bool { - self.args.is_empty() - } - - pub fn rest_mut(&mut self) -> Option<&mut [ExprKind]> { - self.args.split_first_mut().map(|x| x.1) - } - - pub fn first_ident_mut(&mut self) -> Option<&mut InternedString> { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(s), - .. - }, - })) = self.args.first_mut() - { - Some(s) - } else { - None - } - } - - pub fn is_require(&self) -> bool { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Require, - .. - }, - })) = self.args.first() - { - true - } else { - false - } - } - - pub fn is_define_syntax(&self) -> bool { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::DefineSyntax, - .. - }, - })) = self.args.first() - { - self.args.len() == 3 - } else { - false - } - } - - pub fn is_syntax_rules(&self) -> bool { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::SyntaxRules, - .. - }, - })) = self.args.first() - { - self.args.len() > 2 - } else { - false - } - } - - pub fn is_quote(&self) -> bool { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Quote, - .. - }, - })) = self.args.first() - { - true - } else { - false - } - } - - pub fn first_ident(&self) -> Option<&InternedString> { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(s), - .. - }, - })) = self.args.first() - { - Some(s) - } else { - None - } - } - - pub fn second_ident(&self) -> Option<&InternedString> { - if let Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(s), - .. - }, - })) = self.args.get(1) - { - Some(s) - } else { - None - } - } - - pub fn is_anonymous_function_call(&self) -> bool { - matches!(self.args.get(0), Some(ExprKind::LambdaFunction(_))) - } - - pub fn is_a_builtin_expr(&self) -> bool { - matches!(self.first_ident(), Some(func) if *func == *UNREADABLE_MODULE_GET || *func == *STANDARD_MODULE_GET) - } - - pub fn first_func_mut(&mut self) -> Option<&mut LambdaFunction> { - if let Some(ExprKind::LambdaFunction(l)) = self.args.get_mut(0) { - Some(l) - } else { - None - } - } - - pub fn first_func(&self) -> Option<&LambdaFunction> { - if let Some(ExprKind::LambdaFunction(l)) = self.args.get(0) { - Some(l) - } else { - None - } - } -} - -impl ToDoc for List { - fn to_doc(&self) -> RcDoc<()> { - if let Some(func) = self.first_func() { - let mut args_iter = self.args.iter(); - args_iter.next(); - - let bindings = func.args.iter().zip(args_iter); - - RcDoc::text("(let") - .append(RcDoc::space()) - .append(RcDoc::text("(")) - .append( - RcDoc::intersperse( - bindings.map(|x| { - RcDoc::text("(") - .append(x.0.to_doc()) - .append(RcDoc::space()) - .append(x.1.to_doc()) - .append(RcDoc::text(")")) - }), - RcDoc::line(), - ) - .nest(2) - .group(), - ) - .append(RcDoc::text(")")) - .append(RcDoc::line()) - .append(func.body.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } else { - RcDoc::text("(") - .append( - RcDoc::intersperse(self.args.iter().map(|x| x.to_doc()), RcDoc::line()) - .nest(1) - .group(), - ) - .append(RcDoc::text(")")) - .nest(2) - .group() - } - } -} - -impl fmt::Display for List { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "({})", self.args.iter().join(" ")) - } -} - -impl From for ExprKind { - fn from(val: List) -> Self { - ExprKind::List(val) - } -} - -impl Deref for List { - type Target = [ExprKind]; - - fn deref(&self) -> &[ExprKind] { - &self.args - } -} - -// and we'll implement IntoIterator -impl IntoIterator for List { - type Item = ExprKind; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.args.into_iter() - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Quote { - pub expr: ExprKind, - pub location: SyntaxObject, -} - -impl Quote { - pub fn new(expr: ExprKind, location: SyntaxObject) -> Self { - Quote { expr, location } - } -} - -impl ToDoc for Quote { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(quote") - .append(RcDoc::line()) - .append(self.expr.to_doc()) - .append(RcDoc::text(")")) - .nest(2) - } -} - -impl fmt::Display for Quote { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(quote {})", self.expr) - } -} - -impl From for ExprKind { - fn from(val: Quote) -> Self { - ExprKind::Quote(Box::new(val)) - } -} - -// TODO figure out how many fields a macro has -// put it into here nicely -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct Macro { - pub name: Box, - pub syntax_rules: SyntaxRules, - pub location: SyntaxObject, -} - -impl fmt::Display for Macro { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "(define-syntax {} {})", self.name, self.syntax_rules) - } -} - -impl ToDoc for Macro { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(define-syntax") - .append(RcDoc::line()) - .append(self.name.to_doc()) - .append(RcDoc::line()) - .append(self.syntax_rules.to_doc()) - .append(RcDoc::text(")")) - .nest(1) - .group() - } -} - -impl Macro { - pub fn new(name: ExprKind, syntax_rules: SyntaxRules, location: SyntaxObject) -> Self { - Macro { - name: Box::new(name), - syntax_rules, - location, - } - } -} - -impl From for ExprKind { - fn from(val: Macro) -> Self { - ExprKind::Macro(val) - } -} - -// TODO figure out a good mapping immediately to a macro that can be interpreted -// by the expander -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct SyntaxRules { - pub syntax: Vec, - pub patterns: Vec, - pub location: SyntaxObject, -} - -impl SyntaxRules { - pub fn new(syntax: Vec, patterns: Vec, location: SyntaxObject) -> Self { - SyntaxRules { - syntax, - patterns, - location, - } - } -} - -impl fmt::Display for SyntaxRules { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!( - f, - "(syntax-rules ({}) {})", - self.syntax.iter().map(|x| x.to_string()).join(" "), - self.patterns.iter().map(|x| x.to_string()).join("\n") - ) - } -} - -impl ToDoc for SyntaxRules { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("(syntax-rules") - .append(RcDoc::line()) - .append(RcDoc::text("(")) - .append( - RcDoc::intersperse(self.syntax.iter().map(|x| x.to_doc()), RcDoc::line()) - .nest(1) - .group(), - ) - .append(RcDoc::text(")")) - .append(RcDoc::line()) - .append( - RcDoc::intersperse(self.patterns.iter().map(|x| x.to_doc()), RcDoc::line()) - .nest(2) - .group(), - ) - .append(RcDoc::text(")")) - .nest(2) - } -} - -impl From for ExprKind { - fn from(val: SyntaxRules) -> Self { - ExprKind::SyntaxRules(val) - } -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct PatternPair { - pub pattern: ExprKind, - pub body: ExprKind, -} - -impl PatternPair { - pub fn new(pattern: ExprKind, body: ExprKind) -> Self { - PatternPair { pattern, body } - } -} - -impl ToDoc for PatternPair { - fn to_doc(&self) -> RcDoc<()> { - RcDoc::text("[") - .append(self.pattern.to_doc()) - .append(RcDoc::line()) - .append(self.body.to_doc()) - .append(RcDoc::text("]")) - .nest(1) - .group() - } -} - -impl fmt::Display for PatternPair { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "[{}\n{}]", self.pattern, self.body) - } -} - -#[inline] -pub(crate) fn parse_if( - mut value_iter: I, - syn: SyntaxObject, -) -> std::result::Result -where - I: Iterator, -{ - // let mut value_iter = value.into_iter(); - value_iter.next(); - - let ret_value = If::new( - value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "if expects a test condition, found none".to_string(), - syn.span, - None, - ) - })?, - value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "if expects a then condition, found none".to_string(), - syn.span, - None, - ) - })?, - // Replace else condition with just a void if its not found! - value_iter - .next() - .unwrap_or_else(|| ExprKind::ident("#%prim.void")), - // ok_or_else(|| { - // ParseError::SyntaxError( - // "if expects an else condition, found none".to_string(), - // syn.span, - // None, - // ) - // })?, - syn.clone(), - ) - .into(); - - if value_iter.next().is_some() { - Err(ParseError::SyntaxError( - "if takes only 3 expressions".to_string(), - syn.span, - None, - )) - } else { - Ok(ret_value) - } -} - -#[inline] -pub(crate) fn parse_define( - mut value_iter: I, - syn: SyntaxObject, -) -> std::result::Result -where - I: Iterator, -{ - value_iter.next(); - - match value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "define expects an identifier, found none".to_string(), - syn.span, - None, - ) - })? { - // TODO maybe add implicit begin here - // maybe do it later, not sure - ExprKind::List(l) => { - let name_ref = l.args.first().ok_or_else(|| { - ParseError::SyntaxError( - "define expected a function name, found none".to_string(), - syn.span, - None, - ) - })?; - - if let ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Identifier(datum_syntax), - .. - }, - }) = name_ref - { - if *datum_syntax == *DATUM_SYNTAX { - return Ok(ExprKind::Define(Box::new(Define::new( - ExprKind::List(List::new(l.args)), - { - let v = value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "define statement expected a body, found none".to_string(), - syn.span, - None, - ) - })?; - if value_iter.next().is_some() { - return Err(ParseError::SyntaxError( - "Define expected only one expression after the identifier" - .to_string(), - syn.span, - None, - )); - } - v - }, - syn, - )))); - } - } - - let mut args = l.args.into_iter(); - - let name = args.next().ok_or_else(|| { - ParseError::SyntaxError( - "define expected a function name, found none".to_string(), - syn.span, - None, - ) - })?; - - let args = args.collect(); - - let body_exprs: Vec<_> = value_iter.collect(); - - if body_exprs.is_empty() { - return Err(ParseError::SyntaxError( - "Function body cannot be empty".to_string(), - syn.span, - None, - )); - } - - let body = if body_exprs.len() == 1 { - body_exprs[0].clone() - } else { - ExprKind::Begin(Begin::new( - body_exprs, - SyntaxObject::default(TokenType::Begin), - )) - }; - - let lambda = ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - args, - body, - SyntaxObject::new(TokenType::Lambda, syn.span), - ))); - - Ok(ExprKind::Define(Box::new(Define::new(name, lambda, syn)))) - } - ExprKind::Atom(a) => Ok(ExprKind::Define(Box::new(Define::new( - ExprKind::Atom(a), - { - let v = value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "define statement expected a body, found none".to_string(), - syn.span, - None, - ) - })?; - if value_iter.next().is_some() { - return Err(ParseError::SyntaxError( - "Define expected only one expression after the identifier".to_string(), - syn.span, - None, - )); - } - v - }, - syn, - )))), - - _ => Err(ParseError::SyntaxError( - "Define expects either an identifier or a list with the function name and arguments" - .to_string(), - syn.span, - None, - )), - } -} - -#[inline] -pub(crate) fn parse_new_let( - mut value_iter: I, - syn: SyntaxObject, -) -> std::result::Result -where - I: Iterator, -{ - value_iter.next(); - - let let_pairs = if let ExprKind::List(l) = value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "let expected a list of variable bindings pairs in the second position, found none" - .to_string(), - syn.span, - None, - ) - })? { - l.args - } else { - return Err(ParseError::SyntaxError( - "let expects a list of variable bindings pairs in the second position".to_string(), - syn.span, - None, - )); - }; - - let body_exprs: Vec<_> = value_iter.collect(); - - if body_exprs.is_empty() { - return Err(ParseError::SyntaxError( - "let expects an expression, found none".to_string(), - syn.span, - None, - )); - } - - let body = if body_exprs.len() == 1 { - body_exprs[0].clone() - } else { - ExprKind::Begin(Begin::new( - body_exprs, - SyntaxObject::default(TokenType::Begin), - )) - }; - - let mut pairs = Vec::with_capacity(let_pairs.len()); - - for pair in let_pairs { - if let ExprKind::List(l) = pair { - let pair = l.args; - - if pair.len() != 2 { - return Err(ParseError::SyntaxError( - format!("let expected a list of variable binding pairs, found a pair with length {}", - pair.len()), - syn.span, None - )); - } - - let mut iter = pair.into_iter(); - - let identifier = iter.next().unwrap(); - let application_arg = iter.next().unwrap(); - pairs.push((identifier, application_arg)) - } else { - return Err(ParseError::SyntaxError( - "let expected a list of variable binding pairs".to_string(), - syn.span, - None, - )); - } - } - - Ok(ExprKind::Let(Let::new(pairs, body, syn).into())) -} - -#[inline] -fn parse_named_let( - mut value_iter: I, - syn: SyntaxObject, - name: ExprKind, -) -> std::result::Result -where - I: Iterator, -{ - let pairs = if let ExprKind::List(l) = value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "named let expects a list of argument id and init expr pairs, found none".to_string(), - syn.span, - None, - ) - })? { - l.args - } else { - return Err(ParseError::SyntaxError( - "named let expects a list of variable bindings pairs in the second position" - .to_string(), - syn.span, - None, - )); - }; - - let body_exprs: Vec<_> = value_iter.collect(); - - if body_exprs.is_empty() { - return Err(ParseError::SyntaxError( - "let expects an expression, found none".to_string(), - syn.span, - None, - )); - } - - let body = if body_exprs.len() == 1 { - body_exprs[0].clone() - } else { - ExprKind::Begin(Begin::new( - body_exprs, - SyntaxObject::default(TokenType::Begin), - )) - }; - - let mut arguments = Vec::with_capacity(pairs.len()); - - // insert args at the end - // put the function in the inside - let mut application_args = Vec::with_capacity(pairs.len()); - - for pair in pairs { - if let ExprKind::List(l) = pair { - let pair = l.args; - - if pair.len() != 2 { - return Err(ParseError::SyntaxError( - format!("let expected a list of variable binding pairs, found a pair with length {}", - pair.len()), - syn.span, None - )); - } - - let identifier = pair[0].clone(); - let application_arg = pair[1].clone(); - - arguments.push(identifier); - application_args.push(application_arg); - } else { - return Err(ParseError::SyntaxError( - "let expected a list of variable binding pairs".to_string(), - syn.span, - None, - )); - } - } - - // This is the body of the define - let function: ExprKind = LambdaFunction::new(arguments, body, syn.clone()).into(); - - let define: ExprKind = Define::new(name.clone(), function, syn.clone()).into(); - - let application: ExprKind = { - let mut application = vec![name]; - application.append(&mut application_args); - List::new(application).into() - }; - - let begin = ExprKind::Begin(Begin::new(vec![define, application], syn.clone())); - - // Wrap the whole thing inside of an empty function application, to create a new scope - - Ok(List::new(vec![LambdaFunction::new(vec![], begin, syn).into()]).into()) -} - -#[inline] -pub(crate) fn parse_let( - mut value_iter: I, - syn: SyntaxObject, -) -> std::result::Result -where - I: Iterator, -{ - value_iter.next(); - - let let_pairs = match value_iter.next().ok_or_else(|| { - ParseError::SyntaxError( - "let expected a list of variable bindings pairs in the second position, found none" - .to_string(), - syn.span, - None, - ) - })? { - // Standard let - ExprKind::List(l) => l.args, - // Named let - name @ ExprKind::Atom(_) => return parse_named_let(value_iter, syn, name), - _ => { - return Err(ParseError::SyntaxError( - "let expects a list of variable bindings pairs in the second position".to_string(), - syn.span, - None, - )); - } - }; - - let body_exprs: Vec<_> = value_iter.collect(); - - if body_exprs.is_empty() { - return Err(ParseError::SyntaxError( - "let expects an expression, found none".to_string(), - syn.span, - None, - )); - } - - let body = if body_exprs.len() == 1 { - body_exprs[0].clone() - } else { - ExprKind::Begin(Begin::new( - body_exprs, - SyntaxObject::default(TokenType::Begin), - )) - }; - - let mut arguments = Vec::with_capacity(let_pairs.len()); - - // insert args at the end - // put the function in the inside - let mut application_args = Vec::with_capacity(let_pairs.len()); - - for pair in let_pairs { - if let ExprKind::List(l) = pair { - let pair = l.args; - - if pair.len() != 2 { - return Err(ParseError::SyntaxError( - format!("let expected a list of variable binding pairs, found a pair with length {}", - pair.len()), - syn.span, None - )); - } - - let identifier = pair[0].clone(); - let application_arg = pair[1].clone(); - - arguments.push(identifier); - application_args.push(application_arg); - } else { - return Err(ParseError::SyntaxError( - "let expected a list of variable binding pairs".to_string(), - syn.span, - None, - )); - } - } - - let mut function: Vec = vec![LambdaFunction::new(arguments, body, syn).into()]; - - function.append(&mut application_args); - - Ok(ExprKind::List(List::new(function))) -} - -#[inline] -pub(crate) fn parse_single_argument( - mut value_iter: I, - syn: SyntaxObject, - name: &'static str, - constructor: fn(ExprKind, SyntaxObject) -> ExprKind, -) -> Result -where - I: Iterator, -{ - value_iter.next(); - - let func = value_iter.next().ok_or_else(|| { - ParseError::ArityMismatch( - format!("{name} expected one argument, found none"), - syn.span, - None, - ) - })?; - - if value_iter.next().is_some() { - Err(ParseError::SyntaxError( - format!("{name} expects only one argument"), - syn.span, - None, - )) - } else { - Ok(constructor(func, syn)) - } -} - -impl TryFrom> for ExprKind { - type Error = ParseError; - fn try_from(value: Vec) -> std::result::Result { - // let mut value = value.into_iter().peekable(); - - // TODO -> get rid of this clone on the first value - if let Some(f) = value.first().cloned() { - match f { - ExprKind::Atom(a) => { - // let value = value.into_iter(); - - match &a.syn.ty { - // Have this also match on the first argument being a TokenType::Identifier("if") - // Do the same for the rest of the arguments - TokenType::If => parse_if(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *IF => { - parse_if(value.into_iter(), a.syn.clone()) - } - - TokenType::Define => parse_define(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *DEFINE => { - parse_define(value.into_iter(), a.syn.clone()) - } - - TokenType::Let => parse_let(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *LET => { - parse_let(value.into_iter(), a.syn.clone()) - } - - // TODO: Deprecate - TokenType::TestLet => parse_new_let(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *PLAIN_LET => { - parse_new_let(value.into_iter(), a.syn.clone()) - } - - TokenType::Quote => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "quote", - |expr, syn| Quote::new(expr, syn).into(), - ), - TokenType::Identifier(expr) if *expr == *QUOTE => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "quote", - |expr, syn| Quote::new(expr, syn).into(), - ), - - TokenType::Return => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "return!", - |expr, syn| Return::new(expr, syn).into(), - ), - TokenType::Identifier(expr) if *expr == *RETURN => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "return!", - |expr, syn| Return::new(expr, syn).into(), - ), - - TokenType::Require => parse_require(&a, value), - TokenType::Identifier(expr) if *expr == *REQUIRE => { - parse_require(&a, value) - } - - TokenType::Set => parse_set(&a, value), - TokenType::Identifier(expr) if *expr == *SET => parse_set(&a, value), - - TokenType::Begin => parse_begin(&a, value), - TokenType::Identifier(expr) if *expr == *BEGIN => parse_begin(&a, value), - - TokenType::Lambda => parse_lambda(&a, value), - TokenType::Identifier(expr) - if *expr == *LAMBDA - || *expr == *LAMBDA_FN - || *expr == *LAMBDA_SYMBOL => - { - parse_lambda(&a, value) - } - - TokenType::DefineSyntax => { - let syn = a.syn.clone(); - - if value.len() < 3 { - return Err(ParseError::SyntaxError( - format!("define-syntax expects 2 arguments - the name of the macro and the syntax-rules, found {}", value.len()), syn.span, None - )); - } - - // println!("{}", value.iter().map(|x| x.to_pretty(60)).join("\n\n")); - - let mut value_iter = value.into_iter(); - value_iter.next(); - - let name = value_iter.next().unwrap(); - - let syntax = value_iter.next(); - - // println!("{:?}", syntax); - - let syntax_rules = if let Some(ExprKind::SyntaxRules(s)) = syntax { - s - } else { - return Err(ParseError::SyntaxError( - "define-syntax expected a syntax-rules object".to_string(), - syn.span, - None, - )); - }; - - Ok(ExprKind::Macro(Macro::new(name, syntax_rules, syn))) - } - TokenType::SyntaxRules => { - let syn = a.syn.clone(); - - if value.len() < 3 { - return Err(ParseError::SyntaxError( - format!("syntax-rules expects a list of introduced syntax, and at least one pattern-body pair, found {} arguments", value.len()), syn.span, None - )); - } - - let mut value_iter = value.into_iter(); - value_iter.next(); - - let syntax_vec = if let Some(ExprKind::List(l)) = value_iter.next() { - l.args - } else { - return Err(ParseError::SyntaxError( - "syntax-rules expects a list of new syntax forms used in the macro".to_string(), syn.span, None)); - }; - - let mut pairs = Vec::new(); - let rest: Vec<_> = value_iter.collect(); - - for pair in rest { - if let ExprKind::List(l) = pair { - if l.args.len() != 2 { - return Err(ParseError::SyntaxError( - "syntax-rules requires only one pattern to one body" - .to_string(), - syn.span, - None, - )); - } - - let mut pair_iter = l.args.into_iter(); - let pair_object = PatternPair::new( - pair_iter.next().unwrap(), - pair_iter.next().unwrap(), - ); - pairs.push(pair_object); - } else { - return Err(ParseError::SyntaxError( - "syntax-rules requires pattern to expressions to be in a list".to_string(), syn.span, None - )); - } - } - - Ok(ExprKind::SyntaxRules(SyntaxRules::new( - syntax_vec, pairs, syn, - ))) - } - _ => Ok(ExprKind::List(List::new(value))), - } - } - _ => Ok(ExprKind::List(List::new(value))), - } - } else { - Ok(ExprKind::List(List::new(vec![]))) - } - } -} - -pub(crate) fn parse_lambda(a: &Atom, value: Vec) -> Result { - let syn = a.syn.clone(); - if value.len() < 3 { - return Err(ParseError::SyntaxError( - format!( - "lambda expected at least 2 arguments - the bindings list and one or more expressions, found {} instead", - value.len() - ), - syn.span, None - )); - } - let mut value_iter = value.into_iter(); - value_iter.next(); - let arguments = value_iter.next(); - match arguments { - Some(ExprKind::List(l)) => { - let args = l.args; - - for arg in &args { - if let ExprKind::Atom(_) = arg { - continue; - } else { - return Err(ParseError::SyntaxError( - format!( - "lambda function expects a list of identifiers, found: {}", - List::new(args) - ), - syn.span, - None, - )); - } - } - - let body_exprs: Vec<_> = value_iter.collect(); - - let body = if body_exprs.len() == 1 { - body_exprs.into_iter().next().unwrap() - } else { - ExprKind::Begin(Begin::new( - body_exprs, - SyntaxObject::default(TokenType::Begin), - )) - }; - - Ok(ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - args, body, syn, - )))) - } - Some(ExprKind::Atom(a)) => { - let body_exprs: Vec<_> = value_iter.collect(); - - let body = if body_exprs.len() == 1 { - body_exprs.into_iter().next().unwrap() - } else { - ExprKind::Begin(Begin::new( - body_exprs, - SyntaxObject::default(TokenType::Begin), - )) - }; - - // (lambda x ...) => x is a rest arg, becomes a list at run time - Ok(ExprKind::LambdaFunction(Box::new( - LambdaFunction::new_with_rest_arg(vec![ExprKind::Atom(a)], body, syn), - ))) - } - _ => { - // TODO -> handle case like - // (lambda x 10) <- where x is immediately bound to be a rest arg - // This should be fairly trivial in this case since we can just put the - // first thing into a vec for the lambda node - Err(ParseError::SyntaxError( - format!("lambda function expected a list of identifiers, found: {arguments:?}"), - syn.span, - None, - )) - } - } -} - -pub(crate) fn parse_set(a: &Atom, value: Vec) -> Result { - let syn = a.syn.clone(); - if value.len() != 3 { - return Err(ParseError::ArityMismatch( - "set! expects an identifier and an expression".to_string(), - syn.span, - None, - )); - } - let mut value_iter = value.into_iter(); - value_iter.next(); - let identifier = value_iter.next().unwrap(); - let expression = value_iter.next().unwrap(); - Ok(ExprKind::Set(Box::new(Set::new( - identifier, expression, syn, - )))) -} - -pub(crate) fn parse_require(a: &Atom, value: Vec) -> Result { - let syn = a.syn.clone(); - if value.len() < 2 { - return Err(ParseError::ArityMismatch( - "require expects at least one identifier or string".to_string(), - syn.span, - None, - )); - } - let mut value_iter = value.into_iter(); - value_iter.next(); - let expressions = value_iter - .map(|x| { - match &x { - ExprKind::Atom(_) | ExprKind::List(_) => Ok(x), - _ => Err(ParseError::SyntaxError( - "require expects atoms".to_string(), - syn.span, - None, - )), - } - - // if let ExprKind::Atom(a) = x { - // Ok(a) - // } else { - - // } - }) - .collect::, ParseError>>()?; - Ok(ExprKind::Require(Require::new(expressions, syn))) -} - -pub(crate) fn parse_begin(a: &Atom, value: Vec) -> Result { - let syn = a.syn.clone(); - let mut value_iter = value.into_iter(); - value_iter.next(); - Ok(ExprKind::Begin(Begin::new(value_iter.collect(), syn))) -} - -#[cfg(test)] -mod display_tests { - - use super::*; - use crate::parser::parser::{Parser, Result}; - - fn parse(expr: &str) -> ExprKind { - let a: Result> = Parser::new(expr, None).collect(); - - a.unwrap()[0].clone() - } - - #[test] - fn display_lambda_quote() { - let expression = "(lambda (x) (quote x))"; - let parsed_expr = parse(expression); - let expected = "(lambda (x) (quote x))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_list() { - let expression = "(list 1 2 3 4)"; - let parsed_expr = parse(expression); - let expected = "(list 1 2 3 4)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_lambda() { - let expression = "(lambda (x) (+ x 10))"; - let parsed_expr = parse(expression); - let expected = "(lambda (x) (+ x 10))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_set() { - let expression = "(set! x 10)"; - let parsed_expr = parse(expression); - let expected = "(set! x 10)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_panic() { - let expression = "(panic! 12345)"; - let parsed_expr = parse(expression); - let expected = "(panic! 12345)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_begin() { - let expression = "(begin 1 2 3 4 5)"; - let parsed_expr = parse(expression); - let expected = "(begin 1 2 3 4 5)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_define_normal() { - let expression = "(define a 10)"; - let parsed_expr = parse(expression); - let expected = "(define a 10)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_define_function() { - let expression = "(define (applesauce x y z) (+ x y z))"; - let parsed_expr = parse(expression); - let expected = "(define applesauce (lambda (x y z) (+ x y z)))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_let() { - let expression = "(let ((x 10)) (+ x 10))"; - let parsed_expr = parse(expression); - let expected = "((lambda (x) (+ x 10)) 10)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_apply() { - let expression = "(apply + (list 1 2 3 4))"; - let parsed_expr = parse(expression); - let expected = "(apply + (list 1 2 3 4))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_transduce() { - let expression = "(transduce 1 2 3 4)"; - let parsed_expr = parse(expression); - let expected = "(transduce 1 2 3 4)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_execute_two_args() { - let expression = "(execute 1 2)"; - let parsed_expr = parse(expression); - let expected = "(execute 1 2)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_execute_three_args() { - let expression = "(execute 1 2 3)"; - let parsed_expr = parse(expression); - let expected = "(execute 1 2 3)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_if() { - let expression = "(if 1 2 3)"; - let parsed_expr = parse(expression); - let expected = "(if 1 2 3)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_quote() { - let expression = "'(1 2 3 4)"; - let parsed_expr = parse(expression); - let expected = "(quote (1 2 3 4))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_read() { - let expression = "(read '(1 2 3 4))"; - let parsed_expr = parse(expression); - let expected = "(read (quote (1 2 3 4)))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_return() { - let expression = "(return! 10)"; - let parsed_expr = parse(expression); - let expected = "(return! 10)"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_struct() { - let expression = "(struct Apple (a b c))"; - let parsed_expr = parse(expression); - let expected = "(struct Apple (a b c))"; - assert_eq!(parsed_expr.to_string(), expected); - } - - #[test] - fn display_eval() { - let expression = "(eval 'a)"; - let parsed_expr = parse(expression); - let expected = "(eval (quote a))"; - assert_eq!(parsed_expr.to_string(), expected); - } -} - -#[cfg(test)] -mod pretty_print_tests { - use super::*; - use crate::parser::parser::{Parser, Result}; - - // pub fn to_pretty(&self, width: usize) -> String { - // let mut w = Vec::new(); - // self.to_doc().render(width, &mut w).unwrap(); - // String::from_utf8(w).unwrap() - // } - - fn parse(expr: &str) -> ExprKind { - let a: Result> = Parser::new(expr, None).collect(); - - a.unwrap()[0].clone() - } - - #[test] - fn pretty_set() { - let expression = r#" - (define test-function - (lambda (a b c) - (begin - (set! bananas 10) - (if applesauce 100 #f) - (if applesauce 100 (if applesauce 100 #f)))))"#; - let parsed_expr = parse(expression); - let _output = parsed_expr.to_pretty(45); - - assert!(true) - } -} diff --git a/crates/steel-core/src/parser/expand_visitor.rs b/crates/steel-core/src/parser/expand_visitor.rs index 8f77ff328..4f059688d 100644 --- a/crates/steel-core/src/parser/expand_visitor.rs +++ b/crates/steel-core/src/parser/expand_visitor.rs @@ -6,6 +6,7 @@ use crate::compiler::passes::reader::MultipleArityFunctions; use crate::compiler::passes::Folder; use crate::compiler::program::REQUIRE_DYLIB; use crate::parser::ast::ExprKind; +use crate::parser::parser::SyntaxObject; use crate::steel_vm::builtin::BuiltInModule; use crate::steel_vm::engine::ModuleContainer; use crate::{compiler::program::REQUIRE_BUILTIN, rvals::Result}; @@ -14,7 +15,8 @@ use crate::{ compiler::program::{AS_KEYWORD, DOC_MACRO}, parser::tokens::TokenType, }; -use crate::{expr_list, parser::parser::SyntaxObject}; + +use steel_parser::expr_list; use super::{ ast::{Atom, Begin, Define, LambdaFunction, List, Quote}, diff --git a/crates/steel-core/src/parser/expander.rs b/crates/steel-core/src/parser/expander.rs index d2c3bd9af..e62accd0e 100644 --- a/crates/steel-core/src/parser/expander.rs +++ b/crates/steel-core/src/parser/expander.rs @@ -34,15 +34,6 @@ pub fn path_from_working_dir>(path: P) -> std::io::Result Otherwise we're in for deep trouble -// trying to serialize and deserialize this -#[derive(Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash)] -#[repr(transparent)] -pub struct InternedString(Spur); - -impl Serialize for InternedString { - fn serialize(&self, serializer: S) -> Result - where - S: serde::Serializer, - { - self.resolve().serialize(serializer) - } -} - -impl<'de> Deserialize<'de> for InternedString { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - let key = <&str>::deserialize(deserializer)?; - - Ok(InternedString::from(key)) - } -} - -impl InternedString { - pub fn from_static(ident: &'static str) -> Self { - Self( - INTERNER - .get_or_init(|| Arc::new(ThreadedRodeo::new())) - .get_or_intern_static(ident), - ) - } - - pub fn from_string(ident: String) -> Self { - Self( - INTERNER - .get_or_init(|| Arc::new(ThreadedRodeo::new())) - .get_or_intern(ident), - ) - } - - pub fn new(key: usize) -> Self { - Self(Spur::try_from_usize(key).unwrap()) - } - - pub fn get(self) -> Spur { - self.0 - } - - pub fn try_get(ident: &str) -> Option { - INTERNER.get().unwrap().get(ident).map(InternedString) - } - - #[doc(hidden)] - pub fn as_u32(self) -> u32 { - self.get().into_usize() as u32 - } - - pub fn resolve(&self) -> &str { - resolve(&self.0) - } -} - -impl From<&str> for InternedString { - fn from(ident: &str) -> Self { - Self( - INTERNER - .get_or_init(|| Arc::new(ThreadedRodeo::new())) - .get_or_intern(ident), - ) - } -} - -impl From for InternedString { - fn from(ident: String) -> Self { - Self::from_string(ident) - } -} - -impl From for InternedString { - fn from(spur: Spur) -> Self { - Self(spur) - } -} - -impl fmt::Debug for InternedString { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.get().into_usize()) - } -} - -impl fmt::Display for InternedString { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.resolve()) - } -} +use crate::{rvals::SteelString, SteelVal}; impl From for SteelVal { fn from(value: InternedString) -> Self { @@ -114,63 +13,3 @@ impl From for SteelString { value.resolve().into() } } - -// impl Serialize for InternedString { -// fn serialize(&self, serializer: S) -> Result -// where -// S: Serializer, -// { -// serializer.serialize_str(self.resolve()) -// } -// } - -// impl Dese - -use lasso::ThreadedRodeo; -// use std::sync::OnceLock; - -use crate::{rvals::SteelString, SteelVal}; - -static INTERNER: OnceCell> = OnceCell::new(); - -pub fn take_interner() -> Arc { - Arc::clone(INTERNER.get().unwrap()) -} - -pub fn initialize_with(interner: Arc) -> Result<(), Arc> { - INTERNER.set(interner) -} - -pub fn get_interner() -> Option<&'static Arc> { - INTERNER.get() -} - -pub fn add_interner(interner: Arc) { - let guard = INTERNER.get().unwrap(); - - for key in interner.strings() { - guard.get_or_intern(key); - } -} - -#[test] -fn test_initialization() { - INTERNER.get_or_init(|| Arc::new(ThreadedRodeo::new())); - let key = INTERNER.get().unwrap().get_or_intern_static("hello world"); - - let resolved_string = INTERNER.get().unwrap().resolve(&key); - - println!("resolved string: {resolved_string:?}"); -} - -// fn intern(key: String) -> Spur { -// INTERNER.get().unwrap().get_or_intern(key) -// } - -// fn intern_static(key: &'static str) -> Spur { -// INTERNER.get().unwrap().get_or_intern_static(key) -// } - -fn resolve(key: &Spur) -> &str { - INTERNER.get().unwrap().resolve(key) -} diff --git a/crates/steel-core/src/parser/kernel.rs b/crates/steel-core/src/parser/kernel.rs index d8bf0809b..9076bcec1 100644 --- a/crates/steel-core/src/parser/kernel.rs +++ b/crates/steel-core/src/parser/kernel.rs @@ -7,7 +7,6 @@ use steel_parser::tokens::TokenType; use crate::{ compiler::{passes::analysis::SemanticAnalysis, program::RawProgramWithSymbols}, - expr_list, parser::{ ast::{Atom, Set}, parser::SyntaxObject, @@ -17,6 +16,8 @@ use crate::{ }; use crate::{stdlib::KERNEL, steel_vm::engine::Engine, SteelVal}; +use steel_parser::expr_list; + use super::{ ast::{ExprKind, TryFromSteelValVisitorForExprKind}, interner::InternedString, diff --git a/crates/steel-core/src/parser/parser.rs b/crates/steel-core/src/parser/parser.rs index 173240910..23eb0c39a 100644 --- a/crates/steel-core/src/parser/parser.rs +++ b/crates/steel-core/src/parser/parser.rs @@ -1,28 +1,11 @@ -use crate::{ - compiler::program::{ - BEGIN, DEFINE, IF, LAMBDA, LAMBDA_FN, LAMBDA_SYMBOL, LET, PLAIN_LET, QUASIQUOTE, QUOTE, - RAW_UNQUOTE, RAW_UNQUOTE_SPLICING, REQUIRE, RETURN, SET, UNQUOTE, UNQUOTE_SPLICING, - }, - parser::lexer::TokenStream, - rvals::IntoSteelVal, -}; -use crate::{ - parser::tokens::{Token, TokenType, TokenType::*}, - rvals::FromSteelVal, -}; +use crate::rvals::IntoSteelVal; +use crate::{parser::tokens::TokenType::*, rvals::FromSteelVal}; -use std::result; use std::str; +use std::sync::{Arc, Mutex}; use std::{collections::HashMap, path::PathBuf}; -use std::{ - rc::Rc, - sync::{Arc, Mutex}, -}; -use steel_parser::{lexer::OwnedTokenStream, lexer::ToOwnedString, tokens::MaybeBigInt}; - -use crate::parser::span::Span; +use steel_parser::tokens::MaybeBigInt; -use crate::parser::ast::*; use serde::{Deserialize, Serialize}; use std::convert::TryFrom; @@ -30,20 +13,11 @@ use crate::rerrs::{ErrorKind, SteelErr}; use crate::rvals::SteelVal; use crate::rvals::SteelVal::*; -use super::{ - ast::{self, SyntaxRules}, - interner::InternedString, +pub use steel_parser::parser::{ + lower_entire_ast, lower_macro_and_require_definitions, lower_syntax_rules, FunctionId, ListId, + ParseError, Parser, RawSyntaxObject, SourceId, SyntaxObject, SyntaxObjectId, SYNTAX_OBJECT_ID, }; -use std::sync::atomic::{AtomicUsize, Ordering}; - -pub use steel_parser::parser::SourceId; - -// #[derive( -// Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, -// )] -// pub struct SourceId(pub(crate) usize); - impl IntoSteelVal for SourceId { fn into_steelval(self) -> crate::rvals::Result { self.0.into_steelval() @@ -128,188 +102,8 @@ impl Sources { pub fn size_in_bytes(&self) -> usize { self.sources.lock().unwrap().size_in_bytes() } - - // pub fn get(&self, source_id: SourceId) -> MutexGuard<'_, InteriorSources> { - // let guard = self.sources.lock().unwrap(); - - // guard.get(source_id) - // } - - // pub fn get(&'_ self, source_id: SourceId) -> Option> { - // Ref::filter_map(self.sources.borrow(), |x| x.get(source_id)).ok() - // } - - // pub fn get_path(&self, source_id: &SourceId) -> Option> { - // Ref::filter_map(self.sources.borrow(), |x| x.paths.get(source_id)).ok() - // } - - // pub fn get_path(&self, source_id: &SourceId) -> Option<&PathBuf> { - // self.sources.lock().unwrap().paths.get(source_id) - // } -} - -pub(crate) static SYNTAX_OBJECT_ID: AtomicUsize = AtomicUsize::new(0); - -#[derive( - Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, -)] -pub struct SyntaxObjectId(pub usize); - -impl SyntaxObjectId { - #[inline] - pub fn fresh() -> Self { - SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)) - } -} - -impl From for usize { - fn from(value: SyntaxObjectId) -> Self { - value.0 - } -} - -#[derive( - Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, -)] -pub struct ListId(usize); - -#[derive( - Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, -)] -pub struct FunctionId(usize); - -impl std::fmt::Display for SyntaxObjectId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self:?}") - } -} - -/// A syntax object that can hold anything as the syntax -/// In this case, we're using the token type emitted by logos -/// -/// This should open the door to interning our strings to make -/// parsing (and optimizations later) faster -#[derive(Serialize, Deserialize)] -pub struct RawSyntaxObject { - pub(crate) ty: T, - pub(crate) span: Span, - // TODO: Just nuke this source here, we don't need it. - // pub(crate) source: Option>, - // pub(crate) metadata: Option, - pub(crate) syntax_object_id: SyntaxObjectId, -} - -impl Clone for RawSyntaxObject { - fn clone(&self) -> Self { - Self { - ty: self.ty.clone(), - span: self.span, - // source: self.source.clone(), - // metadata: self.metadata.clone(), - syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), - } - } -} - -/// Denotes what kind of identifier we actually have -#[derive(Clone, Serialize, Deserialize)] -pub enum IdentifierType { - Free, - Global, - Local, - Macro, - Module, -} - -#[derive(Clone, Serialize, Deserialize, Hash, Eq, PartialOrd, Ord, PartialEq, Default)] -pub struct IdentifierMetadata { - // kind: IdentifierType, - // built_in: bool, - pub(crate) depth: usize, -} - -impl std::fmt::Debug for RawSyntaxObject { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("RawSyntaxObject") - .field("ty", &self.ty) - .field("span", &self.span) - // .field("source", &self.source) - .finish() - } -} - -// Implementing hash here just on the token type - we dont want the span included -// For determining the hash here -impl std::hash::Hash for RawSyntaxObject { - fn hash(&self, state: &mut H) { - self.ty.hash(state); - self.span.hash(state); - } -} - -pub type SyntaxObject = RawSyntaxObject>; - -// #[derive(Debug, Clone, Serialize, Deserialize)] -// pub struct SyntaxObject { -// pub(crate) ty: TokenType, -// pub(crate) span: Span, -// pub(crate) source: Option>, -// } - -impl PartialEq for SyntaxObject { - fn eq(&self, other: &Self) -> bool { - self.ty == other.ty - } -} - -impl SyntaxObject { - pub fn new(ty: TokenType, span: Span) -> Self { - SyntaxObject { - ty, - span, - // source: None, - syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), - } - } - - pub fn default(ty: TokenType) -> Self { - SyntaxObject { - ty, - span: Span::new(0, 0, None), - // source: None, - syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), - } - } - - pub fn set_span(&mut self, span: Span) { - self.span = span - } - - pub fn from_token_with_source( - val: &Token<'_, InternedString>, - _source: &Option>, - ) -> Self { - SyntaxObject { - ty: val.ty.clone(), - span: val.span, - // source: source.as_ref().map(Rc::clone), - syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), - } - } -} - -impl From<&Token<'_, InternedString>> for SyntaxObject { - fn from(val: &Token<'_, InternedString>) -> SyntaxObject { - SyntaxObject::new(val.ty.clone(), val.span) - } } -// impl From<&Token<'_, String>> for SyntaxObject { -// fn from(val: &Token<'_, String>) -> SyntaxObject { -// SyntaxObject::new(val.ty.clone(), val.span) -// } -// } - impl TryFrom for SteelVal { type Error = SteelErr; @@ -378,2411 +172,3 @@ impl TryFrom for SteelVal { } } } - -#[derive(Clone, Debug, PartialEq)] -pub enum ParseError { - Unexpected(TokenType, Option>), - UnexpectedEOF(Option>), - UnexpectedChar(char, Span, Option>), - IncompleteString(String, Span, Option>), - SyntaxError(String, Span, Option>), - ArityMismatch(String, Span, Option>), -} - -impl std::fmt::Display for ParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ParseError::Unexpected(l, _) => write!(f, "Parse: Unexpected token: {:?}", l), - ParseError::UnexpectedEOF(_) => write!(f, "Parse: Unexpected EOF"), - ParseError::UnexpectedChar(l, _, _) => { - write!(f, "Parse: Unexpected character: {:?}", l) - } - ParseError::IncompleteString(l, _, _) => write!(f, "Parse: Incomplete String: {}", l), - ParseError::SyntaxError(l, _, _) => write!(f, "Parse: Syntax Error: {}", l), - ParseError::ArityMismatch(l, _, _) => write!(f, "Parse: Arity mismatch: {}", l), - } - } -} - -impl std::error::Error for ParseError {} - -impl ParseError { - pub fn span(&self) -> Option { - match self { - // ParseError::TokenError(_) => None, - ParseError::Unexpected(_, _) => None, - ParseError::UnexpectedEOF(_) => None, - ParseError::UnexpectedChar(_, s, _) => Some(*s), - ParseError::IncompleteString(_, s, _) => Some(*s), - ParseError::SyntaxError(_, s, _) => Some(*s), - ParseError::ArityMismatch(_, s, _) => Some(*s), - } - } - - pub fn set_source(self, source: Option>) -> Self { - use ParseError::*; - match self { - ParseError::Unexpected(l, _) => Unexpected(l, source), - ParseError::UnexpectedEOF(_) => UnexpectedEOF(source), - ParseError::UnexpectedChar(l, s, _) => UnexpectedChar(l, s, source), - ParseError::IncompleteString(l, s, _) => IncompleteString(l, s, source), - ParseError::SyntaxError(l, s, _) => SyntaxError(l, s, source), - ParseError::ArityMismatch(l, s, _) => ArityMismatch(l, s, source), - } - } -} - -pub struct InternString; - -impl ToOwnedString for InternString { - fn own(&self, s: &str) -> InternedString { - s.into() - } -} - -// #[derive(Debug)] -pub struct Parser<'a> { - tokenizer: OwnedTokenStream<'a, InternedString, InternString>, - quote_stack: Vec, - quasiquote_depth: isize, - quote_context: bool, - shorthand_quote_stack: Vec, - source_name: Option>, - context: Vec, - comment_buffer: Vec<&'a str>, - collecting_comments: bool, - keep_lists: bool, -} - -#[derive(Debug, Copy, Clone, PartialEq)] -enum ParsingContext { - // Inside of a quote. Expressions should be parsed without being coerced into a typed variant of the AST - Quote(usize), - // Shortened version of a quote - QuoteTick(usize), - // Inside of an unquote - expressions should actually be parsed as usual - Unquote(usize), - // Shortened version of unquote - UnquoteTick(usize), - // Treat this like a normal quote - Quasiquote(usize), - // Shortened version of quasiquote - QuasiquoteTick(usize), - // expressions should parsed as normal - UnquoteSplicing(usize), - // Shorted version of Unquote Splicing - UnquoteSplicingTick(usize), -} - -impl<'a> Parser<'a> { - // #[cfg(test)] - pub fn parse(expr: &str) -> Result> { - Parser::new(expr, None).collect() - } - - pub fn parse_without_lowering(expr: &str) -> Result> { - Parser::new(expr, None).without_lowering().collect() - } - - pub fn offset(&self) -> usize { - self.tokenizer.offset() - } -} - -pub type Result = result::Result; - -fn tokentype_error_to_parse_error(t: &Token<'_, InternedString>) -> ParseError { - if let TokenType::Error = t.ty { - // println!("Found an error: {}", t); - - if t.source.starts_with('\"') { - ParseError::IncompleteString(t.source.to_string(), t.span, None) - } else { - ParseError::UnexpectedChar(t.source.chars().next().unwrap(), t.span, None) - } - } else { - ParseError::UnexpectedEOF(None) - } -} - -impl<'a> Parser<'a> { - pub fn new(input: &'a str, source_id: Option) -> Self { - Parser { - tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), - quote_stack: Vec::new(), - quasiquote_depth: 0, - quote_context: false, - shorthand_quote_stack: Vec::new(), - source_name: None, - context: Vec::new(), - comment_buffer: Vec::new(), - collecting_comments: false, - keep_lists: false, - } - } - - pub fn without_lowering(mut self) -> Self { - self.keep_lists = true; - self - } - - pub fn new_flat(input: &'a str, source_id: Option) -> Self { - Parser { - tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), - quote_stack: Vec::new(), - quasiquote_depth: 0, - quote_context: false, - shorthand_quote_stack: Vec::new(), - source_name: None, - context: Vec::new(), - comment_buffer: Vec::new(), - collecting_comments: false, - keep_lists: true, - } - } - - pub fn new_from_source( - input: &'a str, - source_name: PathBuf, - source_id: Option, - ) -> Self { - Parser { - tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), - quote_stack: Vec::new(), - quasiquote_depth: 0, - quote_context: false, - shorthand_quote_stack: Vec::new(), - source_name: Some(Rc::from(source_name)), - context: Vec::new(), - comment_buffer: Vec::new(), - collecting_comments: false, - keep_lists: false, - } - } - - // Attach comments! - pub fn doc_comment_parser(input: &'a str, source_id: Option) -> Self { - Parser { - tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), - quote_stack: Vec::new(), - quasiquote_depth: 0, - quote_context: false, - shorthand_quote_stack: Vec::new(), - source_name: None, - context: Vec::new(), - comment_buffer: Vec::new(), - collecting_comments: false, - keep_lists: false, - } - } - - // TODO this is definitely wrong - fn construct_quote(&mut self, val: ExprKind, span: Span) -> ExprKind { - // let q = { - // let rc_val = TokenType::Quote; - // ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - // // let val = ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))); - // // // self.intern.insert("quote".to_string(), rc_val); - // // val - // }; - - // ExprKind::List(List::new(vec![q, val])) - - ExprKind::Quote(Box::new(ast::Quote::new( - val, - SyntaxObject::new(TokenType::Quote, span), - ))) - } - - fn _expand_reader_macro( - &mut self, - token: TokenType, - val: ExprKind, - span: Span, - ) -> ExprKind { - let q = ExprKind::Atom(Atom::new(SyntaxObject::new(token, span))); - - ExprKind::List(List::new(vec![q, val])) - } - - fn construct_quote_vec(&mut self, val: ExprKind, span: Span) -> Vec { - // println!("Inside construct quote vec with: {:?}", val); - - let q = { - let rc_val = TokenType::Quote; - ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - // let val = ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))); - // // self.intern.insert("quote".to_string(), rc_val); - // val - }; - - vec![q, val] - } - - // Reader macro for ` - fn construct_quasiquote(&mut self, val: ExprKind, span: Span) -> ExprKind { - let q = { - let rc_val = TokenType::Identifier(*QUASIQUOTE); - ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - }; - - ExprKind::List(List::new(vec![q, val])) - } - - // Reader macro for , - fn construct_unquote(&mut self, val: ExprKind, span: Span) -> ExprKind { - let q = { - let rc_val = TokenType::Identifier(*UNQUOTE); - ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - }; - - ExprKind::List(List::new(vec![q, val])) - } - - fn construct_raw_unquote(&mut self, val: ExprKind, span: Span) -> ExprKind { - let q = { - // let rc_val = TokenType::Identifier(*UNQUOTE); - let rc_val = TokenType::Identifier(*RAW_UNQUOTE); - ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - }; - - ExprKind::List(List::new(vec![q, val])) - } - // Reader macro for ,@ - fn construct_unquote_splicing(&mut self, val: ExprKind, span: Span) -> ExprKind { - let q = { - let rc_val = TokenType::Identifier(*UNQUOTE_SPLICING); - ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - }; - - ExprKind::List(List::new(vec![q, val])) - } - - // Reader macro for ,@ - fn construct_raw_unquote_splicing(&mut self, val: ExprKind, span: Span) -> ExprKind { - let q = { - let rc_val = TokenType::Identifier(*RAW_UNQUOTE_SPLICING); - ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) - }; - - ExprKind::List(List::new(vec![q, val])) - } - - fn increment_quasiquote_context_if_not_in_quote_context(&mut self) { - // println!("INCREMENTING"); - if !self.quote_context { - self.quasiquote_depth += 1; - } - } - - fn decrement_quasiquote_context_if_not_in_quote_context(&mut self) { - // println!("DECREMENTING"); - if !self.quote_context { - self.quasiquote_depth -= 1; - } - } - - fn maybe_lower(&self, expr: Vec) -> Result { - if self.keep_lists { - Ok(ExprKind::List(List::new(expr))) - } else { - ExprKind::try_from(expr) - } - } - - fn read_from_tokens(&mut self) -> Result { - let mut stack: Vec> = Vec::new(); - let mut current_frame: Vec = Vec::new(); - - self.quote_stack = Vec::new(); - - // println!("READING FROM TOKENS"); - // self.quasiquote_depth = 0; - - loop { - match self.tokenizer.next() { - Some(token) => { - match token.ty { - TokenType::Comment => { - // println!("Found a comment!"); - // Internal comments, we're gonna skip for now - continue; - } - TokenType::Error => return Err(tokentype_error_to_parse_error(&token)), // TODO - TokenType::QuoteTick => { - // quote_count += 1; - // self.quote_stack.push(current_frame.len()); - self.shorthand_quote_stack.push(stack.len()); - - let last_context = self.quote_context; - - if self.quasiquote_depth == 0 { - self.quote_context = true; - } - - // println!("Entering context: Quote Tick in read from tokens"); - - self.context.push(ParsingContext::QuoteTick(stack.len())); - - let quote_inner = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| { - // if self.quasiquote_depth == 0 { - self.construct_quote(x, token.span) - // } else { - // self.construct_fake_quote(x, token.span) - // } - }); - // self.quote_stack.pop(); - self.shorthand_quote_stack.pop(); - - self.quote_context = last_context; - - // println!( - // "Exiting Context: {:?} in read from tokens", - // self.context.pop() - // ); - - // self.context.pop(); - - let popped_value = self.context.pop(); - - if let Some(popped) = popped_value { - // dbg!(&popped); - debug_assert!(matches!(popped, ParsingContext::QuoteTick(_))) - } - - current_frame.push(quote_inner?); - } - TokenType::Unquote => { - // println!("Entering context: Unquote"); - - // This could underflow and panic - if its negative then we have a problem. Maybe just use an isize and let it underflow? - self.decrement_quasiquote_context_if_not_in_quote_context(); - - self.context.push(ParsingContext::UnquoteTick(stack.len())); - - let quote_inner = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| { - // dbg!(self.quasiquote_depth); - // dbg!(self.quote_context); - if self.quasiquote_depth == 0 && !self.quote_context { - self.construct_raw_unquote(x, token.span) - } else { - self.construct_unquote(x, token.span) - } - }); - - let popped_value = self.context.pop(); - - self.increment_quasiquote_context_if_not_in_quote_context(); - - if let Some(popped) = popped_value { - debug_assert!(matches!(popped, ParsingContext::UnquoteTick(_))) - } - // println!("Exiting Context: {:?}", self.context.pop()); - current_frame.push(quote_inner?); - } - TokenType::QuasiQuote => { - // println!("Entering context: Quasiquote"); - - self.increment_quasiquote_context_if_not_in_quote_context(); - - self.context - .push(ParsingContext::QuasiquoteTick(stack.len())); - - let quote_inner = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| self.construct_quasiquote(x, token.span)); - - // self.context.pop(); - // println!( - // ">>>>>>>>>>>>>>>>>>> Exiting Context: {:?}", - // self.context.pop() - // ); - - let popped_value = self.context.pop(); - - self.decrement_quasiquote_context_if_not_in_quote_context(); - - if let Some(popped) = popped_value { - // println!("Popped: {:?}", popped); - debug_assert!(matches!(popped, ParsingContext::QuasiquoteTick(_))) - } - - current_frame.push(quote_inner?); - } - TokenType::UnquoteSplice => { - // println!("Entering context: UnquoteSplicing"); - - self.decrement_quasiquote_context_if_not_in_quote_context(); - - self.context - .push(ParsingContext::UnquoteSplicingTick(stack.len())); - - let quote_inner = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| { - if self.quasiquote_depth == 0 && !self.quote_context { - self.construct_raw_unquote_splicing(x, token.span) - } else { - self.construct_unquote_splicing(x, token.span) - } - }); - - // self.context.pop(); - - let popped_value = self.context.pop(); - - self.increment_quasiquote_context_if_not_in_quote_context(); - - if let Some(popped) = popped_value { - debug_assert!(matches!( - popped, - ParsingContext::UnquoteSplicingTick(_) - )) - } - - // println!("Exiting Context: {:?}", self.context.pop()); - current_frame.push(quote_inner?); - } - TokenType::OpenParen => { - stack.push(current_frame); - current_frame = Vec::new(); - } - TokenType::CloseParen => { - // This is the match that we'll want to move inside the below stack.pop() match statement - // As we close the current context, we check what our current state is - - - if let Some(mut prev_frame) = stack.pop() { - match prev_frame.first_mut().and_then(|x| x.atom_identifier_mut()) { - Some(ident) if *ident == *UNQUOTE => { - // self.increment_quasiquote_context_if_not_in_quote_context(); - if self.quasiquote_depth == 0 && !self.quote_context { - *ident = *RAW_UNQUOTE; - } - self.increment_quasiquote_context_if_not_in_quote_context(); - - // println!("Exiting unquote"); - } - Some(ident) if *ident == *QUASIQUOTE => { - self.decrement_quasiquote_context_if_not_in_quote_context(); - - // println!("Exiting quasiquote"); - } - Some(ident) if *ident == *UNQUOTE_SPLICING => { - // self.increment_quasiquote_context_if_not_in_quote_context(); - - if self.quasiquote_depth == 0 && !self.quote_context { - *ident = *RAW_UNQUOTE_SPLICING; - } - self.increment_quasiquote_context_if_not_in_quote_context(); - - // println!("Exiting unquote"); - } - _ => {} - } - - match self.context.last().cloned() { - // TODO: Change this -> This should really be just Some(ParsingContext::Quote) - // If we have _anything_ then we should check if we need to parse it differently. If we're at the last_quote_index, - // then we can pop it off inside there. - Some(ParsingContext::Quote(last_quote_index)) - | Some(ParsingContext::Quasiquote(last_quote_index)) => { - if stack.len() <= last_quote_index { - self.context.pop(); - } - - match current_frame.first() { - Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Quote, - .. - }, - })) => match self.context.last() { - Some( - ParsingContext::Quasiquote(_) - | ParsingContext::QuasiquoteTick(_) - | ParsingContext::Quote(_) - | ParsingContext::QuoteTick(_), - ) => prev_frame - .push(ExprKind::List(List::new(current_frame))), - _ => { - prev_frame.push( - self.maybe_lower(current_frame).map_err( - |x| { - x.set_source( - self.source_name.clone(), - ) - }, - )?, - ); - } - }, - _ => { - // println!("Converting to list"); - // println!("Context here: {:?}", self.context); - prev_frame - .push(ExprKind::List(List::new(current_frame))) - } - } - } - - Some(ParsingContext::QuoteTick(_)) - | Some(ParsingContext::QuasiquoteTick(_)) => { - match current_frame.first() { - Some(ExprKind::Atom(Atom { - syn: - SyntaxObject { - ty: TokenType::Quote, - .. - }, - })) => { - // println!("Converting to quote inside quote tick"); - prev_frame.push( - self.maybe_lower(current_frame).map_err( - |x| x.set_source(self.source_name.clone()), - )?, - ); - } - _ => { - // if let Some(ParsingContext::QuasiquoteTick(_)) = - // self.context.last() - // { - // self.decrement_quasiquote_context_if_not_in_quote_context(); - // } - - // println!("Converting to list inside quote tick"); - prev_frame - .push(ExprKind::List(List::new(current_frame))) - } - } - } - - // If we're in the short hand reader world, just ignore popping off the stack - // but still treat it as a normal expression - Some(ParsingContext::UnquoteTick(_)) - | Some(ParsingContext::UnquoteSplicingTick(_)) => { - // self.quasiquote_depth += 1; - - // self.increment_quasiquote_context_if_not_in_quote_context(); - - // println!( - // "UQ/UQS: Stack length: {:?}, last_quote_index: {:?}", - // stack.len(), - // last_quote_index - // ); - - // if stack.len() <= *last_quote_index { - // // println!("Exiting Context: {:?}", self.context.pop()); - // self.context.pop(); - // } - - prev_frame.push( - self.maybe_lower(current_frame).map_err(|x| { - x.set_source(self.source_name.clone()) - })?, - ); - } - - Some(ParsingContext::Unquote(last_quote_index)) - | Some(ParsingContext::UnquoteSplicing(last_quote_index)) => { - // self.quasiquote_depth += 1; - - // self.increment_quasiquote_context_if_not_in_quote_context(); - - // println!( - // "UQ/UQS: Stack length: {:?}, last_quote_index: {:?}", - // stack.len(), - // last_quote_index - // ); - - if stack.len() <= last_quote_index { - // println!("{} - {}", stack.len(), last_quote_index); - // println!("Exiting Context: {:?}", self.context.pop()); - self.context.pop(); - } - - prev_frame.push( - self.maybe_lower(current_frame).map_err(|x| { - x.set_source(self.source_name.clone()) - })?, - ); - } - - // Else case, just go ahead and assume it is a normal frame - _ => prev_frame.push( - self.maybe_lower(current_frame) - .map_err(|x| x.set_source(self.source_name.clone()))?, - ), - } - - // Reinitialize current frame here - current_frame = prev_frame; - } else { - // println!("Else case: {:?}", current_frame); - // println!("Context: {:?}", self.context); - - // dbg!(&self.quote_stack); - // dbg!(&self.context); - // dbg!(&self.shorthand_quote_stack); - match self.context.last() { - Some(ParsingContext::QuoteTick(_)) - | Some(ParsingContext::QuasiquoteTick(_)) => { - // | Some(ParsingContext::Quote(d)) && d > 0 => { - return Ok(ExprKind::List(List::new(current_frame))); - } - Some(ParsingContext::Quote(x)) if *x > 0 => { - self.context.pop(); - - return Ok(ExprKind::List(List::new(current_frame))); - } - Some(ParsingContext::Quote(0)) => { - self.context.pop(); - - return self - .maybe_lower(current_frame) - .map_err(|x| x.set_source(self.source_name.clone())); - } - _ => { - // dbg!(self.quasiquote_depth); - // println!("=> {}", List::new(current_frame.clone())); - // println!("----------------------------------------"); - - if self.quasiquote_depth > 0 { - // TODO/HACK - @Matt - // If we're in a define syntax situation, go ahead and just return a normal one - if current_frame - .first() - .map(|x| x.define_syntax_ident()) - .unwrap_or_default() - { - return self.maybe_lower(current_frame).map_err( - |x| x.set_source(self.source_name.clone()), - ); - } - - // println!("Should still be quoted here"); - - return Ok(ExprKind::List(List::new(current_frame))); - } - - return self - .maybe_lower(current_frame) - .map_err(|x| x.set_source(self.source_name.clone())); - } - } - } - } - - _ => { - if let TokenType::Quote = &token.ty { - // self.quote_stack.push(current_frame.len()); - self.quote_stack.push(stack.len()); - } - - // dbg!(&self.context); - - // Mark what context we're inside with the context stack: - // This only works when its the first argument - check the function call in open paren? - if current_frame.is_empty() { - match &token.ty { - TokenType::Quote => { - if self.context == &[ParsingContext::QuoteTick(0)] { - self.context.push(ParsingContext::Quote(1)) - } else { - self.context.push(ParsingContext::Quote(stack.len())) - } - - // self.context.push(ParsingContext::Quote(stack.len())) - } - TokenType::Identifier(ident) if *ident == *UNQUOTE => { - // println!("Entering unquote"); - - self.context.push(ParsingContext::Unquote(stack.len())); - self.decrement_quasiquote_context_if_not_in_quote_context(); - } - TokenType::Identifier(ident) if *ident == *QUASIQUOTE => { - // println!("Entering quasiquote"); - - self.context.push(ParsingContext::Quasiquote(stack.len())); - self.increment_quasiquote_context_if_not_in_quote_context(); - } - TokenType::Identifier(ident) if *ident == *UNQUOTE_SPLICING => { - self.context - .push(ParsingContext::UnquoteSplicing(stack.len())); - self.decrement_quasiquote_context_if_not_in_quote_context(); - } - _ => {} - } - - // println!("Context on application: {:?}", self.context); - } - - // println!("{}", token); - - current_frame.push(ExprKind::Atom(Atom::new( - SyntaxObject::from_token_with_source( - &token, - &self.source_name.clone(), - ), - ))) - } - } - } - - None => return Err(ParseError::UnexpectedEOF(self.source_name.clone())), - } - } - } -} - -fn wrap_in_doc_function(expr: ExprKind, comment: String) -> ExprKind { - // println!("Found comment : {} for expr {}", comment, expr); - - ExprKind::List(List::new(vec![ - ExprKind::ident("@doc"), - ExprKind::string_lit(comment), - expr, - ])) -} - -impl<'a> Parser<'a> { - fn get_next_and_maybe_wrap_in_doc(&mut self) -> Option> { - let mut next; - - loop { - next = self.tokenizer.next(); - - if let Some(res) = next { - match res.ty { - TokenType::Comment => { - if self.comment_buffer.is_empty() && !self.collecting_comments { - if res.source().trim_start_matches(';').starts_with("@doc") { - self.collecting_comments = true; - - // println!("Found a comment to collect, starting collection..."); - - continue; - } - } - - if self.collecting_comments { - let doc_line = res.source().trim_start_matches(';'); - - // If we hit another comment, clear it - if doc_line.starts_with("@doc") { - // println!("Clearing buffer"); - - self.comment_buffer.clear(); - continue; - } - - // println!("Collecting line: {}", doc_line); - - self.comment_buffer.push(doc_line.trim_start()); - } - - continue; - } - - TokenType::QuoteTick => { - // See if this does the job - self.shorthand_quote_stack.push(0); - - let last = self.quote_context; - - if self.quasiquote_depth == 0 { - self.quote_context = true; - } - - // self.quote_context = true; - - // println!("Entering Context: Quote Tick"); - self.context.push(ParsingContext::QuoteTick(0)); - - let value = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| self.construct_quote_vec(x, res.span)); - - self.shorthand_quote_stack.pop(); - - let popped_value = self.context.pop(); - - if let Some(popped) = popped_value { - // dbg!(&popped); - debug_assert!(matches!(popped, ParsingContext::QuoteTick(_))) - } - - self.quote_context = last; - - // println!("Exiting context: {:?}", self.context.pop()); - // println!("Result: {:?}", value); - - // println!("{}", List::new(value.clone().unwrap())); - - return Some(match value { - Ok(v) => { - // Ok(ExprKind::List(List::new(v))) - - self.maybe_lower(v) - } - Err(e) => Err(e), - }); - } - - TokenType::Unquote => { - // println!("Entering Context: Unquote"); - self.context.push(ParsingContext::UnquoteTick(0)); - - self.decrement_quasiquote_context_if_not_in_quote_context(); - - let value = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| { - // dbg!(&self.quasiquote_depth); - if self.quasiquote_depth == 0 && !self.quote_context { - self.construct_raw_unquote(x, res.span) - } else { - self.construct_unquote(x, res.span) - } - }); - - let popped_value = self.context.pop(); - - self.increment_quasiquote_context_if_not_in_quote_context(); - - if let Some(popped) = popped_value { - debug_assert!(matches!(popped, ParsingContext::UnquoteTick(_))) - } - // println!("Exiting context: {:?}", self.context.pop()); - - return Some(value); - } - - TokenType::UnquoteSplice => { - // println!("Entering Context: Unquotesplicing"); - self.context.push(ParsingContext::UnquoteSplicingTick(0)); - - self.decrement_quasiquote_context_if_not_in_quote_context(); - - let value = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| { - if self.quasiquote_depth == 0 && !self.quote_context { - self.construct_raw_unquote_splicing(x, res.span) - } else { - self.construct_unquote_splicing(x, res.span) - } - }); - - let popped_value = self.context.pop(); - - self.increment_quasiquote_context_if_not_in_quote_context(); - - if let Some(popped) = popped_value { - debug_assert!(matches!(popped, ParsingContext::UnquoteSplicingTick(_))) - } - - // println!("Exiting context: {:?}", self.context.pop()); - - return Some(value); - } - - TokenType::QuasiQuote => { - // println!("Entering Context: Quasiquote - top level"); - self.context.push(ParsingContext::QuasiquoteTick(0)); - - self.increment_quasiquote_context_if_not_in_quote_context(); - - let value = self - .next() - .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - .map(|x| self.construct_quasiquote(x, res.span)); - - // println!("{:?}", self.context.pop()); - - // println!("Top level Context: {:?}", self.context); - - let popped_value = self.context.pop(); - - if let Some(popped) = popped_value { - debug_assert!(matches!(popped, ParsingContext::QuasiquoteTick(_))) - } - - self.decrement_quasiquote_context_if_not_in_quote_context(); - - // println!("Exiting context: {:?}", self.context.pop()); - - return Some(value); - } - - TokenType::OpenParen => return Some(self.read_from_tokens()), - TokenType::CloseParen => { - return Some(Err(ParseError::Unexpected( - TokenType::CloseParen, - self.source_name.clone(), - ))) - } - TokenType::Error => return Some(Err(tokentype_error_to_parse_error(&res))), - _ => return Some(Ok(ExprKind::Atom(Atom::new(SyntaxObject::from(&res))))), - }; - } else { - // We're done consuming input - return None; - } - } - } -} - -impl<'a> Iterator for Parser<'a> { - type Item = Result; - - // TODO -> put the - fn next(&mut self) -> Option { - // self.shorthand_quote_stack = Vec::new(); - // self.quote_stack = Vec::new(); - - if self.quote_stack.is_empty() - && self.shorthand_quote_stack.is_empty() - && self.context.is_empty() - { - // println!("RESETTING QUASIQUOTE DEPTH"); - self.quasiquote_depth = 0; - self.comment_buffer.clear(); - } - - // println!("Clearing the comment buffer!"); - - self.get_next_and_maybe_wrap_in_doc().map(|res| { - if self.comment_buffer.is_empty() { - res - } else { - // Reset the comment collection until next @doc statement - self.collecting_comments = false; - - // println!("Setting collecting comments to false, found: {:?}", res); - - res.map(|x| { - wrap_in_doc_function(x, self.comment_buffer.join("\n").drain(..).collect()) - }) - } - }) - } - - // self.tokenizer.next().map(|res| match res.ty { - // // Collect the comment until theres something to attach to - // TokenType::Comment => { - // // // todo!() - - // // println!("Found a comment: {}", res.source()); - // // println!("Buffer now: {:?}", self.comment_buffer); - - // self.comment_buffer - // .push(res.source().trim_start_matches(';').trim_start()); - - // // TODO: Instead of making a recursive call, we should just wrap - // // this whole thing in a loop, and accumulate until we return - // match self.next() { - // Some(v) => { - // println!("Next thing found: {:?}", v); - // v - // // v.map(|x| { - // // wrap_in_doc_function(x, self.comment_buffer.join("\n").to_string()) - // // }) - // } - // None => Err(ParseError::SyntaxError( - // "Doc comment not associated with a top level definition".to_string(), - // res.span(), - // None, - // )), - // } - - // // Ok(self.next().unwrap) - - // // println!("Found a comment!"); - // } - - // // Err(e) => Err(ParseError::TokenError(e)), - // TokenType::QuoteTick => { - // // See if this does the job - // self.shorthand_quote_stack.push(0); - - // // println!("Entering Context: Quote Tick"); - // self.context.push(ParsingContext::QuoteTick(0)); - - // let value = self - // .next() - // .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - // .map(|x| self.construct_quote_vec(x, res.span)); - - // self.shorthand_quote_stack.pop(); - - // let popped_value = self.context.pop(); - - // if let Some(popped) = popped_value { - // debug_assert!(matches!(popped, ParsingContext::QuoteTick(_))) - // } - - // // println!("Exiting context: {:?}", self.context.pop()); - // // println!("Result: {:?}", value); - - // match value { - // Ok(v) => ExprKind::try_from(v), - // Err(e) => Err(e), - // } - // } - // TokenType::Unquote => { - // // println!("Entering Context: Unquote"); - // self.context.push(ParsingContext::UnquoteTick(0)); - - // let value = self - // .next() - // .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - // .map(|x| self.construct_unquote(x, res.span)); - - // let popped_value = self.context.pop(); - - // if let Some(popped) = popped_value { - // debug_assert!(matches!(popped, ParsingContext::UnquoteTick(_))) - // } - // // println!("Exiting context: {:?}", self.context.pop()); - - // value - // } - // TokenType::UnquoteSplice => { - // // println!("Entering Context: Unquotesplicing"); - // self.context.push(ParsingContext::UnquoteSplicingTick(0)); - - // let value = self - // .next() - // .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - // .map(|x| self.construct_unquote_splicing(x, res.span)); - - // let popped_value = self.context.pop(); - - // if let Some(popped) = popped_value { - // debug_assert!(matches!(popped, ParsingContext::UnquoteSplicingTick(_))) - // } - - // // println!("Exiting context: {:?}", self.context.pop()); - - // value - // } - // TokenType::QuasiQuote => { - // // println!("Entering Context: Quasiquote"); - // self.context.push(ParsingContext::QuasiquoteTick(0)); - - // let value = self - // .next() - // .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) - // .map(|x| self.construct_quasiquote(x, res.span)); - - // // println!("{:?}", self.context.pop()); - - // // println!("Top level Context: {:?}", self.context); - - // let popped_value = self.context.pop(); - - // if let Some(popped) = popped_value { - // debug_assert!(matches!(popped, ParsingContext::QuasiquoteTick(_))) - // } - - // // println!("Exiting context: {:?}", self.context.pop()); - - // value - // } - // TokenType::OpenParen => self.read_from_tokens(), - // TokenType::CloseParen => Err(ParseError::Unexpected( - // TokenType::CloseParen, - // self.source_name.clone(), - // )), - // TokenType::Error => Err(tokentype_error_to_parse_error(&res)), - // _ => Ok(ExprKind::Atom(Atom::new(SyntaxObject::from(&res)))), - // }) -} - -// Lower the syntax rules down from the list representation -pub(crate) fn lower_syntax_rules(expr: ExprKind) -> Result { - let mut value_iter = expr.into_list().into_iter(); - let syn = value_iter - .next() - .unwrap() - .into_atom_syntax_object() - .unwrap(); - - let syntax_vec = if let Some(ExprKind::List(l)) = value_iter.next() { - l.args - } else { - return Err(ParseError::SyntaxError( - "syntax-rules expects a list of new syntax forms used in the macro".to_string(), - syn.span, - None, - )); - }; - - let mut pairs = Vec::new(); - let rest: Vec<_> = value_iter.collect(); - - for pair in rest { - if let ExprKind::List(l) = pair { - if l.args.len() != 2 { - return Err(ParseError::SyntaxError( - "syntax-rules requires only one pattern to one body".to_string(), - syn.span, - None, - )); - } - - let mut pair_iter = l.args.into_iter(); - let pair_object = - PatternPair::new(pair_iter.next().unwrap(), pair_iter.next().unwrap()); - pairs.push(pair_object); - } else { - return Err(ParseError::SyntaxError( - "syntax-rules requires pattern to expressions to be in a list".to_string(), - syn.span, - None, - )); - } - } - - Ok(SyntaxRules::new(syntax_vec, pairs, syn)) -} - -// Lower define-syntax down from the list representation -pub(crate) fn lower_macro_and_require_definitions(expr: ExprKind) -> Result { - let as_list = expr.list(); - - // If this qualifies as - if as_list.map(List::is_define_syntax).unwrap_or_default() - && as_list - .unwrap() - .get(2) - .and_then(ExprKind::list) - .map(List::is_syntax_rules) - .unwrap_or_default() - { - let mut value_iter = expr.into_list().into_iter(); - - let define_syntax = value_iter.next().unwrap(); - - let name = value_iter.next().unwrap(); - let syntax = lower_syntax_rules(value_iter.next().unwrap())?; - - return Ok(ExprKind::Macro(Macro::new( - name, - syntax, - define_syntax.into_atom_syntax_object().unwrap(), - ))); - } - - if as_list.map(List::is_require).unwrap_or_default() { - let mut raw = expr.into_list().args; - - let syn = raw.remove(0).into_atom_syntax_object().unwrap(); - - if raw.is_empty() { - return Err(ParseError::ArityMismatch( - "require expects at least one identifier or string".to_string(), - syn.span, - None, - )); - } - - return Ok(ExprKind::Require(ast::Require::new(raw, syn))); - } - - Ok(expr) -} - -struct ASTLowerPass { - quote_depth: usize, -} - -impl ASTLowerPass { - // TODO: Make this mutable references, otherwise we'll be re-boxing everything for now reason - fn lower(&mut self, expr: ExprKind) -> Result { - match expr { - ExprKind::List(mut value) => { - if value.is_quote() { - // println!("Found quote"); - self.quote_depth += 1; - } - - // Visit the children first, on the way back up, assign into the - // correct AST node - value.args = value - .args - .into_iter() - .map(|x| self.lower(x)) - .collect::>()?; - - if value.is_quote() { - self.quote_depth -= 1; - } - - if let Some(f) = value.first().cloned() { - match f { - ExprKind::Atom(a) if self.quote_depth == 0 && value.is_quote() => { - match &a.syn.ty { - TokenType::Quote => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "quote", - |expr, syn| ast::Quote::new(expr, syn).into(), - ), - _ => unreachable!(), - } - } - ExprKind::Atom(a) if self.quote_depth == 0 => { - match &a.syn.ty { - // Have this also match on the first argument being a TokenType::Identifier("if") - // Do the same for the rest of the arguments - TokenType::If => parse_if(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *IF => { - parse_if(value.into_iter(), a.syn.clone()) - } - - TokenType::Define => parse_define(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *DEFINE => { - parse_define(value.into_iter(), a.syn.clone()) - } - - TokenType::Let => parse_let(value.into_iter(), a.syn.clone()), - TokenType::Identifier(expr) if *expr == *LET => { - parse_let(value.into_iter(), a.syn.clone()) - } - - // TODO: Deprecate - TokenType::TestLet => { - parse_new_let(value.into_iter(), a.syn.clone()) - } - TokenType::Identifier(expr) if *expr == *PLAIN_LET => { - parse_new_let(value.into_iter(), a.syn.clone()) - } - - TokenType::Quote => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "quote", - |expr, syn| ast::Quote::new(expr, syn).into(), - ), - TokenType::Identifier(expr) if *expr == *QUOTE => { - parse_single_argument( - value.into_iter(), - a.syn.clone(), - "quote", - |expr, syn| ast::Quote::new(expr, syn).into(), - ) - } - - TokenType::Return => parse_single_argument( - value.into_iter(), - a.syn.clone(), - "return!", - |expr, syn| ast::Return::new(expr, syn).into(), - ), - TokenType::Identifier(expr) if *expr == *RETURN => { - parse_single_argument( - value.into_iter(), - a.syn.clone(), - "return!", - |expr, syn| ast::Return::new(expr, syn).into(), - ) - } - - TokenType::Require => parse_require(&a, value.args), - TokenType::Identifier(expr) if *expr == *REQUIRE => { - parse_require(&a, value.args) - } - - TokenType::Set => parse_set(&a, value.args), - TokenType::Identifier(expr) if *expr == *SET => { - parse_set(&a, value.args) - } - - TokenType::Begin => parse_begin(&a, value.args), - TokenType::Identifier(expr) if *expr == *BEGIN => { - parse_begin(&a, value.args) - } - - TokenType::Lambda => parse_lambda(&a, value.args), - TokenType::Identifier(expr) - if *expr == *LAMBDA - || *expr == *LAMBDA_FN - || *expr == *LAMBDA_SYMBOL => - { - parse_lambda(&a, value.args) - } - - _ => Ok(ExprKind::List(value)), - } - } - _ => Ok(ExprKind::List(value)), - } - } else { - Ok(ExprKind::List(List::new(vec![]))) - } - } - ExprKind::Atom(_) => Ok(expr), - ExprKind::If(iff) => Ok(ExprKind::If(Box::new(ast::If::new( - self.lower(iff.test_expr)?, - self.lower(iff.then_expr)?, - self.lower(iff.else_expr)?, - iff.location, - )))), - ExprKind::Let(l) => Ok(ExprKind::Let(Box::new(ast::Let::new( - l.bindings - .into_iter() - .map(|(a, b)| Ok((self.lower(a)?, self.lower(b)?))) - .collect::>()?, - self.lower(l.body_expr)?, - l.location, - )))), - ExprKind::Define(d) => Ok(ExprKind::Define(Box::new(ast::Define::new( - self.lower(d.name)?, - self.lower(d.body)?, - d.location, - )))), - ExprKind::LambdaFunction(f) => Ok(ExprKind::LambdaFunction(Box::new( - ast::LambdaFunction::new_maybe_rest( - f.args - .into_iter() - .map(|x| self.lower(x)) - .collect::>()?, - self.lower(f.body)?, - f.location, - f.rest, - ), - ))), - ExprKind::Begin(b) => Ok(ExprKind::Begin(ast::Begin::new( - b.exprs - .into_iter() - .map(|x| self.lower(x)) - .collect::>()?, - b.location, - ))), - ExprKind::Return(r) => Ok(ExprKind::Return(Box::new(ast::Return::new( - self.lower(r.expr)?, - r.location, - )))), - ExprKind::Quote(_) => Ok(expr), - ExprKind::Macro(_) => Ok(expr), - ExprKind::SyntaxRules(_) => Ok(expr), - ExprKind::Set(s) => Ok(ExprKind::Set(Box::new(ast::Set::new( - self.lower(s.variable)?, - self.lower(s.expr)?, - s.location, - )))), - ExprKind::Require(_) => Ok(expr), // _ => Ok(expr), - } - } -} - -// TODO: Lower the rest of the AST post expansion, such that -pub(crate) fn lower_entire_ast(expr: ExprKind) -> Result { - ASTLowerPass { quote_depth: 0 }.lower(expr) -} - -#[cfg(test)] -mod parser_tests { - // use super::TokenType::*; - use super::*; - use crate::parser::ast::ExprKind; - use crate::parser::ast::{Begin, Define, If, LambdaFunction, Quote, Return}; - - fn atom(ident: &str) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default(Identifier(ident.into())))) - } - - fn int(num: isize) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default(IntegerLiteral( - MaybeBigInt::Small(num), - )))) - } - - fn character(c: char) -> ExprKind { - ExprKind::Atom(Atom::new(SyntaxObject::default( - TokenType::CharacterLiteral(c), - ))) - } - - #[test] - fn check_quote_parsing() { - println!("{:?}", Parser::parse("'(a b 'c)")); - } - - fn parses(s: &str) { - let a: Result> = Parser::new(s, None).collect(); - a.unwrap(); - } - - fn assert_parse(s: &str, result: &[ExprKind]) { - let a: Result> = Parser::new(s, None).collect(); - let a = a.unwrap(); - assert_eq!(a.as_slice(), result); - } - - fn assert_parse_err(s: &str, err: ParseError) { - let a: Result> = Parser::new(s, None).collect(); - assert_eq!(a, Err(err)); - } - - fn assert_parse_is_err(s: &str) { - let a: Result> = Parser::new(s, None).collect(); - assert!(a.is_err()); - } - - #[test] - fn check_resulting_parsing() { - let expr = r#"`(a `(b ,(+ 1 2) ,(foo ,(+ 1 3) d) e) f)"#; - - let a: Result> = Parser::new(expr, None).collect(); - let a = a.unwrap(); - - println!("{}", a[0]); - } - - #[test] - fn check_double_unquote_parsing() { - let expr = r#"(let ([name1 'x] [name2 'y]) `(a `(b ,,name1 ,',name2 d) e))"#; - - let a: Result> = Parser::new(expr, None).collect(); - let a = a.unwrap(); - - println!("{}", a[0]); - } - - #[test] - fn check_parser_with_doc_comments() { - let expr = r#" - ;;@doc - ;; This is a fancy cool comment, that I want to attach to a top level definition! - ;; This is the second line of the comment, I want this attached as well! - ;; Macro for creating a new struct, in the form of: - ;; `(struct (fields ...) options ...)` - ;; The options can consist of the following: - ;; - ;; Single variable options (those which their presence indicates #true) - ;; - #:mutable - ;; - #:transparent - ;; - ;; Other options must be presented as key value pairs, and will get stored - ;; in the struct instance. They will also be bound to the variable - ;; ___-options___ in the same lexical environment where the - ;; struct was defined. For example: - ;; - ;; (Applesauce (a b c) #:mutable #:transparent #:unrecognized-option 1234) - ;; - ;; Will result in the value `___Applesauce-options___` like so: - ;; (hash #:mutable #true #:transparent #true #:unrecognized-option 1234) - ;; - ;; By default, structs are immutable, which means setter functions will not - ;; be generated. Also by default, structs are not transparent, which means - ;; printing them will result in an opaque struct that does not list the fields - (define foo 12345) - "#; - - let parser = Parser::doc_comment_parser(expr, None); - - let result: Result> = parser.collect(); - - println!("{:?}", result.unwrap()); - } - - #[test] - fn parses_make_struct() { - parses("(define make-struct (lambda (struct-name fields) (map (lambda (field) (list (quote define) (concat-symbols struct-name field) (quote (lambda (this) (vector-ref this 0))))) fields)))") - } - - #[test] - fn parses_quasiquote() { - parses(r#"(quasiquote ((unquote x) xs ...)) "#); - } - - #[test] - fn parse_syntax_rules() { - parses( - r#" - (syntax-rules (unquote unquote-splicing) - ((quasiquote ((unquote x) xs ...)) (cons x (quasiquote (xs ...)))) - ((quasiquote ((unquote-splicing x))) (append (list x) '())) - ((quasiquote ((unquote-splicing x) xs ...)) (append x (quasiquote (xs ...)))) - ((quasiquote (unquote x)) x) - ((quasiquote (x)) '(x)) - ((quasiquote (x xs ...)) (cons (quasiquote x) (quasiquote (xs ...)))) - ((quasiquote x) 'x)) - "#, - ); - } - - #[test] - fn parse_define_syntax() { - parses( - r#" - (define-syntax quasiquote - (syntax-rules (unquote unquote-splicing) - ((quasiquote ((unquote x) xs ...)) (cons x (quasiquote (xs ...)))) - ((quasiquote ((unquote-splicing x))) (append (list x) '())) - ((quasiquote ((unquote-splicing x) xs ...)) (append x (quasiquote (xs ...)))) - ((quasiquote (unquote x)) x) - ((quasiquote (x)) '(x)) - ((quasiquote (x xs ...)) (cons (quasiquote x) (quasiquote (xs ...)))) - ((quasiquote x) 'x))) - "#, - ); - } - - #[test] - fn parse_quote() { - // parses("(displayln (match (quote (lambda y z)) '(x y z)))") - parses("(displayln (match '(lambda y z) '(x y z)))") - } - - #[test] - fn parse_unicode() { - assert_parse("#\\¡", &[character('¡')]); - assert_parse("#\\u{b}", &[character('\u{b}')]); - } - - #[test] - fn parse_more_unicode() { - assert_parse("#\\u{a0}", &[character('\u{a0}')]); - } - - #[test] - fn parse_strange_characters() { - assert_parse("#\\^", &[character('^')]); - } - - #[test] - fn parse_character_sequence() { - assert_parse( - "#\\¡ #\\SPACE #\\g", - &[character('¡'), character(' '), character('g')], - ) - } - - #[test] - fn parse_character_sequence_inside_if() { - assert_parse( - "(if #\\¡ #\\SPACE #\\g)", - &[ExprKind::If(Box::new(If::new( - character('¡'), - character(' '), - character('g'), - SyntaxObject::default(TokenType::If), - )))], - ) - } - - #[test] - fn parse_close_paren_character() { - assert_parse("#\\)", &[character(')')]); - assert_parse("#\\]", &[character(']')]) - } - - #[test] - fn parse_open_paren_character() { - assert_parse("#\\(", &[character('(')]) - } - - #[test] - fn test_error() { - assert_parse_err("(", ParseError::UnexpectedEOF(None)); - assert_parse_err("(abc", ParseError::UnexpectedEOF(None)); - assert_parse_err("(ab 1 2", ParseError::UnexpectedEOF(None)); - assert_parse_err("((((ab 1 2) (", ParseError::UnexpectedEOF(None)); - assert_parse_err("())", ParseError::Unexpected(TokenType::CloseParen, None)); - assert_parse_err("() ((((", ParseError::UnexpectedEOF(None)); - assert_parse_err("')", ParseError::Unexpected(TokenType::CloseParen, None)); - assert_parse_err("(')", ParseError::Unexpected(TokenType::CloseParen, None)); - assert_parse_err("('", ParseError::UnexpectedEOF(None)); - } - - #[test] - fn quote_multiple_args_should_err() { - assert_parse_is_err("(quote a b c)"); - } - - #[test] - fn test_let_should_err() { - assert_parse_is_err("(let)"); - assert_parse_is_err("(let (a) 10)"); - } - - #[test] - fn test_if_should_err() { - assert_parse_is_err("(if)"); - assert_parse_is_err("(if 1)"); - // assert_parse_is_err("(if 1 2)"); - assert_parse_is_err("(if 1 2 3 4)"); - } - - #[test] - fn test_define_should_err() { - assert_parse_is_err("(define)"); - assert_parse_is_err("(define blagh)"); - assert_parse_is_err("(define test 1 2)"); - assert_parse_is_err("(define () test"); - } - - #[test] - fn test_lambda_should_err() { - assert_parse_is_err("(lambda)"); - assert_parse_is_err("(lambda (x))"); - } - - #[test] - fn test_empty() { - assert_parse("", &[]); - assert_parse("()", &[ExprKind::List(List::new(vec![]))]); - } - - #[test] - fn test_empty_quote_inside_if() { - assert_parse( - "(if #\\¡ (quote ()) #\\g)", - &[ExprKind::If(Box::new(If::new( - character('¡'), - ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - character('g'), - SyntaxObject::default(TokenType::If), - )))], - ) - } - - #[test] - fn test_empty_quote() { - assert_parse( - "'()", - &[ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - )], - ) - } - - #[test] - fn test_empty_quote_nested() { - assert_parse( - "(list '())", - &[ExprKind::List(List::new(vec![ - atom("list"), - ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ]))], - ) - } - - #[test] - fn test_multi_parse_simple() { - assert_parse("a b +", &[atom("a"), atom("b"), atom("+")]); - } - - #[test] - fn test_multi_parse_complicated() { - assert_parse( - "a b (funcall 1 (+ 2 3.5))", - &[ - atom("a"), - atom("b"), - ExprKind::List(List::new(vec![ - atom("funcall"), - int(1), - ExprKind::List(List::new(vec![ - atom("+"), - int(2), - ExprKind::Atom(Atom::new(SyntaxObject::default(NumberLiteral(3.5)))), - ])), - ])), - ], - ) - } - #[test] - fn test_parse_simple() { - assert_parse( - "(+ 1 2 3) (- 4 3)", - &[ - ExprKind::List(List::new(vec![atom("+"), int(1), int(2), int(3)])), - ExprKind::List(List::new(vec![atom("-"), int(4), int(3)])), - ], - ); - } - - #[test] - fn test_parse_nested() { - assert_parse( - "(+ 1 (foo (bar 2 3)))", - &[ExprKind::List(List::new(vec![ - atom("+"), - int(1), - ExprKind::List(List::new(vec![ - atom("foo"), - ExprKind::List(List::new(vec![atom("bar"), int(2), int(3)])), - ])), - ]))], - ); - assert_parse( - "(+ 1 (+ 2 3) (foo (bar 2 3)))", - &[ExprKind::List(List::new(vec![ - atom("+"), - int(1), - ExprKind::List(List::new(vec![atom("+"), int(2), int(3)])), - ExprKind::List(List::new(vec![ - atom("foo"), - ExprKind::List(List::new(vec![atom("bar"), int(2), int(3)])), - ])), - ]))], - ); - } - - #[test] - fn test_if() { - assert_parse( - "(+ 1 (if 2 3 4) (foo (+ (bar 1 1) 3) 5))", - &[ExprKind::List(List::new(vec![ - atom("+"), - int(1), - ExprKind::If(Box::new(If::new( - int(2), - int(3), - int(4), - SyntaxObject::default(If), - ))), - ExprKind::List(List::new(vec![ - atom("foo"), - ExprKind::List(List::new(vec![ - atom("+"), - ExprKind::List(List::new(vec![atom("bar"), int(1), int(1)])), - int(3), - ])), - int(5), - ])), - ]))], - ); - } - - #[test] - fn test_quote() { - assert_parse( - "(quote (if 1 2))", - &[ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - int(1), - int(2), - ])), - SyntaxObject::default(TokenType::Quote), - )))], - ) - } - - #[test] - fn test_quote_shorthand() { - assert_parse( - "'(if 1 2)", - &[ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - int(1), - int(2), - ])), - SyntaxObject::default(TokenType::Quote), - )))], - ) - } - - #[test] - fn test_quote_nested() { - assert_parse( - "(quote (if (if 1 2) 3))", - &[ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - int(1), - int(2), - ])), - int(3), - ])), - SyntaxObject::default(TokenType::Quote), - )))], - ) - } - - #[test] - fn test_quote_shorthand_nested() { - assert_parse( - "'(if (if 1 2) 3)", - &[ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - int(1), - int(2), - ])), - int(3), - ])), - SyntaxObject::default(TokenType::Quote), - )))], - ) - } - - #[test] - fn test_quote_shorthand_multiple_exprs() { - assert_parse( - "'(if (if 1 2) 3) (+ 1 (if 2 3 4) (foo (+ (bar 1 1) 3) 5))", - &[ - ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), - int(1), - int(2), - ])), - int(3), - ])), - SyntaxObject::default(TokenType::Quote), - ))), - ExprKind::List(List::new(vec![ - atom("+"), - int(1), - ExprKind::If(Box::new(If::new( - int(2), - int(3), - int(4), - SyntaxObject::default(If), - ))), - ExprKind::List(List::new(vec![ - atom("foo"), - ExprKind::List(List::new(vec![ - atom("+"), - ExprKind::List(List::new(vec![atom("bar"), int(1), int(1)])), - int(3), - ])), - int(5), - ])), - ])), - ], - ) - } - - #[test] - fn test_quote_inner() { - assert_parse( - "'(applesauce 'one)", - &[ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - atom("applesauce"), - ExprKind::Quote(Box::new(Quote::new( - atom("one"), - SyntaxObject::default(TokenType::Quote), - ))), - ])), - SyntaxObject::default(TokenType::Quote), - )))], - ) - } - - #[test] - fn test_quote_inner_without_shorthand() { - assert_parse( - "(quote (applesauce 'one))", - &[ExprKind::Quote(Box::new(Quote::new( - ExprKind::List(List::new(vec![ - atom("applesauce"), - ExprKind::Quote(Box::new(Quote::new( - atom("one"), - SyntaxObject::default(TokenType::Quote), - ))), - ])), - SyntaxObject::default(TokenType::Quote), - )))], - ) - } - - #[test] - fn test_quasiquote_shorthand() { - assert_parse( - "`(+ 1 2)", - &[ExprKind::List(List::new(vec![ - atom("quasiquote"), - ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), - ]))], - ) - } - - #[test] - fn test_quasiquote_normal() { - assert_parse( - "(quasiquote (+ 1 2))", - &[ExprKind::List(List::new(vec![ - atom("quasiquote"), - ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), - ]))], - ) - } - - #[test] - fn test_unquote_shorthand() { - assert_parse( - ",(+ 1 2)", - &[ExprKind::List(List::new(vec![ - atom("unquote"), - ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), - ]))], - ) - } - - #[test] - fn test_unquote_normal() { - assert_parse( - "(unquote (+ 1 2))", - &[ExprKind::List(List::new(vec![ - atom("unquote"), - ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), - ]))], - ) - } - - #[test] - fn test_unquote_splicing_shorthand() { - assert_parse( - ",@(+ 1 2)", - &[ExprKind::List(List::new(vec![ - atom("unquote-splicing"), - ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), - ]))], - ) - } - - #[test] - fn test_unquote_splicing_normal() { - assert_parse( - "(unquote-splicing (+ 1 2))", - &[ExprKind::List(List::new(vec![ - atom("unquote-splicing"), - ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), - ]))], - ) - } - - #[test] - fn test_define_simple() { - assert_parse( - "(define a 10)", - &[ExprKind::Define(Box::new(Define::new( - atom("a"), - int(10), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_define_func_simple() { - assert_parse( - "(define (foo x) (+ x 10))", - &[ExprKind::Define(Box::new(Define::new( - atom("foo"), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("x")], - ExprKind::List(List::new(vec![atom("+"), atom("x"), int(10)])), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_define_func_multiple_args() { - assert_parse( - "(define (foo x y z) (+ x 10))", - &[ExprKind::Define(Box::new(Define::new( - atom("foo"), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("x"), atom("y"), atom("z")], - ExprKind::List(List::new(vec![atom("+"), atom("x"), int(10)])), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_define_func_multiple_args_multiple_body_exprs() { - assert_parse( - "(define (foo x y z) (+ x 10) (+ y 20) (+ z 30))", - &[ExprKind::Define(Box::new(Define::new( - atom("foo"), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("x"), atom("y"), atom("z")], - ExprKind::Begin(Begin::new( - vec![ - ExprKind::List(List::new(vec![atom("+"), atom("x"), int(10)])), - ExprKind::List(List::new(vec![atom("+"), atom("y"), int(20)])), - ExprKind::List(List::new(vec![atom("+"), atom("z"), int(30)])), - ], - SyntaxObject::default(TokenType::Begin), - )), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_recursive_function() { - assert_parse( - "(define (test) (define (foo) (bar)) (define (bar) (foo)))", - &[ExprKind::Define(Box::new(Define::new( - atom("test"), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![], - ExprKind::Begin(Begin::new( - vec![ - ExprKind::Define(Box::new(Define::new( - atom("foo"), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![], - ExprKind::List(List::new(vec![atom("bar")])), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - ))), - ExprKind::Define(Box::new(Define::new( - atom("bar"), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![], - ExprKind::List(List::new(vec![atom("foo")])), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - ))), - ], - SyntaxObject::default(TokenType::Begin), - )), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_return_normal() { - assert_parse( - "(return! 10)", - &[ExprKind::Return(Box::new(Return::new( - int(10), - SyntaxObject::default(TokenType::Return), - )))], - ) - } - - #[test] - fn test_begin() { - assert_parse( - "(begin 1 2 3)", - &[ExprKind::Begin(Begin::new( - vec![int(1), int(2), int(3)], - SyntaxObject::default(TokenType::Begin), - ))], - ) - } - - #[test] - fn test_lambda_function() { - assert_parse( - "(lambda (x) 10)", - &[ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("x")], - int(10), - SyntaxObject::default(TokenType::Lambda), - )))], - ) - } - - #[test] - fn test_lambda_matches_let() { - assert_parse( - "((lambda (a) (+ a 20)) 10)", - &[ExprKind::List(List::new(vec![ - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("a")], - ExprKind::List(List::new(vec![atom("+"), atom("a"), int(20)])), - SyntaxObject::default(TokenType::Lambda), - ))), - int(10), - ]))], - ); - } - - #[test] - fn test_let() { - assert_parse( - "(let ([a 10]) (+ a 20))", - &[ExprKind::List(List::new(vec![ - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("a")], - ExprKind::List(List::new(vec![atom("+"), atom("a"), int(20)])), - SyntaxObject::default(TokenType::Let), - ))), - int(10), - ]))], - ) - } - - #[test] - fn test_quote_with_inner_nested() { - assert_parse( - "'(#f '())", - &[ExprKind::Quote( - Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default( - TokenType::BooleanLiteral(false), - ))), - ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ])), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - )], - ) - } - - #[test] - fn test_quote_with_inner_nested_sub_expr() { - assert_parse( - "(if (null? contents) - '(#f '()) - (list (car contents) (cdr contents)))", - &[ExprKind::If(Box::new(If::new( - ExprKind::List(List::new(vec![atom("null?"), atom("contents")])), - ExprKind::Quote( - Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default( - TokenType::BooleanLiteral(false), - ))), - ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ])), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ExprKind::List(List::new(vec![ - atom("list"), - ExprKind::List(List::new(vec![atom("car"), atom("contents")])), - ExprKind::List(List::new(vec![atom("cdr"), atom("contents")])), - ])), - SyntaxObject::default(TokenType::If), - )))], - ); - } - - #[test] - fn test_quote_normal_with_inner_nested_sub_expr() { - assert_parse( - "(if (null? contents) - (quote (#f '())) - (list (car contents) (cdr contents)))", - &[ExprKind::If(Box::new(If::new( - ExprKind::List(List::new(vec![atom("null?"), atom("contents")])), - ExprKind::Quote( - Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default( - TokenType::BooleanLiteral(false), - ))), - ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ])), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ExprKind::List(List::new(vec![ - atom("list"), - ExprKind::List(List::new(vec![atom("car"), atom("contents")])), - ExprKind::List(List::new(vec![atom("cdr"), atom("contents")])), - ])), - SyntaxObject::default(TokenType::If), - )))], - ); - } - - #[test] - fn test_quote_with_inner_sub_expr_even_more_nested() { - assert_parse( - "(list - (if (null? contents) - '(#f '()) - (list (car contents) (cdr contents))))", - &[ExprKind::List(List::new(vec![ - atom("list"), - ExprKind::If(Box::new(If::new( - ExprKind::List(List::new(vec![atom("null?"), atom("contents")])), - ExprKind::Quote( - Quote::new( - ExprKind::List(List::new(vec![ - ExprKind::Atom(Atom::new(SyntaxObject::default( - TokenType::BooleanLiteral(false), - ))), - ExprKind::Quote( - Quote::new( - List::new(vec![]).into(), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ])), - SyntaxObject::default(TokenType::Quote), - ) - .into(), - ), - ExprKind::List(List::new(vec![ - atom("list"), - ExprKind::List(List::new(vec![atom("car"), atom("contents")])), - ExprKind::List(List::new(vec![atom("cdr"), atom("contents")])), - ])), - SyntaxObject::default(TokenType::If), - ))), - ]))], - ); - } - - #[test] - fn test_define_with_datum_syntax_name() { - assert_parse( - "(define (datum->syntax var) (car ret-value))", - &[ExprKind::Define(Box::new(Define::new( - ExprKind::List(List::new(vec![atom("datum->syntax"), atom("var")])), - ExprKind::List(List::new(vec![atom("car"), atom("ret-value")])), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_define_with_datum_syntax_function_name() { - assert_parse( - "(define ((datum->syntax var) arg) 10)", - &[ExprKind::Define(Box::new(Define::new( - ExprKind::List(List::new(vec![atom("datum->syntax"), atom("var")])), - ExprKind::LambdaFunction(Box::new(LambdaFunction::new( - vec![atom("arg")], - int(10), - SyntaxObject::default(TokenType::Lambda), - ))), - SyntaxObject::default(TokenType::Define), - )))], - ) - } - - #[test] - fn test_parse_without_lowering_ast() { - let a: Result> = - Parser::new_flat("(define (quote a) 10) (require foo bar)", None) - .map(|x| x.and_then(lower_macro_and_require_definitions)) - .collect(); - - let a = a.unwrap(); - - println!("{:#?}", a); - } - - #[test] - fn test_delayed_lowering() { - let a: Result> = Parser::new_flat( - r#" - ;; (define foo (quote a)) (require foo bar) - - ;; (define (foo) (if 10 20 30)) - - ;; (define (foo) (quote (define 10 20))) - - ;; (define foo '()) - - (define-syntax - with-handler - (syntax-rules - () - ((with-handler handler expr) - (reset - (call-with-exception-handler - (λ (err) - (begin (handler err) (shift k (k void)))) - (λ () - expr)))) - ((with-handler handler expr ...) - (reset - (call-with-exception-handler - (λ (err) - (begin (handler err) (shift k (k void)))) - (λ () - (begin expr ...))))))) - - - - "#, - None, - ) - .map(|x| x.and_then(lower_macro_and_require_definitions)) - .map(|x| x.and_then(lower_entire_ast)) - .collect(); - - let a = a.unwrap(); - - println!("{:#?}", a); - } -} diff --git a/crates/steel-core/src/primitives.rs b/crates/steel-core/src/primitives.rs index 4cd30ec0f..e3d082600 100644 --- a/crates/steel-core/src/primitives.rs +++ b/crates/steel-core/src/primitives.rs @@ -24,9 +24,6 @@ pub mod web; #[cfg(feature = "sqlite")] pub mod sqlite; -#[cfg(feature = "blocking_requests")] -pub mod blocking_requests; - pub use lists::UnRecoverableResult; use crate::values::closed::HeapRef; diff --git a/crates/steel-core/src/primitives/blocking_requests.rs b/crates/steel-core/src/primitives/blocking_requests.rs deleted file mode 100644 index 578f0025a..000000000 --- a/crates/steel-core/src/primitives/blocking_requests.rs +++ /dev/null @@ -1,78 +0,0 @@ -// use reqwest::{ -// blocking::{get, Client, RequestBuilder, Response}, -// StatusCode, -// }; - -use crate::{ - rvals::Custom, - steel_vm::{builtin::BuiltInModule, register_fn::RegisterFn}, -}; - -use ureq::{Request, Response}; - -impl Custom for Request {} -impl Custom for Response {} -impl Custom for ureq::Error {} - -fn get(url: String) -> Request { - ureq::get(&url) -} - -impl Custom for std::io::Error {} - -struct SteelResponse { - response: Option, -} - -impl SteelResponse { - fn into_text(&mut self) -> crate::rvals::Result { - let resp = self.response.take(); - - if let Some(resp) = resp { - Ok(resp.into_string()?) - } else { - crate::stop!(Generic => "Response already consumed, can only consume the response one time!") - } - } - - fn into_json(&mut self) -> crate::rvals::Result { - let resp = self.response.take(); - - if let Some(resp) = resp { - Ok(resp.into_json()?) - - // fn json(&mut self) -> Option> { - // self.response.take().map(|x| x.json::()) - // } - } else { - crate::stop!(Generic => "Response already consumed, can only consume the response one time!") - } - } -} - -impl From for SteelResponse { - fn from(value: ureq::Response) -> Self { - SteelResponse { - response: Some(value), - } - } -} - -impl Custom for SteelResponse {} - -pub fn blocking_requests_module() -> BuiltInModule { - let mut module = BuiltInModule::new("steel/web/blocking/requests".to_string()); - - module - .register_fn("get", get) - .register_fn( - "call", - |request: Request| -> Result { - Request::call(request).map(|x| x.into()) - }, - ) - .register_fn("response->text", SteelResponse::into_text) - .register_fn("response->json", SteelResponse::into_json); - - module -} diff --git a/crates/steel-core/src/primitives/strings.rs b/crates/steel-core/src/primitives/strings.rs index c98f13430..099d7e0c6 100644 --- a/crates/steel-core/src/primitives/strings.rs +++ b/crates/steel-core/src/primitives/strings.rs @@ -29,6 +29,8 @@ pub fn string_module() -> BuiltInModule { .register_native_fn_definition(TRIM_START_DEFINITION) .register_native_fn_definition(TRIM_END_DEFINITION) .register_native_fn_definition(SPLIT_WHITESPACE_DEFINITION) + .register_native_fn_definition(SPLIT_ONCE_DEFINITION) + .register_native_fn_definition(SPLIT_MANY_DEFINITION) .register_native_fn_definition(STRING_TO_INT_DEFINITION) .register_native_fn_definition(INT_TO_STRING_DEFINITION) .register_native_fn_definition(STRING_TO_SYMBOL_DEFINITION) @@ -444,6 +446,23 @@ pub fn split_whitespace(value: &SteelString) -> SteelVal { split.into() } +#[function(name = "split-once")] +pub fn split_once(value: &SteelString, pat: &SteelString) -> SteelVal { + let split: Option> = value + .split_once(pat.as_str()) + .map(|(x, y)| vec![SteelVal::StringV(x.into()), SteelVal::StringV(y.into())].into()); + split.into() +} + +#[function(name = "split-many")] +pub fn split_many(value: &SteelString, pat: &SteelString) -> SteelVal { + let split: List = value + .split(pat.as_str()) + .map(|x| SteelVal::StringV(x.into())) + .collect(); + split.into() +} + /// Checks if the input string starts with a prefix /// /// (starts-with? input pattern) -> bool? diff --git a/crates/steel-core/src/primitives/web/mod.rs b/crates/steel-core/src/primitives/web/mod.rs index cf6a58389..0ad326568 100644 --- a/crates/steel-core/src/primitives/web/mod.rs +++ b/crates/steel-core/src/primitives/web/mod.rs @@ -1,2 +1 @@ pub mod requests; -pub mod websockets; diff --git a/crates/steel-core/src/primitives/web/websockets.rs b/crates/steel-core/src/primitives/web/websockets.rs deleted file mode 100644 index ecd859c67..000000000 --- a/crates/steel-core/src/primitives/web/websockets.rs +++ /dev/null @@ -1,71 +0,0 @@ -use std::net::TcpStream; - -use tungstenite::{ - connect, handshake::client::Response, stream::MaybeTlsStream, Message, WebSocket, -}; - -use crate::{ - rvals::Custom, - steel_vm::{builtin::BuiltInModule, register_fn::RegisterFn}, -}; - -type SteelWebSocket = WebSocket>; - -impl Custom for SteelWebSocket {} -impl Custom for Message { - fn fmt(&self) -> Option> { - Some(Ok(format!("{:?}", self))) - } -} -impl Custom for Response {} -impl Custom for tungstenite::Error { - fn fmt(&self) -> Option> { - Some(Ok(format!("{:?}", self))) - } -} - -/// In lieu of a direct bytes type in Steel, since we need to typically respond to a ping with a pong -/// containing the matching payload, we can skip the deserializing into the Steel type and directly -/// construct the pong here -fn ping_to_pong(message: &Message) -> Option { - if let Message::Ping(payload) = message { - Some(Message::Pong(payload.clone())) - } else { - None - } -} - -/// Extracts the payload from a given message -pub fn text_payload(message: &Message) -> Option { - if let Message::Text(text) = &message { - Some(text.clone()) - } else { - None - } -} - -// TODO: @Matt -> perhaps explore this and/or explore using tokio tungstenite for this -fn _make_non_blocking(socket: &mut SteelWebSocket) -> std::result::Result<(), std::io::Error> { - match socket.get_mut() { - MaybeTlsStream::Plain(s) => s.set_nonblocking(true), - MaybeTlsStream::Rustls(s) => s.get_mut().set_nonblocking(true), - _ => todo!(), - } -} - -pub fn websockets_module() -> BuiltInModule { - let mut module = BuiltInModule::new("steel/web/ws"); - - module - .register_fn("ws/message-ping?", Message::is_ping) - .register_fn("ws/message-pog?", Message::is_pong) - .register_fn("ws/message-text?", Message::is_text) - .register_fn("ws/message-text", Message::Text) - .register_fn("ws/message-ping->pong", ping_to_pong) - .register_fn("ws/message->text-payload", text_payload) - .register_fn("ws/connect", connect::) - .register_fn("ws/read-message!", SteelWebSocket::read_message) - .register_fn("ws/write-message!", SteelWebSocket::write_message); - - module -} diff --git a/crates/steel-core/src/rvals.rs b/crates/steel-core/src/rvals.rs index e2408cd92..48e180a7a 100644 --- a/crates/steel-core/src/rvals.rs +++ b/crates/steel-core/src/rvals.rs @@ -13,19 +13,14 @@ use crate::{ values::port::SteelPort, values::{ closed::{HeapRef, MarkAndSweepContext}, - // contracts::{ContractType, ContractedFunction}, functions::ByteCodeLambda, lazy_stream::LazyStream, structs::SerializableUserDefinedStruct, - // lists::ListDropHandler, transducers::{Reducer, Transducer}, }, values::{functions::BoxedDynFunction, structs::UserDefinedStruct}, }; -// #[cfg(feature = "jit")] -// use crate::jit::sig::JitFunctionPointer; - use std::{ any::{Any, TypeId}, cell::{Ref, RefCell, RefMut}, @@ -85,31 +80,12 @@ pub fn new_rc_ref_cell(x: SteelVal) -> RcRefSteelVal { pub type Result = result::Result; pub type FunctionSignature = fn(&[SteelVal]) -> Result; pub type MutFunctionSignature = fn(&mut [SteelVal]) -> Result; -// pub type FunctionSignature = fn(&[SteelVal]) -> Result; - -// TODO: This increases the size of the SteelVal enum by 8 bytes. Consider boxing it instead pub type BoxedFunctionSignature = Rc Result>>; - pub type BoxedAsyncFunctionSignature = Box Result>>; - -// Do something like this: -// vector of async functions -// then for a wait group, make a closure that looks something like this: -// async move vec |_| { -// let values = Vec::new(); -// for func in vec { -// values.push(func(args).await) -// } -// values -// } - -// pub type BoxedFutureResult = Shared>>; pub type AsyncSignature = fn(&[SteelVal]) -> FutureResult; pub type BoxedFutureResult = Pin>>>; -// Pin + 'a + Send>>; - #[derive(Clone)] pub struct FutureResult(Shared); @@ -286,29 +262,6 @@ impl IntoSerializableSteelVal for } } -// impl<'a, T: CustomType + Clone + ?Sized + 'a> FromSteelVal for &'a T { -// fn from_steelval(val: SteelVal) -> Result { -// if let SteelVal::Custom(v) = val { -// let left_type = v.as_any(); -// let left: Option = left_type.downcast_ref::().cloned(); -// left.ok_or_else(|| { -// let error_message = format!( -// "Type Mismatch: Type of SteelVal did not match the given type: {}", -// std::any::type_name::() -// ); -// SteelErr::new(ErrorKind::ConversionError, error_message) -// }) -// } else { -// let error_message = format!( -// "Type Mismatch: Type of SteelVal did not match the given type: {}", -// std::any::type_name::() -// ); - -// Err(SteelErr::new(ErrorKind::ConversionError, error_message)) -// } -// } -// } - // TODO: Marshalling out of the type could also try to yoink from a native steel struct. // If possible, we can try to line the constructor up with the fields impl FromSteelVal for T { @@ -338,29 +291,6 @@ impl FromSteelVal for T { } } -// impl<'a, T: CustomType + Clone> FromSteelVal for &'a T { -// fn from_steelval(val: &SteelVal) -> Result<&'a T> { -// if let SteelVal::Custom(v) = val { -// let left_type = v.as_any_ref(); -// let left = left_type.downcast_ref::(); -// left.ok_or_else(|| { -// let error_message = format!( -// "Type Mismatch: Type of SteelVal did not match the given type: {}", -// std::any::type_name::() -// ); -// SteelErr::new(ErrorKind::ConversionError, error_message) -// }) -// } else { -// let error_message = format!( -// "Type Mismatch: Type of SteelVal did not match the given type: {}", -// std::any::type_name::() -// ); - -// Err(SteelErr::new(ErrorKind::ConversionError, error_message)) -// } -// } -// } - /// The entry point for turning values into SteelVals /// The is implemented for most primitives and collections /// You can also manually implement this for any type, or can optionally @@ -611,54 +541,6 @@ impl AsRefMutSteelVal for T { } } -// ListV(l) => { -// // Rooted - things operate as normal -// if self.qq_depth == 0 { -// let maybe_special_form = l.first().and_then(|x| x.as_string()); - -// match maybe_special_form { -// Some(x) if x.as_str() == "quote" => { -// if self.quoted { -// let items: std::result::Result, &'static str> = -// l.iter().map(|x| self.visit(x)).collect(); - -// return Ok(ExprKind::List(List::new(items?))); -// } - -// self.quoted = true; - -// let return_value = l -// .into_iter() -// .map(|x| self.visit(x)) -// .collect::, _>>()? -// .try_into() -// .map_err(|_| { -// "parse error! If you're running into this, please file an issue" -// }); - -// self.quoted = false; - -// return return_value; -// } // "quasiquote" => { -// // self.qq_depth += 1; -// // } -// _ => {} -// } -// } - -// l.into_iter() -// .map(|x| self.visit(x)) -// .collect::, _>>()? -// .try_into() -// .map_err(|_| "If you're running into this, please file an issue") - -// // If we're not quoted, we need to just return this pushed down to an ast -// // let items: std::result::Result, &'static str> = -// // l.iter().map(|x| self.visit(x)).collect(); - -// // Ok(ExprKind::List(List::new(items?))) -// } - impl ast::TryFromSteelValVisitorForExprKind { pub fn visit_syntax_object(&mut self, value: &Syntax) -> Result { let span = value.span; diff --git a/crates/steel-core/src/steel_vm/contracts.rs b/crates/steel-core/src/steel_vm/contracts.rs deleted file mode 100644 index bb1c10a61..000000000 --- a/crates/steel-core/src/steel_vm/contracts.rs +++ /dev/null @@ -1,742 +0,0 @@ -// impl ContractedFunction { -// pub fn apply( -// &self, -// arguments: Vec, -// cur_inst_span: &Span, -// ctx: &mut VmCore, -// ) -> Result { -// // Walk back and find the contracts to apply -// { -// let mut parent = self.contract.parent(); -// while let Some(p) = parent { -// // println!("Applying parents"); -// p.apply(&self.name, &self.function, &arguments, cur_inst_span, ctx)?; - -// parent = p.parent() -// } -// } - -// self.contract -// .apply(&self.name, &self.function, &arguments, cur_inst_span, ctx) -// } -// } - -// impl FlatContract { -// pub fn apply(&self, arg: SteelVal, cur_inst_span: &Span, ctx: &mut VmCore) -> Result<()> { -// // TODO make this not clone the argument -// let output = match self.predicate() { -// SteelVal::FuncV(func) => func(&[arg.clone()]).map_err(|x| x.set_span(*cur_inst_span)), -// SteelVal::BoxedFunction(func) => { -// func.func()(&[arg.clone()]).map_err(|x| x.set_span(*cur_inst_span)) -// } -// SteelVal::Closure(closure) => ctx.call_with_one_arg(closure, arg.clone()), -// SteelVal::ContractedFunction(c) => c.apply(vec![arg.clone()], cur_inst_span, ctx), -// _ => { -// stop!(TypeMismatch => format!("contract expected a function, found: {:?}", self.predicate()); *cur_inst_span) -// } -// }?; - -// if output.is_truthy() { -// Ok(()) -// } else { -// stop!(ContractViolation => format!("Found in the application of a flat contract for {}: the given input: {} resulted in a contract violation", &self.name, arg); *cur_inst_span); -// } -// } -// } - -// /// Extension trait for the application of function contracts -// pub(crate) trait FunctionContractExt { -// fn apply( -// &self, -// name: &Option, -// // function: &Gc, -// function: &SteelVal, -// arguments: &[SteelVal], -// cur_inst_span: &Span, -// ctx: &mut VmCore, -// ) -> Result; -// } - -// impl FunctionContractExt for FunctionKind { -// fn apply( -// &self, -// name: &Option, -// function: &SteelVal, -// arguments: &[SteelVal], -// cur_inst_span: &Span, -// ctx: &mut VmCore, -// ) -> Result { -// match self { -// Self::Basic(fc) => fc.apply(name, function, arguments, cur_inst_span, ctx), -// Self::Dependent(dc) => dc.apply(name, function, arguments, cur_inst_span, ctx), -// } -// } -// } - -// impl FunctionContractExt for DependentContract { -// fn apply( -// &self, -// name: &Option, -// function: &SteelVal, -// arguments: &[SteelVal], -// cur_inst_span: &Span, -// ctx: &mut VmCore, -// ) -> Result { -// let mut verified_args: Vec = Vec::new(); - -// for (i, (arg, dependent_pair)) in -// arguments.iter().zip(self.pre_conditions.iter()).enumerate() -// { -// let thunk = &dependent_pair.thunk; - -// let arg_stack = dependent_pair -// .arguments -// .iter() -// .map(|named_argument| { -// self.arg_positions -// .get(named_argument) -// .and_then(|x| arguments.get(*x)) -// .cloned() -// }) -// .collect::>>() -// .expect("missing argument in dependent contract"); - -// let contract = { -// ctx.call_with_args(thunk, arg_stack)?.contract_or_else( -// throw!(TypeMismatch => "dependent contract expected a contract"), -// )? -// }; - -// match contract.as_ref() { -// ContractType::Flat(f) => { -// debug!("applying flat contract in pre condition: {}", f.name); - -// if let Err(e) = f.apply(arg.clone(), cur_inst_span, ctx) { -// debug!( -// "Blame locations: {:?}, {:?}", -// self.contract_attachment_location, name -// ); - -// let message = format!("This function call caused an error - it occured in the domain position: {}, with the contract: {}, {}, blaming: {:?} (callsite)", i, self, e, self.contract_attachment_location); - -// stop!(ContractViolation => message; *cur_inst_span); -// } - -// verified_args.push(arg.clone()); -// } -// ContractType::Function(fc) => match arg { -// SteelVal::ContractedFunction(contracted_function) => { -// let mut pre_parent = contracted_function.contract.clone(); -// pre_parent.set_attachment_location(contracted_function.name.clone()); - -// let parent = Gc::new(pre_parent); - -// let func = contracted_function.function.clone(); - -// debug!( -// "Setting the parent: {} on a precondition function: {}", -// parent.to_string(), -// fc.to_string() -// ); - -// // Get the contract down from the -// let mut fc = fc.clone(); -// fc.set_parent(parent); -// debug!( -// "Inside: {:?}, Setting attachment location in range to: {:?}", -// name, contracted_function.name -// ); -// fc.set_attachment_location(contracted_function.name.clone()); - -// // TODO Don't pass in None -// let new_arg = ContractedFunction::new(fc, func, name.clone()).into(); - -// verified_args.push(new_arg); -// } - -// _ => verified_args.push( -// ContractedFunction::new(fc.clone(), arg.clone(), name.clone()).into(), -// ), -// }, -// } -// } - -// let output = match function { -// SteelVal::Closure(function) => ctx.call_with_args(function, verified_args)?, -// SteelVal::BoxedFunction(f) => { -// f.func()(&verified_args).map_err(|x| x.set_span(*cur_inst_span))? -// } -// SteelVal::FuncV(f) => f(&verified_args).map_err(|x| x.set_span(*cur_inst_span))?, -// SteelVal::FutureFunc(f) => SteelVal::FutureV(Gc::new( -// f(&verified_args).map_err(|x| x.set_span(*cur_inst_span))?, -// )), -// _ => { -// todo!("Implement contract application for non bytecode values"); -// } -// }; - -// let thunk = &self.post_condition.thunk; - -// let arg_stack = self -// .post_condition -// .arguments -// .iter() -// .map(|named_argument| { -// self.arg_positions -// .get(named_argument) -// .and_then(|x| arguments.get(*x)) -// .cloned() -// }) -// .collect::>>() -// .expect("missing argument in dependent contract"); - -// let contract = { -// ctx.call_with_args(thunk, arg_stack)?.contract_or_else( -// throw!(TypeMismatch => "dependent contract expected a contract"), -// )? -// }; - -// match contract.as_ref() { -// ContractType::Flat(f) => { -// debug!("applying flat contract in post condition: {}", f.name); - -// if let Err(e) = f.apply(output.clone(), cur_inst_span, ctx) { -// debug!( -// "Blame locations: {:?}, {:?}", -// self.contract_attachment_location, name -// ); - -// debug!("Parent exists: {}", self.parent().is_some()); - -// let blame_location = if self.contract_attachment_location.is_none() { -// name -// } else { -// &self.contract_attachment_location -// }; - -// // TODO clean this up -// if let Some(blame_location) = blame_location { -// let error_message = format!("this function call resulted in an error - occured in the range position of this contract: {self} \n -// {e} -// blaming: {blame_location} - broke its own contract"); - -// stop!(ContractViolation => error_message; *cur_inst_span); -// } else { -// let error_message = format!("this function call resulted in an error - occured in the range position of this contract: {self} \n -// {e} -// blaming: None - broke its own contract"); - -// stop!(ContractViolation => error_message; *cur_inst_span); -// } -// } - -// Ok(output) -// } -// ContractType::Function(fc) => match output { -// SteelVal::ContractedFunction(contracted_function) => { -// let mut pre_parent = contracted_function.contract.clone(); -// pre_parent.set_attachment_location(contracted_function.name.clone()); - -// let parent = Gc::new(pre_parent); - -// let func = contracted_function.function.clone(); - -// debug!( -// "Setting the parent: {} on a postcondition function: {}", -// parent.to_string(), -// fc.to_string() -// ); - -// // Get the contract down from the parent -// let mut fc = fc.clone(); -// fc.set_parent(parent); -// debug!( -// "Inside: {:?}, Setting attachment location in range to: {:?}", -// name, contracted_function.name -// ); - -// // TODO Don't pass in None here -// let output = ContractedFunction::new(fc, func, name.clone()).into(); - -// Ok(output) -// } - -// _ => Ok(ContractedFunction::new(fc.clone(), output, name.clone()).into()), -// }, -// } -// } -// } - -// impl FunctionContract { -// pub fn apply( -// &self, -// name: &Option, -// function: &SteelVal, -// arguments: &[SteelVal], -// cur_inst_span: &Span, -// ctx: &mut VmCore, -// ) -> Result { -// let verified_args = self.verify_preconditions(arguments, cur_inst_span, ctx, name)?; - -// // TODO use actual VM with real stack instead - -// /* -// Ideas for this: call a builtin - -// */ -// let output = match function { -// SteelVal::Closure(function) => { -// // TODO: Here is the problem - we recur by calling the function -// // What we should do is actually leverage the stack in the VM directly instead of making a recursive call here -// ctx.call_with_args(function, verified_args.into_iter())? -// } -// SteelVal::BoxedFunction(f) => { -// f.func()(&verified_args).map_err(|x| x.set_span(*cur_inst_span))? -// } -// SteelVal::FuncV(f) => f(&verified_args).map_err(|x| x.set_span(*cur_inst_span))?, -// SteelVal::FutureFunc(f) => SteelVal::FutureV(Gc::new( -// f(&verified_args).map_err(|x| x.set_span(*cur_inst_span))?, -// )), -// _ => { -// todo!("Implement contract application for non bytecode values"); -// } -// }; - -// match self.post_condition().as_ref() { -// ContractType::Flat(f) => { -// // unimplemented!(); - -// debug!("applying flat contract in post condition: {}", f.name); - -// if let Err(e) = f.apply(output.clone(), cur_inst_span, ctx) { -// debug!( -// "Blame locations: {:?}, {:?}", -// self.contract_attachment_location, name -// ); - -// debug!("Parent exists: {}", self.parent().is_some()); - -// let blame_location = if self.contract_attachment_location.is_none() { -// name -// } else { -// &self.contract_attachment_location -// }; - -// // TODO clean this up -// if let Some(blame_location) = blame_location { -// let error_message = format!("this function call resulted in an error - occured in the range position of this contract: {self} \n -// {e} -// blaming: {blame_location} - broke its own contract"); - -// stop!(ContractViolation => error_message; *cur_inst_span); -// } else { -// let error_message = format!("this function call resulted in an error - occured in the range position of this contract: {self} \n -// {e} -// blaming: None - broke its own contract"); - -// stop!(ContractViolation => error_message; *cur_inst_span); -// } -// } - -// Ok(output) -// } -// ContractType::Function(fc) => match output { -// SteelVal::ContractedFunction(contracted_function) => { -// let mut pre_parent = contracted_function.contract.clone(); -// pre_parent.set_attachment_location(contracted_function.name.clone()); - -// let parent = Gc::new(pre_parent); - -// let func = contracted_function.function.clone(); - -// debug!( -// "Setting the parent: {} on a postcondition function: {}", -// parent.to_string(), -// fc.to_string() -// ); - -// // Get the contract down from the parent -// let mut fc = fc.clone(); -// fc.set_parent(parent); -// debug!( -// "Inside: {:?}, Setting attachment location in range to: {:?}", -// name, contracted_function.name -// ); - -// // TODO Don't pass in None here -// let output = ContractedFunction::new(fc, func, name.clone()).into(); - -// Ok(output) -// } - -// _ => Ok(ContractedFunction::new(fc.clone(), output, name.clone()).into()), -// }, -// } -// } -// } - -// impl FunctionContract { -// fn verify_preconditions( -// &self, -// arguments: &[SteelVal], -// cur_inst_span: &Span, -// ctx: &mut VmCore, -// name: &Option, -// ) -> Result> { -// let mut verified_args = Vec::new(); - -// for (i, (arg, contract)) in arguments -// .iter() -// .zip(self.pre_conditions().iter()) -// .enumerate() -// { -// match contract.as_ref() { -// ContractType::Flat(f) => { -// debug!("applying flat contract in pre condition: {}", f.name); - -// if let Err(e) = f.apply(arg.clone(), cur_inst_span, ctx) { -// debug!( -// "Blame locations: {:?}, {:?}", -// self.contract_attachment_location, name -// ); - -// let message = format!("This function call caused an error - it occured in the domain position: {}, with the contract: {}, {}, blaming: {:?} (callsite)", i, self, e, self.contract_attachment_location); - -// stop!(ContractViolation => message; *cur_inst_span); -// } - -// verified_args.push(arg.clone()); -// } -// ContractType::Function(fc) => match arg { -// SteelVal::ContractedFunction(contracted_function) => { -// let mut pre_parent = contracted_function.contract.clone(); -// pre_parent.set_attachment_location(contracted_function.name.clone()); - -// let parent = Gc::new(pre_parent); - -// let func = contracted_function.function.clone(); - -// debug!( -// "Setting the parent: {} on a precondition function: {}", -// parent.to_string(), -// fc.to_string() -// ); - -// // Get the contract down from the -// let mut fc = fc.clone(); -// fc.set_parent(parent); -// debug!( -// "Inside: {:?}, Setting attachment location in range to: {:?}", -// name, contracted_function.name -// ); -// fc.set_attachment_location(contracted_function.name.clone()); - -// // TODO Don't pass in None -// let new_arg = ContractedFunction::new(fc, func, name.clone()).into(); - -// verified_args.push(new_arg); -// } - -// _ => verified_args.push( -// ContractedFunction::new(fc.clone(), arg.clone(), name.clone()).into(), -// ), -// }, -// } -// } - -// Ok(verified_args) -// } -// } - -#[cfg(test)] -mod contract_tests { - use crate::steel_vm::test_util::{assert_script, assert_script_error}; - - #[test] - fn simple_flat_contract() { - let script = r#" - (define/contract (test x y) - (->/c even? even? odd?) - (+ x y 1)) - - (assert! (equal? (test 2 2) 5)) - "#; - assert_script(script); - } - - #[test] - fn simple_flat_contract_domain_violation() { - let script = r#" - (define/contract (test x y) - (->/c even? even? odd?) - (+ x y 1)) - - (test 1 2) - "#; - assert_script_error(script); - } - - #[test] - fn simple_higher_order_contract() { - let script = r#" - (define/contract (blagh func y) - (->/c (->/c even? odd?) even? even?) - (+ 1 (func y))) - - (assert! (equal? (blagh (lambda (x) (+ x 1)) 2) 4)) - "#; - assert_script(script); - } - - #[test] - fn simple_higher_order_contract_violation() { - let script = r#" - (define/contract (blagh func y) - (->/c (->/c even? odd?) even? even?) - (+ 1 (func y))) - - (blagh (lambda (x) (+ x 2)) 2) - "#; - assert_script_error(script); - } - - // TODO: This will fail on old contract implementation - // #[test] - // fn tail_call_mutual_recursion() { - // let script = r#" - // (define/contract (foo x) - // (->/c int? int?) - // (if (= x 100) - // x - // (bar (+ x 1)))) - - // (define/contract (bar x) - // (->/c int? int?) - // (if (= x 100) - // x - // (foo (+ x 1)))) - - // (assert! (equal? (foo 0) 100)) - // "#; - // assert_script(script); - // } - - #[test] - fn tail_call_contract_still_works() { - let script = r#" - (define/contract (loop x) - (->/c int? int?) - (if (= x 100) - x - (loop (+ x 1)))) - - (assert! (equal? (loop 0) 100)) - "#; - assert_script(script); - } - - #[test] - fn contract_checking_on_application_success() { - let script = r#" - - (define/contract (output) - (->/c (->/c string? int?)) - (lambda (x) 10)) - - (define/contract (accept func) - (->/c (->/c string? int?) string?) - "cool cool cool") - - (assert! (equal? (accept (output)) "cool cool cool")) - "#; - assert_script(script); - } - - #[test] - fn contract_checking_not_called_since_not_applied() { - let script = r#" - - (define/contract (output) - (->/c (->/c string? int?)) - (lambda (x) 10)) - - (define/contract (accept func) - (->/c (->/c string? string?) string?) - "cool cool cool") - - (assert! (equal? (accept (output)) "cool cool cool")) - "#; - assert_script(script); - } - - #[test] - fn contract_checking_on_return_does_not_happen() { - let script = r#" - - (define/contract (output) - (->/c (->/c string? int?)) - (lambda (x) 10)) - - (define/contract (accept) - (->/c (->/c string? string?)) - (output)) - - (accept) - "#; - - assert_script(script); - } - - #[test] - fn contract_application_in_map() { - let script = r#" - (define (test x) - (->/c int? int?) - (+ x 1)) - (define result (map test (list 1 2 3 4))) - (assert! (equal? result (list 2 3 4 5))) - "#; - assert_script(script); - } - - #[test] - fn contract_application_in_filter() { - let script = r#" - (define (test x) - (->/c int? boolean?) - (even? x)) - (define result (filter test (list 1 2 3 4))) - (assert! (equal? result (list 2 4))) - "#; - assert_script(script); - } - - #[test] - fn contract_checking_with_weaker() { - let script = r#" - (define/contract (output) - (->/c (->/c string? int?)) - (lambda (x) 10.0)) - - (define/contract (accept) - (->/c (->/c string? number?)) - (output)) - - ((accept) "test") - "#; - - assert_script_error(script) - } - - #[test] - fn contract_checking_pre_condition_later() { - let script = r#" - (define/contract (output) - (->/c (->/c list? int?)) - (lambda (x) 10.0)) - - (define/contract (accept) - (->/c (->/c string? number?)) - (output)) - - ((accept) "test") - "#; - - assert_script_error(script); - } - - #[test] - fn three_levels_of_contracts() { - let script = r#" - (define (any? x) #t) - - (define/contract (level1) - (->/c (->/c int?)) - (lambda () 10.2)) - - (define/contract (level2) - (->/c (->/c number?)) - (level1)) - - (define/contract (level3) - (->/c (->/c any?)) - (level2)) - - ((level3)) - "#; - - assert_script_error(script); - } - - #[test] - fn contract_with_filter() { - let script = r#" - (define/contract (is-even? x) - (->/c number? boolean?) - (even? x)) - - (define res (filter is-even? (range 0 10))) - - (assert! (equal? res '(0 2 4 6 8))) - "#; - - assert_script(script); - } - - #[test] - fn contract_with_map() { - let script = r#" - (define/contract (adding1 x) - (->/c number? number?) - (+ x 1)) - - (define res (map adding1 (range 0 5))) - - (assert! (equal? res '(1 2 3 4 5))) - "#; - - assert_script(script); - } - - #[test] - fn contract_with_reduce() { - let script = r#" - (define/contract (reducer accum elem) - (->/c number? number? number?) - (+ accum elem)) - - (define res (transduce (range 0 10) (taking 5) (into-reducer + 0))) - - (assert! (equal? res 10)) - "#; - assert_script(script); - } - - #[test] - fn transducers_containing_contracts() { - let script = r#" - (define/contract (mapper x) - (->/c number? number?) - (+ x 1)) - - (define/contract (is-even? x) - (->/c number? boolean?) - (even? x)) - - (define x (mapping mapper)) - (define y (filtering is-even?)) - (define z (taking 10)) - - (define xyz (compose x y z)) - - (define exec-list (transduce (range 0 100) xyz (into-list))) - (define exec-vector (transduce (range 0 100) xyz (into-vector))) - (define my-sum (transduce (range 0 100) xyz (into-reducer + 0))) - - (assert! (equal? exec-list '(2 4 6 8 10 12 14 16 18 20))) - (assert! (equal? exec-vector (vector 2 4 6 8 10 12 14 16 18 20))) - (assert! (equal? my-sum 110)) - "#; - assert_script(script); - } -} diff --git a/crates/steel-core/src/steel_vm/mod.rs b/crates/steel-core/src/steel_vm/mod.rs index 5eeb8708e..b79a93d29 100644 --- a/crates/steel-core/src/steel_vm/mod.rs +++ b/crates/steel-core/src/steel_vm/mod.rs @@ -2,7 +2,6 @@ pub mod builtin; pub mod cache; pub(crate) mod const_evaluation; pub mod contract_checker; -mod contracts; #[cfg(feature = "dylibs")] pub mod dylib; pub mod engine; diff --git a/crates/steel-core/src/steel_vm/primitives.rs b/crates/steel-core/src/steel_vm/primitives.rs index 3cae68674..2677092d2 100644 --- a/crates/steel-core/src/steel_vm/primitives.rs +++ b/crates/steel-core/src/steel_vm/primitives.rs @@ -51,7 +51,7 @@ use crate::{ }; #[cfg(feature = "web")] -use crate::primitives::web::{requests::requests_module, websockets::websockets_module}; +use crate::primitives::web::requests::requests_module; use crate::values::lists::List; use im_rc::HashMap; @@ -311,16 +311,9 @@ thread_local! { pub static MUTABLE_VECTOR_MODULE: BuiltInModule = mutable_vector_module(); pub static PRIVATE_READER_MODULE: BuiltInModule = reader_module(); - #[cfg(feature = "web")] - pub static WEBSOCKETS_MODULE: BuiltInModule = websockets_module(); - #[cfg(feature = "web")] pub static REQUESTS_MODULE: BuiltInModule = requests_module(); - #[cfg(feature = "blocking_requests")] - pub static BLOCKING_REQUESTS_MODULE: BuiltInModule = crate::primitives::blocking_requests::blocking_requests_module(); - - #[cfg(feature = "sqlite")] pub static SQLITE_MODULE: BuiltInModule = crate::primitives::sqlite::sqlite_module(); } @@ -467,15 +460,10 @@ pub fn register_builtin_modules(engine: &mut Engine) { engine.register_module(PRIVATE_READER_MODULE.with(|x| x.clone())); #[cfg(feature = "web")] - engine - .register_module(WEBSOCKETS_MODULE.with(|x| x.clone())) - .register_module(REQUESTS_MODULE.with(|x| x.clone())); + engine.register_module(REQUESTS_MODULE.with(|x| x.clone())); #[cfg(feature = "sqlite")] engine.register_module(SQLITE_MODULE.with(|x| x.clone())); - - #[cfg(feature = "blocking_requests")] - engine.register_module(BLOCKING_REQUESTS_MODULE.with(|x| x.clone())); } pub static MODULE_IDENTIFIERS: Lazy> = Lazy::new(|| { diff --git a/crates/steel-core/src/steel_vm/tests.rs b/crates/steel-core/src/steel_vm/tests.rs index 4742d4850..8b6c4c7d0 100644 --- a/crates/steel-core/src/steel_vm/tests.rs +++ b/crates/steel-core/src/steel_vm/tests.rs @@ -284,37 +284,295 @@ mod register_type_tests { } } -// #[cfg(test)] -// mod stream_tests { -// use std::cell::RefCell; -// use std::rc::Rc; - -// use crate::parser::span::Span; -// use crate::steel_vm::evaluation_progress::EvaluationProgress; -// use crate::steel_vm::lazy_stream::LazyStreamIter; -// use crate::steel_vm::options::ApplyContract; -// use crate::steel_vm::options::UseCallback; -// use crate::values::lazy_stream::LazyStream; -// use crate::{compiler::constants::ConstantMap, env::Env}; - -// // #[test] -// // fn test_empty_stream_creates_no_iter() { -// // let constants = ConstantMap::new(); -// // let cur_inst_span = Span::new(0, 0); -// // let callback = EvaluationProgress::new(); -// // let mut global_env = Env::root(); -// // let mut mut_ref = &mut global_env; - -// // let lazy_iter = LazyStreamIter::new( -// // LazyStream::new_empty_stream(), -// // &constants, -// // &cur_inst_span, -// // &callback, -// // Rc::new(RefCell::new(&mut mut_ref)), -// // UseCallback, -// // ApplyContract, -// // ); - -// // assert!(lazy_iter.into_iter().next().is_none()); -// // } -// } +#[cfg(test)] +mod contract_tests { + use crate::steel_vm::test_util::{assert_script, assert_script_error}; + + #[test] + fn simple_flat_contract() { + let script = r#" + (define/contract (test x y) + (->/c even? even? odd?) + (+ x y 1)) + + (assert! (equal? (test 2 2) 5)) + "#; + assert_script(script); + } + + #[test] + fn simple_flat_contract_domain_violation() { + let script = r#" + (define/contract (test x y) + (->/c even? even? odd?) + (+ x y 1)) + + (test 1 2) + "#; + assert_script_error(script); + } + + #[test] + fn simple_higher_order_contract() { + let script = r#" + (define/contract (blagh func y) + (->/c (->/c even? odd?) even? even?) + (+ 1 (func y))) + + (assert! (equal? (blagh (lambda (x) (+ x 1)) 2) 4)) + "#; + assert_script(script); + } + + #[test] + fn simple_higher_order_contract_violation() { + let script = r#" + (define/contract (blagh func y) + (->/c (->/c even? odd?) even? even?) + (+ 1 (func y))) + + (blagh (lambda (x) (+ x 2)) 2) + "#; + assert_script_error(script); + } + + // TODO: This will fail on old contract implementation + // #[test] + // fn tail_call_mutual_recursion() { + // let script = r#" + // (define/contract (foo x) + // (->/c int? int?) + // (if (= x 100) + // x + // (bar (+ x 1)))) + + // (define/contract (bar x) + // (->/c int? int?) + // (if (= x 100) + // x + // (foo (+ x 1)))) + + // (assert! (equal? (foo 0) 100)) + // "#; + // assert_script(script); + // } + + #[test] + fn tail_call_contract_still_works() { + let script = r#" + (define/contract (loop x) + (->/c int? int?) + (if (= x 100) + x + (loop (+ x 1)))) + + (assert! (equal? (loop 0) 100)) + "#; + assert_script(script); + } + + #[test] + fn contract_checking_on_application_success() { + let script = r#" + + (define/contract (output) + (->/c (->/c string? int?)) + (lambda (x) 10)) + + (define/contract (accept func) + (->/c (->/c string? int?) string?) + "cool cool cool") + + (assert! (equal? (accept (output)) "cool cool cool")) + "#; + assert_script(script); + } + + #[test] + fn contract_checking_not_called_since_not_applied() { + let script = r#" + + (define/contract (output) + (->/c (->/c string? int?)) + (lambda (x) 10)) + + (define/contract (accept func) + (->/c (->/c string? string?) string?) + "cool cool cool") + + (assert! (equal? (accept (output)) "cool cool cool")) + "#; + assert_script(script); + } + + #[test] + fn contract_checking_on_return_does_not_happen() { + let script = r#" + + (define/contract (output) + (->/c (->/c string? int?)) + (lambda (x) 10)) + + (define/contract (accept) + (->/c (->/c string? string?)) + (output)) + + (accept) + "#; + + assert_script(script); + } + + #[test] + fn contract_application_in_map() { + let script = r#" + (define (test x) + (->/c int? int?) + (+ x 1)) + (define result (map test (list 1 2 3 4))) + (assert! (equal? result (list 2 3 4 5))) + "#; + assert_script(script); + } + + #[test] + fn contract_application_in_filter() { + let script = r#" + (define (test x) + (->/c int? boolean?) + (even? x)) + (define result (filter test (list 1 2 3 4))) + (assert! (equal? result (list 2 4))) + "#; + assert_script(script); + } + + #[test] + fn contract_checking_with_weaker() { + let script = r#" + (define/contract (output) + (->/c (->/c string? int?)) + (lambda (x) 10.0)) + + (define/contract (accept) + (->/c (->/c string? number?)) + (output)) + + ((accept) "test") + "#; + + assert_script_error(script) + } + + #[test] + fn contract_checking_pre_condition_later() { + let script = r#" + (define/contract (output) + (->/c (->/c list? int?)) + (lambda (x) 10.0)) + + (define/contract (accept) + (->/c (->/c string? number?)) + (output)) + + ((accept) "test") + "#; + + assert_script_error(script); + } + + #[test] + fn three_levels_of_contracts() { + let script = r#" + (define (any? x) #t) + + (define/contract (level1) + (->/c (->/c int?)) + (lambda () 10.2)) + + (define/contract (level2) + (->/c (->/c number?)) + (level1)) + + (define/contract (level3) + (->/c (->/c any?)) + (level2)) + + ((level3)) + "#; + + assert_script_error(script); + } + + #[test] + fn contract_with_filter() { + let script = r#" + (define/contract (is-even? x) + (->/c number? boolean?) + (even? x)) + + (define res (filter is-even? (range 0 10))) + + (assert! (equal? res '(0 2 4 6 8))) + "#; + + assert_script(script); + } + + #[test] + fn contract_with_map() { + let script = r#" + (define/contract (adding1 x) + (->/c number? number?) + (+ x 1)) + + (define res (map adding1 (range 0 5))) + + (assert! (equal? res '(1 2 3 4 5))) + "#; + + assert_script(script); + } + + #[test] + fn contract_with_reduce() { + let script = r#" + (define/contract (reducer accum elem) + (->/c number? number? number?) + (+ accum elem)) + + (define res (transduce (range 0 10) (taking 5) (into-reducer + 0))) + + (assert! (equal? res 10)) + "#; + assert_script(script); + } + + #[test] + fn transducers_containing_contracts() { + let script = r#" + (define/contract (mapper x) + (->/c number? number?) + (+ x 1)) + + (define/contract (is-even? x) + (->/c number? boolean?) + (even? x)) + + (define x (mapping mapper)) + (define y (filtering is-even?)) + (define z (taking 10)) + + (define xyz (compose x y z)) + + (define exec-list (transduce (range 0 100) xyz (into-list))) + (define exec-vector (transduce (range 0 100) xyz (into-vector))) + (define my-sum (transduce (range 0 100) xyz (into-reducer + 0))) + + (assert! (equal? exec-list '(2 4 6 8 10 12 14 16 18 20))) + (assert! (equal? exec-vector (vector 2 4 6 8 10 12 14 16 18 20))) + (assert! (equal? my-sum 110)) + "#; + assert_script(script); + } +} diff --git a/crates/steel-core/src/values/structs.rs b/crates/steel-core/src/values/structs.rs index 8f0ad6d17..e245a01fd 100644 --- a/crates/steel-core/src/values/structs.rs +++ b/crates/steel-core/src/values/structs.rs @@ -149,164 +149,6 @@ impl std::fmt::Display for UserDefinedStruct { } } -enum StructBacking { - Immutable(ImmutableMaybeHeapVec), - Mutable(MaybeHeapVec), -} - -impl StructBacking { - fn get(&self, index: usize) -> Option { - match self { - Self::Immutable(v) => v.get(index), - Self::Mutable(v) => v.get(index), - } - } -} - -#[test] -fn check_sizes() { - println!("MaybeHeapVec: {}", std::mem::size_of::()); - println!( - "ImmutableMaybeHeapVec: {}", - std::mem::size_of::() - ); - println!("StructBacking: {}", std::mem::size_of::()); -} - -/// Array backed fixed size vector -/// Structs should be fixed size - they don't need to grow once they are created, and for -/// unit or single values, they shouldn't need to allocate an array at all. For structs up to -/// length 6, we keep these as individual enum variants, so that we can simple switch on the kind. -/// beyond length 6, we spill into a normal vector. -#[derive(Clone, Debug)] -pub(crate) enum MaybeHeapVec { - Unit, - One(SteelVal), - Two([SteelVal; 2]), - Three([SteelVal; 3]), - Four([SteelVal; 4]), - Five([SteelVal; 5]), - Spilled(Vec), -} - -impl MaybeHeapVec { - pub fn from_slice(args: &[SteelVal]) -> Self { - match args { - [] => Self::Unit, - [one] => Self::One(one.clone()), - [one, two] => Self::Two([one.clone(), two.clone()]), - [one, two, three] => Self::Three([one.clone(), two.clone(), three.clone()]), - [one, two, three, four] => { - Self::Four([one.clone(), two.clone(), three.clone(), four.clone()]) - } - [one, two, three, four, five] => Self::Five([ - one.clone(), - two.clone(), - three.clone(), - four.clone(), - five.clone(), - ]), - _ => Self::Spilled(args.to_vec()), - } - } - - #[inline(always)] - pub fn get(&self, index: usize) -> Option { - match self { - MaybeHeapVec::Unit => None, - MaybeHeapVec::One(v) => Some(v.clone()), - MaybeHeapVec::Two(a) => Some(a[index].clone()), - MaybeHeapVec::Three(a) => Some(a[index].clone()), - MaybeHeapVec::Four(a) => Some(a[index].clone()), - MaybeHeapVec::Five(a) => Some(a[index].clone()), - MaybeHeapVec::Spilled(a) => Some(a[index].clone()), - } - } - - #[inline(always)] - pub fn set(&mut self, index: usize, mut value: SteelVal) -> SteelVal { - match self { - MaybeHeapVec::Unit => panic!("Tried to get the 0th index of a unit struct"), - MaybeHeapVec::One(v) => { - let old = v.clone(); - *v = value; - old - } - MaybeHeapVec::Two(a) => { - std::mem::swap(&mut a[index], &mut value); - value - } - MaybeHeapVec::Three(a) => { - std::mem::swap(&mut a[index], &mut value); - value - } - MaybeHeapVec::Four(a) => { - std::mem::swap(&mut a[index], &mut value); - value - } - MaybeHeapVec::Five(a) => { - std::mem::swap(&mut a[index], &mut value); - value - } - MaybeHeapVec::Spilled(a) => { - std::mem::swap(&mut a[index], &mut value); - value - } - } - } -} - -// TODO: This is an immutable struct, which can has sole ownership of the values -// underneath. Could be useful to separate mutable structs vs immutable structs, since -// that way we can safely access this stuff -pub(crate) enum ImmutableMaybeHeapVec { - Unit, - One(SteelVal), - Two(Rc<[SteelVal; 2]>), - Three(Rc<[SteelVal; 3]>), - Four(Rc<[SteelVal; 4]>), - Five(Rc<[SteelVal; 5]>), - Spilled(Rc<[SteelVal]>), -} - -impl ImmutableMaybeHeapVec { - pub fn from_slice(args: &[SteelVal]) -> Self { - match args { - [] => Self::Unit, - [one] => Self::One(one.clone()), - [one, two] => Self::Two(Rc::new([one.clone(), two.clone()])), - [one, two, three] => Self::Three(Rc::new([one.clone(), two.clone(), three.clone()])), - [one, two, three, four] => Self::Four(Rc::new([ - one.clone(), - two.clone(), - three.clone(), - four.clone(), - ])), - [one, two, three, four, five] => Self::Five(Rc::new([ - one.clone(), - two.clone(), - three.clone(), - four.clone(), - five.clone(), - ])), - _ => Self::Spilled(args.iter().cloned().collect()), - } - } - - #[inline(always)] - pub fn get(&self, index: usize) -> Option { - match self { - ImmutableMaybeHeapVec::Unit => None, - ImmutableMaybeHeapVec::One(v) => Some(v.clone()), - ImmutableMaybeHeapVec::Two(a) => Some(a[index].clone()), - ImmutableMaybeHeapVec::Three(a) => Some(a[index].clone()), - ImmutableMaybeHeapVec::Four(a) => Some(a[index].clone()), - ImmutableMaybeHeapVec::Five(a) => Some(a[index].clone()), - ImmutableMaybeHeapVec::Spilled(a) => Some(a[index].clone()), - } - } -} - impl UserDefinedStruct { fn new( // name: InternedString, diff --git a/crates/steel-parser/Cargo.toml b/crates/steel-parser/Cargo.toml index 0e767e317..77621dcee 100644 --- a/crates/steel-parser/Cargo.toml +++ b/crates/steel-parser/Cargo.toml @@ -10,7 +10,9 @@ description = "Parser for use within steel" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -# logos = "0.12.1" serde = { version = "1.0.152", features = ["derive", "rc"] } serde_derive = "1.0.152" num-bigint = { version = "0.4.0", features = ["serde"] } +lasso = { version = "0.7.2", features = ["multi-threaded", "serialize"] } +once_cell = "1.18.0" +pretty = "0.12.1" diff --git a/crates/steel-parser/src/ast.rs b/crates/steel-parser/src/ast.rs new file mode 100644 index 000000000..195cd5418 --- /dev/null +++ b/crates/steel-parser/src/ast.rs @@ -0,0 +1,2298 @@ +use crate::{ + parser::{ParseError, SyntaxObject}, + tokens::TokenType::{self}, +}; + +use std::{convert::TryFrom, fmt::Write, sync::atomic::Ordering}; + +use crate::tokens::MaybeBigInt; +use pretty::RcDoc; +use serde::{Deserialize, Serialize}; +use std::fmt; +use std::ops::Deref; + +use super::{ + interner::InternedString, + parser::{SyntaxObjectId, SYNTAX_OBJECT_ID}, + span::Span, +}; + +// TODO: Have this macro share the implementation crate wide +macro_rules! define_symbols { + ($($name:tt => $str:expr,) * ) => { + $( + pub static $name: once_cell::sync::Lazy = once_cell::sync::Lazy::new(|| $str.into()); + )* + }; +} + +define_symbols! { + UNREADABLE_MODULE_GET => "##__module-get", + STANDARD_MODULE_GET => "%module-get%", + PROVIDE => "provide", + DATUM_SYNTAX => "datum->syntax", + SYNTAX_SPAN => "#%syntax-span", + IF => "if", + DEFINE => "define", + LET => "let", + QUOTE =>"quote", + RETURN => "return!", + REQUIRE => "require", + SET => "set!", + PLAIN_LET => "%plain-let", + LAMBDA => "lambda", + LAMBDA_SYMBOL => "λ", + LAMBDA_FN => "fn", + BEGIN => "begin", + DOC_MACRO => "@doc", + REQUIRE_BUILTIN => "require-builtin", + REQUIRE_DYLIB => "#%require-dylib", + STRUCT_KEYWORD => "struct", + UNQUOTE => "unquote", + UNQUOTE_COMMA => "#%unquote-comma", + RAW_UNQUOTE => "#%unquote", + UNQUOTE_SPLICING => "unquote-splicing", + RAW_UNQUOTE_SPLICING => "#%unquote-splicing", + QUASIQUOTE => "quasiquote", + RAW_QUOTE => "#%quote", + QUASISYNTAX => "quasisyntax", + UNSYNTAX => "unsyntax", + RAW_UNSYNTAX => "#%unsyntax", + UNSYNTAX_SPLICING => "unsyntax-splicing", + RAW_UNSYNTAX_SPLICING => "#%unsyntax-splicing", + SYNTAX_QUOTE => "syntax", +} + +pub trait AstTools { + fn pretty_print(&self); +} + +impl AstTools for Vec { + fn pretty_print(&self) { + println!("{}", self.iter().map(|x| x.to_pretty(60)).join("\n\n")) + } +} + +impl AstTools for Vec<&ExprKind> { + fn pretty_print(&self) { + println!("{}", self.iter().map(|x| x.to_pretty(60)).join("\n\n")) + } +} + +impl AstTools for &mut Vec { + fn pretty_print(&self) { + println!("{}", self.iter().map(|x| x.to_pretty(60)).join("\n\n")) + } +} + +pub trait IteratorExtensions: Iterator { + fn join(&mut self, sep: &str) -> String + where + Self::Item: std::fmt::Display, + { + match self.next() { + None => String::new(), + Some(first_elt) => { + // estimate lower bound of capacity needed + let (lower, _) = self.size_hint(); + let mut result = String::with_capacity(sep.len() * lower); + write!(&mut result, "{}", first_elt).unwrap(); + self.for_each(|elt| { + result.push_str(sep); + write!(&mut result, "{}", elt).unwrap(); + }); + result + } + } + } +} + +impl IteratorExtensions for T where T: Iterator {} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub enum ExprKind { + Atom(Atom), + If(Box), + Let(Box), + Define(Box), + LambdaFunction(Box), + Begin(Begin), + Return(Box), + Quote(Box), + Macro(Macro), + SyntaxRules(SyntaxRules), + List(List), + Set(Box), + Require(Require), +} + +#[macro_export] +macro_rules! expr_list { + () => { $crate::ast::ExprKind::List($crate::ast::List::new(vec![])) }; + + ( $($x:expr),* ) => {{ + $crate::ast::ExprKind::List($crate::ast::List::new(vec![$( + $x, + ) *])) + }}; + + ( $($x:expr ,)* ) => {{ + $crate::ast::ExprKind::List($crate::ast::List::new(vec![$($x, )*])) + }}; +} + +impl ExprKind { + pub fn into_lambda_function(self) -> Option> { + if let ExprKind::LambdaFunction(func) = self { + Some(func) + } else { + None + } + } + + pub fn quoted_list() -> ExprKind { + ExprKind::Quote(Box::new(Quote::new( + Self::empty(), + SyntaxObject::default(TokenType::QuoteTick), + ))) + } + + pub fn empty() -> ExprKind { + ExprKind::List(List::new(Vec::new())) + } + + pub fn integer_literal(value: isize, span: Span) -> ExprKind { + ExprKind::Atom(crate::ast::Atom::new(SyntaxObject::new( + TokenType::IntegerLiteral(MaybeBigInt::Small(value)), + span, + ))) + } + + pub fn atom>(name: T) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::Identifier( + name.into(), + )))) + } + + pub fn ident(name: &str) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::Identifier( + name.into(), + )))) + } + + pub fn string_lit(input: String) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::StringLiteral( + input, + )))) + } + + pub fn bool_lit(b: bool) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::BooleanLiteral( + b, + )))) + } + + pub fn default_if(test: ExprKind, then: ExprKind, els: ExprKind) -> ExprKind { + ExprKind::If(Box::new(If::new( + test, + then, + els, + SyntaxObject::default(TokenType::If), + ))) + } + + pub fn into_atom_syntax_object(self) -> Option { + match self { + Self::Atom(Atom { syn }) => Some(syn), + _ => None, + } + } + + pub fn atom_syntax_object(&self) -> Option<&SyntaxObject> { + match self { + Self::Atom(Atom { syn }) => Some(syn), + _ => None, + } + } + + pub fn define_syntax_ident(&self) -> bool { + match self { + Self::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::DefineSyntax, + .. + }, + }) => true, + _ => false, + } + } + + pub fn atom_identifier_mut(&mut self) -> Option<&mut InternedString> { + match self { + Self::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(s), + .. + }, + }) => Some(s), + _ => None, + } + } + + pub fn lambda_function(&self) -> Option<&LambdaFunction> { + match self { + Self::LambdaFunction(l) => Some(l), + _ => None, + } + } + + pub fn atom_identifier_or_else E>( + &self, + err: F, + ) -> std::result::Result<&InternedString, E> { + match self { + Self::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(s), + .. + }, + }) => Ok(s), + _ => Err(err()), + } + } + + pub fn atom_identifier(&self) -> Option<&InternedString> { + match self { + Self::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(s), + .. + }, + }) => Some(s), + _ => None, + } + } + + pub fn string_literal(&self) -> Option<&str> { + match self { + Self::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::StringLiteral(s), + .. + }, + }) => Some(s), + _ => None, + } + } + + pub fn list(&self) -> Option<&List> { + if let ExprKind::List(l) = self { + Some(l) + } else { + None + } + } + + pub fn list_or_else E>(&self, err: F) -> std::result::Result<&List, E> { + match self { + Self::List(l) => Ok(l), + _ => Err(err()), + } + } + + pub fn list_mut_or_else E>( + &mut self, + err: F, + ) -> std::result::Result<&mut List, E> { + match self { + Self::List(l) => Ok(l), + _ => Err(err()), + } + } + + pub fn into_list_or_else E>(self, err: F) -> std::result::Result { + match self { + Self::List(l) => Ok(l), + _ => Err(err()), + } + } + + pub fn into_list(self) -> List { + if let ExprKind::List(l) = self { + l + } else { + panic!("Attempted to coerce a non list to a list"); + } + } + + pub fn unwrap_function(self) -> Option> { + if let ExprKind::LambdaFunction(l) = self { + Some(l) + } else { + None + } + } + + pub fn get_list(&self) -> Option<&List> { + if let ExprKind::List(l) = self { + Some(l) + } else { + None + } + } + + pub fn update_string_in_atom(&mut self, ident: InternedString) { + if let ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(ref mut s), + .. + }, + }) = self + { + *s = ident; + } + } +} + +pub trait ToDoc { + fn to_doc(&self) -> RcDoc<()>; +} + +impl ToDoc for ExprKind { + fn to_doc(&self) -> RcDoc<()> { + match self { + ExprKind::Atom(a) => a.to_doc(), + ExprKind::If(i) => i.to_doc(), + ExprKind::Define(d) => d.to_doc(), + ExprKind::LambdaFunction(l) => l.to_doc(), + ExprKind::Begin(b) => b.to_doc(), + ExprKind::Return(r) => r.to_doc(), + ExprKind::Let(l) => l.to_doc(), + ExprKind::Quote(q) => q.to_doc(), + ExprKind::Macro(m) => m.to_doc(), + ExprKind::SyntaxRules(s) => s.to_doc(), + ExprKind::List(l) => l.to_doc(), + ExprKind::Set(s) => s.to_doc(), + ExprKind::Require(r) => r.to_doc(), + } + } +} + +impl ExprKind { + pub fn to_pretty(&self, width: usize) -> String { + let mut w = Vec::new(); + self.to_doc().render(width, &mut w).unwrap(); + String::from_utf8(w).unwrap() + } +} + +impl fmt::Display for ExprKind { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + ExprKind::Atom(a) => write!(f, "{a}"), + ExprKind::If(i) => write!(f, "{i}"), + ExprKind::Define(d) => write!(f, "{d}"), + ExprKind::LambdaFunction(l) => write!(f, "{l}"), + ExprKind::Begin(b) => write!(f, "{b}"), + ExprKind::Return(r) => write!(f, "{r}"), + ExprKind::Let(l) => write!(f, "{l}"), + ExprKind::Quote(q) => write!(f, "{q}"), + ExprKind::Macro(m) => write!(f, "{m}"), + ExprKind::SyntaxRules(s) => write!(f, "{s}"), + ExprKind::List(l) => write!(f, "{l}"), + ExprKind::Set(s) => write!(f, "{s}"), + ExprKind::Require(r) => write!(f, "{r}"), + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Atom { + pub syn: SyntaxObject, +} + +impl Atom { + pub fn new(syn: SyntaxObject) -> Self { + Atom { syn } + } + + pub fn ident(&self) -> Option<&InternedString> { + if let TokenType::Identifier(ref ident) = self.syn.ty { + Some(ident) + } else { + None + } + } + + pub fn ident_mut(&mut self) -> Option<&mut InternedString> { + if let TokenType::Identifier(ref mut ident) = self.syn.ty { + Some(ident) + } else { + None + } + } +} + +impl fmt::Display for Atom { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.syn.ty) + } +} + +impl ToDoc for Atom { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text(self.syn.ty.to_string()) + } +} + +impl From for ExprKind { + fn from(val: Atom) -> Self { + ExprKind::Atom(val) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Let { + pub bindings: Vec<(ExprKind, ExprKind)>, + pub body_expr: ExprKind, + pub location: SyntaxObject, + pub syntax_object_id: usize, +} + +impl Let { + pub fn new( + bindings: Vec<(ExprKind, ExprKind)>, + body_expr: ExprKind, + location: SyntaxObject, + ) -> Self { + Let { + bindings, + body_expr, + location, + syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + } + } + + pub fn local_bindings(&self) -> impl Iterator { + self.bindings.iter().map(|x| &x.0) + } + + pub fn expression_arguments(&self) -> impl Iterator { + self.bindings.iter().map(|x| &x.1) + } +} + +impl fmt::Display for Let { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "(%plain-let ({}) {})", + self.bindings + .iter() + .map(|x| format!("({} {})", x.0, x.1)) + .join(" "), + self.body_expr + ) + } +} + +impl ToDoc for Let { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(%plain-let") + .append(RcDoc::space()) + .append(RcDoc::text("(")) + .append( + RcDoc::intersperse( + self.bindings.iter().map(|x| { + RcDoc::text("(") + .append(x.0.to_doc()) + .append(RcDoc::space()) + .append(x.1.to_doc()) + .append(RcDoc::text(")")) + }), + RcDoc::line(), + ) + .nest(2) + .group(), + ) + .append(RcDoc::text(")")) + .append(RcDoc::line()) + .append(self.body_expr.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } +} + +impl From for ExprKind { + fn from(val: Let) -> Self { + ExprKind::Let(Box::new(val)) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Set { + pub variable: ExprKind, + pub expr: ExprKind, + pub location: SyntaxObject, +} + +impl Set { + pub fn new(variable: ExprKind, expr: ExprKind, location: SyntaxObject) -> Self { + Set { + variable, + expr, + location, + } + } +} + +impl fmt::Display for Set { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(set! {} {})", self.variable, self.expr) + } +} + +impl ToDoc for Set { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(set!") + .append(RcDoc::space()) + .append(self.variable.to_doc()) + .append(RcDoc::line()) + .append(self.expr.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + .group() + } +} + +impl From for ExprKind { + fn from(val: Set) -> Self { + ExprKind::Set(Box::new(val)) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct If { + pub test_expr: ExprKind, + pub then_expr: ExprKind, + pub else_expr: ExprKind, + pub location: SyntaxObject, +} + +impl ToDoc for If { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(if") + .append(RcDoc::space()) + .append(self.test_expr.to_doc()) + .append(RcDoc::line()) + .append(self.then_expr.to_doc()) + .append(RcDoc::line()) + .append(self.else_expr.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + .group() + } +} + +impl fmt::Display for If { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "(if {} {} {})", + self.test_expr, self.then_expr, self.else_expr + ) + } +} + +impl If { + pub fn new( + test_expr: ExprKind, + then_expr: ExprKind, + else_expr: ExprKind, + location: SyntaxObject, + ) -> Self { + If { + test_expr, + then_expr, + else_expr, + location, + } + } +} + +impl From for ExprKind { + fn from(val: If) -> Self { + ExprKind::If(Box::new(val)) + } +} + +// Define normal +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Define { + // This could either be name + args + pub name: ExprKind, + pub body: ExprKind, + pub location: SyntaxObject, +} + +impl fmt::Display for Define { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(define {} {})", self.name, self.body) + } +} + +impl ToDoc for Define { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(define") + .append(RcDoc::space()) + .append(self.name.to_doc()) + .append(RcDoc::line()) + .append(self.body.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } +} + +impl Define { + pub fn new(name: ExprKind, body: ExprKind, location: SyntaxObject) -> Self { + Define { + name, + body, + location, + } + } + + pub fn is_an_alias_definition(&self) -> Option { + if let Some(atom) = self.body.atom_syntax_object() { + if let TokenType::Identifier(_) = atom.ty { + return Some(atom.syntax_object_id); + } + } + + None + } + + pub fn name_id(&self) -> Option { + self.name.atom_syntax_object().map(|x| x.syntax_object_id) + } +} + +impl From for ExprKind { + fn from(val: Define) -> Self { + ExprKind::Define(Box::new(val)) + } +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct LambdaFunction { + pub args: Vec, + pub body: ExprKind, + pub location: SyntaxObject, + pub rest: bool, + pub syntax_object_id: usize, +} + +impl Clone for LambdaFunction { + fn clone(&self) -> Self { + Self { + args: self.args.clone(), + body: self.body.clone(), + location: self.location.clone(), + rest: self.rest, + syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + } + } +} + +impl PartialEq for LambdaFunction { + fn eq(&self, other: &Self) -> bool { + self.args == other.args + && self.body == other.body + && self.location == other.location + && self.rest == other.rest + } +} + +impl fmt::Display for LambdaFunction { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "(lambda ({}) {})", + self.args.iter().map(|x| x.to_string()).join(" "), + self.body + ) + } +} + +impl ToDoc for LambdaFunction { + fn to_doc(&self) -> RcDoc<()> { + if self.rest && self.args.len() == 1 { + RcDoc::text("(λ") + .append(RcDoc::space()) + .append(self.args.first().unwrap().to_doc()) + .append(RcDoc::line()) + .append(self.body.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } else { + RcDoc::text("(λ") + .append(RcDoc::space()) + .append(RcDoc::text("(")) + .append( + RcDoc::intersperse(self.args.iter().map(|x| x.to_doc()), RcDoc::line()) + .nest(2) + .group(), + ) + .append(RcDoc::text(")")) + .append(RcDoc::line()) + .append(self.body.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } + } +} + +impl LambdaFunction { + pub fn new(args: Vec, body: ExprKind, location: SyntaxObject) -> Self { + LambdaFunction { + args, + body, + location, + rest: false, + syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + } + } + + pub fn new_with_rest_arg(args: Vec, body: ExprKind, location: SyntaxObject) -> Self { + LambdaFunction { + args, + body, + location, + rest: true, + syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + } + } + + pub fn new_maybe_rest( + args: Vec, + body: ExprKind, + location: SyntaxObject, + rest: bool, + ) -> Self { + LambdaFunction { + args, + body, + location, + rest, + syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + } + } + + pub fn arguments(&self) -> Option> { + self.args.iter().map(|x| x.atom_identifier()).collect() + } + + pub fn arguments_mut(&mut self) -> impl Iterator { + self.args.iter_mut().filter_map(|x| x.atom_identifier_mut()) + } +} + +impl From for ExprKind { + fn from(val: LambdaFunction) -> Self { + ExprKind::LambdaFunction(Box::new(val)) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Begin { + pub exprs: Vec, + pub location: SyntaxObject, +} + +impl fmt::Display for Begin { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(begin {})", self.exprs.iter().join(" ")) + } +} + +impl ToDoc for Begin { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(begin") + .append(RcDoc::line()) + .nest(5) + .append( + RcDoc::intersperse(self.exprs.iter().map(|x| x.to_doc()), RcDoc::line()) + .nest(5) + .group(), + ) + .append(RcDoc::text(")")) + .nest(1) + .group() + } +} + +impl Begin { + pub fn new(exprs: Vec, location: SyntaxObject) -> Self { + Begin { exprs, location } + } +} + +impl From for ExprKind { + fn from(val: Begin) -> Self { + ExprKind::Begin(val) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Return { + pub expr: ExprKind, + pub location: SyntaxObject, +} + +impl Return { + pub fn new(expr: ExprKind, location: SyntaxObject) -> Self { + Return { expr, location } + } +} + +impl ToDoc for Return { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(return") + .append(RcDoc::line()) + .append(self.expr.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } +} + +impl fmt::Display for Return { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(return! {})", self.expr) + } +} + +impl From for ExprKind { + fn from(val: Return) -> Self { + ExprKind::Return(Box::new(val)) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Require { + pub modules: Vec, + pub location: SyntaxObject, +} + +impl Require { + pub fn new(modules: Vec, location: SyntaxObject) -> Self { + Require { modules, location } + } +} + +impl ToDoc for Require { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(require") + .append(RcDoc::line()) + .append( + RcDoc::intersperse(self.modules.iter().map(|x| x.to_doc()), RcDoc::line()) + .nest(2) + .group(), + ) + .append(RcDoc::text(")")) + .nest(2) + } +} + +impl fmt::Display for Require { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(require {})", self.modules.iter().join(" ")) + } +} + +impl From for ExprKind { + fn from(val: Require) -> Self { + ExprKind::Require(val) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct List { + pub args: Vec, + pub syntax_object_id: usize, +} + +impl PartialEq for List { + fn eq(&self, other: &Self) -> bool { + self.args == other.args + } +} + +impl List { + pub fn new(args: Vec) -> Self { + List { + args, + syntax_object_id: SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed), + } + } + + pub fn is_empty(&self) -> bool { + self.args.is_empty() + } + + pub fn rest_mut(&mut self) -> Option<&mut [ExprKind]> { + self.args.split_first_mut().map(|x| x.1) + } + + pub fn first_ident_mut(&mut self) -> Option<&mut InternedString> { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(s), + .. + }, + })) = self.args.first_mut() + { + Some(s) + } else { + None + } + } + + pub fn is_require(&self) -> bool { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Require, + .. + }, + })) = self.args.first() + { + true + } else { + false + } + } + + pub fn is_define_syntax(&self) -> bool { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::DefineSyntax, + .. + }, + })) = self.args.first() + { + self.args.len() == 3 + } else { + false + } + } + + pub fn is_syntax_rules(&self) -> bool { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::SyntaxRules, + .. + }, + })) = self.args.first() + { + self.args.len() > 2 + } else { + false + } + } + + pub fn is_quote(&self) -> bool { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Quote, + .. + }, + })) = self.args.first() + { + true + } else { + false + } + } + + pub fn first_ident(&self) -> Option<&InternedString> { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(s), + .. + }, + })) = self.args.first() + { + Some(s) + } else { + None + } + } + + pub fn second_ident(&self) -> Option<&InternedString> { + if let Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(s), + .. + }, + })) = self.args.get(1) + { + Some(s) + } else { + None + } + } + + pub fn is_anonymous_function_call(&self) -> bool { + matches!(self.args.get(0), Some(ExprKind::LambdaFunction(_))) + } + + pub fn is_a_builtin_expr(&self) -> bool { + matches!(self.first_ident(), Some(func) if *func == *UNREADABLE_MODULE_GET || *func == *STANDARD_MODULE_GET) + } + + pub fn first_func_mut(&mut self) -> Option<&mut LambdaFunction> { + if let Some(ExprKind::LambdaFunction(l)) = self.args.get_mut(0) { + Some(l) + } else { + None + } + } + + pub fn first_func(&self) -> Option<&LambdaFunction> { + if let Some(ExprKind::LambdaFunction(l)) = self.args.get(0) { + Some(l) + } else { + None + } + } +} + +impl ToDoc for List { + fn to_doc(&self) -> RcDoc<()> { + if let Some(func) = self.first_func() { + let mut args_iter = self.args.iter(); + args_iter.next(); + + let bindings = func.args.iter().zip(args_iter); + + RcDoc::text("(let") + .append(RcDoc::space()) + .append(RcDoc::text("(")) + .append( + RcDoc::intersperse( + bindings.map(|x| { + RcDoc::text("(") + .append(x.0.to_doc()) + .append(RcDoc::space()) + .append(x.1.to_doc()) + .append(RcDoc::text(")")) + }), + RcDoc::line(), + ) + .nest(2) + .group(), + ) + .append(RcDoc::text(")")) + .append(RcDoc::line()) + .append(func.body.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } else { + RcDoc::text("(") + .append( + RcDoc::intersperse(self.args.iter().map(|x| x.to_doc()), RcDoc::line()) + .nest(1) + .group(), + ) + .append(RcDoc::text(")")) + .nest(2) + .group() + } + } +} + +impl fmt::Display for List { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "({})", self.args.iter().join(" ")) + } +} + +impl From for ExprKind { + fn from(val: List) -> Self { + ExprKind::List(val) + } +} + +impl Deref for List { + type Target = [ExprKind]; + + fn deref(&self) -> &[ExprKind] { + &self.args + } +} + +// and we'll implement IntoIterator +impl IntoIterator for List { + type Item = ExprKind; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.args.into_iter() + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Quote { + pub expr: ExprKind, + pub location: SyntaxObject, +} + +impl Quote { + pub fn new(expr: ExprKind, location: SyntaxObject) -> Self { + Quote { expr, location } + } +} + +impl ToDoc for Quote { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(quote") + .append(RcDoc::line()) + .append(self.expr.to_doc()) + .append(RcDoc::text(")")) + .nest(2) + } +} + +impl fmt::Display for Quote { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(quote {})", self.expr) + } +} + +impl From for ExprKind { + fn from(val: Quote) -> Self { + ExprKind::Quote(Box::new(val)) + } +} + +// TODO figure out how many fields a macro has +// put it into here nicely +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct Macro { + pub name: Box, + pub syntax_rules: SyntaxRules, + pub location: SyntaxObject, +} + +impl fmt::Display for Macro { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "(define-syntax {} {})", self.name, self.syntax_rules) + } +} + +impl ToDoc for Macro { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(define-syntax") + .append(RcDoc::line()) + .append(self.name.to_doc()) + .append(RcDoc::line()) + .append(self.syntax_rules.to_doc()) + .append(RcDoc::text(")")) + .nest(1) + .group() + } +} + +impl Macro { + pub fn new(name: ExprKind, syntax_rules: SyntaxRules, location: SyntaxObject) -> Self { + Macro { + name: Box::new(name), + syntax_rules, + location, + } + } +} + +impl From for ExprKind { + fn from(val: Macro) -> Self { + ExprKind::Macro(val) + } +} + +// TODO figure out a good mapping immediately to a macro that can be interpreted +// by the expander +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct SyntaxRules { + pub syntax: Vec, + pub patterns: Vec, + pub location: SyntaxObject, +} + +impl SyntaxRules { + pub fn new(syntax: Vec, patterns: Vec, location: SyntaxObject) -> Self { + SyntaxRules { + syntax, + patterns, + location, + } + } +} + +impl fmt::Display for SyntaxRules { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "(syntax-rules ({}) {})", + self.syntax.iter().map(|x| x.to_string()).join(" "), + self.patterns.iter().map(|x| x.to_string()).join("\n") + ) + } +} + +impl ToDoc for SyntaxRules { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("(syntax-rules") + .append(RcDoc::line()) + .append(RcDoc::text("(")) + .append( + RcDoc::intersperse(self.syntax.iter().map(|x| x.to_doc()), RcDoc::line()) + .nest(1) + .group(), + ) + .append(RcDoc::text(")")) + .append(RcDoc::line()) + .append( + RcDoc::intersperse(self.patterns.iter().map(|x| x.to_doc()), RcDoc::line()) + .nest(2) + .group(), + ) + .append(RcDoc::text(")")) + .nest(2) + } +} + +impl From for ExprKind { + fn from(val: SyntaxRules) -> Self { + ExprKind::SyntaxRules(val) + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct PatternPair { + pub pattern: ExprKind, + pub body: ExprKind, +} + +impl PatternPair { + pub fn new(pattern: ExprKind, body: ExprKind) -> Self { + PatternPair { pattern, body } + } +} + +impl ToDoc for PatternPair { + fn to_doc(&self) -> RcDoc<()> { + RcDoc::text("[") + .append(self.pattern.to_doc()) + .append(RcDoc::line()) + .append(self.body.to_doc()) + .append(RcDoc::text("]")) + .nest(1) + .group() + } +} + +impl fmt::Display for PatternPair { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "[{}\n{}]", self.pattern, self.body) + } +} + +#[inline] +pub(crate) fn parse_if( + mut value_iter: I, + syn: SyntaxObject, +) -> std::result::Result +where + I: Iterator, +{ + // let mut value_iter = value.into_iter(); + value_iter.next(); + + let ret_value = If::new( + value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "if expects a test condition, found none".to_string(), + syn.span, + None, + ) + })?, + value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "if expects a then condition, found none".to_string(), + syn.span, + None, + ) + })?, + // Replace else condition with just a void if its not found! + value_iter + .next() + .unwrap_or_else(|| ExprKind::ident("#%prim.void")), + // ok_or_else(|| { + // ParseError::SyntaxError( + // "if expects an else condition, found none".to_string(), + // syn.span, + // None, + // ) + // })?, + syn.clone(), + ) + .into(); + + if value_iter.next().is_some() { + Err(ParseError::SyntaxError( + "if takes only 3 expressions".to_string(), + syn.span, + None, + )) + } else { + Ok(ret_value) + } +} + +#[inline] +pub(crate) fn parse_define( + mut value_iter: I, + syn: SyntaxObject, +) -> std::result::Result +where + I: Iterator, +{ + value_iter.next(); + + match value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "define expects an identifier, found none".to_string(), + syn.span, + None, + ) + })? { + // TODO maybe add implicit begin here + // maybe do it later, not sure + ExprKind::List(l) => { + let name_ref = l.args.first().ok_or_else(|| { + ParseError::SyntaxError( + "define expected a function name, found none".to_string(), + syn.span, + None, + ) + })?; + + if let ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Identifier(datum_syntax), + .. + }, + }) = name_ref + { + if *datum_syntax == *DATUM_SYNTAX { + return Ok(ExprKind::Define(Box::new(Define::new( + ExprKind::List(List::new(l.args)), + { + let v = value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "define statement expected a body, found none".to_string(), + syn.span, + None, + ) + })?; + if value_iter.next().is_some() { + return Err(ParseError::SyntaxError( + "Define expected only one expression after the identifier" + .to_string(), + syn.span, + None, + )); + } + v + }, + syn, + )))); + } + } + + let mut args = l.args.into_iter(); + + let name = args.next().ok_or_else(|| { + ParseError::SyntaxError( + "define expected a function name, found none".to_string(), + syn.span, + None, + ) + })?; + + let args = args.collect(); + + let body_exprs: Vec<_> = value_iter.collect(); + + if body_exprs.is_empty() { + return Err(ParseError::SyntaxError( + "Function body cannot be empty".to_string(), + syn.span, + None, + )); + } + + let body = if body_exprs.len() == 1 { + body_exprs[0].clone() + } else { + ExprKind::Begin(Begin::new( + body_exprs, + SyntaxObject::default(TokenType::Begin), + )) + }; + + let lambda = ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + args, + body, + SyntaxObject::new(TokenType::Lambda, syn.span), + ))); + + Ok(ExprKind::Define(Box::new(Define::new(name, lambda, syn)))) + } + ExprKind::Atom(a) => Ok(ExprKind::Define(Box::new(Define::new( + ExprKind::Atom(a), + { + let v = value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "define statement expected a body, found none".to_string(), + syn.span, + None, + ) + })?; + if value_iter.next().is_some() { + return Err(ParseError::SyntaxError( + "Define expected only one expression after the identifier".to_string(), + syn.span, + None, + )); + } + v + }, + syn, + )))), + + _ => Err(ParseError::SyntaxError( + "Define expects either an identifier or a list with the function name and arguments" + .to_string(), + syn.span, + None, + )), + } +} + +#[inline] +pub(crate) fn parse_new_let( + mut value_iter: I, + syn: SyntaxObject, +) -> std::result::Result +where + I: Iterator, +{ + value_iter.next(); + + let let_pairs = if let ExprKind::List(l) = value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "let expected a list of variable bindings pairs in the second position, found none" + .to_string(), + syn.span, + None, + ) + })? { + l.args + } else { + return Err(ParseError::SyntaxError( + "let expects a list of variable bindings pairs in the second position".to_string(), + syn.span, + None, + )); + }; + + let body_exprs: Vec<_> = value_iter.collect(); + + if body_exprs.is_empty() { + return Err(ParseError::SyntaxError( + "let expects an expression, found none".to_string(), + syn.span, + None, + )); + } + + let body = if body_exprs.len() == 1 { + body_exprs[0].clone() + } else { + ExprKind::Begin(Begin::new( + body_exprs, + SyntaxObject::default(TokenType::Begin), + )) + }; + + let mut pairs = Vec::with_capacity(let_pairs.len()); + + for pair in let_pairs { + if let ExprKind::List(l) = pair { + let pair = l.args; + + if pair.len() != 2 { + return Err(ParseError::SyntaxError( + format!("let expected a list of variable binding pairs, found a pair with length {}", + pair.len()), + syn.span, None + )); + } + + let mut iter = pair.into_iter(); + + let identifier = iter.next().unwrap(); + let application_arg = iter.next().unwrap(); + pairs.push((identifier, application_arg)) + } else { + return Err(ParseError::SyntaxError( + "let expected a list of variable binding pairs".to_string(), + syn.span, + None, + )); + } + } + + Ok(ExprKind::Let(Let::new(pairs, body, syn).into())) +} + +#[inline] +fn parse_named_let( + mut value_iter: I, + syn: SyntaxObject, + name: ExprKind, +) -> std::result::Result +where + I: Iterator, +{ + let pairs = if let ExprKind::List(l) = value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "named let expects a list of argument id and init expr pairs, found none".to_string(), + syn.span, + None, + ) + })? { + l.args + } else { + return Err(ParseError::SyntaxError( + "named let expects a list of variable bindings pairs in the second position" + .to_string(), + syn.span, + None, + )); + }; + + let body_exprs: Vec<_> = value_iter.collect(); + + if body_exprs.is_empty() { + return Err(ParseError::SyntaxError( + "let expects an expression, found none".to_string(), + syn.span, + None, + )); + } + + let body = if body_exprs.len() == 1 { + body_exprs[0].clone() + } else { + ExprKind::Begin(Begin::new( + body_exprs, + SyntaxObject::default(TokenType::Begin), + )) + }; + + let mut arguments = Vec::with_capacity(pairs.len()); + + // insert args at the end + // put the function in the inside + let mut application_args = Vec::with_capacity(pairs.len()); + + for pair in pairs { + if let ExprKind::List(l) = pair { + let pair = l.args; + + if pair.len() != 2 { + return Err(ParseError::SyntaxError( + format!("let expected a list of variable binding pairs, found a pair with length {}", + pair.len()), + syn.span, None + )); + } + + let identifier = pair[0].clone(); + let application_arg = pair[1].clone(); + + arguments.push(identifier); + application_args.push(application_arg); + } else { + return Err(ParseError::SyntaxError( + "let expected a list of variable binding pairs".to_string(), + syn.span, + None, + )); + } + } + + // This is the body of the define + let function: ExprKind = LambdaFunction::new(arguments, body, syn.clone()).into(); + + let define: ExprKind = Define::new(name.clone(), function, syn.clone()).into(); + + let application: ExprKind = { + let mut application = vec![name]; + application.append(&mut application_args); + List::new(application).into() + }; + + let begin = ExprKind::Begin(Begin::new(vec![define, application], syn.clone())); + + // Wrap the whole thing inside of an empty function application, to create a new scope + + Ok(List::new(vec![LambdaFunction::new(vec![], begin, syn).into()]).into()) +} + +#[inline] +pub(crate) fn parse_let( + mut value_iter: I, + syn: SyntaxObject, +) -> std::result::Result +where + I: Iterator, +{ + value_iter.next(); + + let let_pairs = match value_iter.next().ok_or_else(|| { + ParseError::SyntaxError( + "let expected a list of variable bindings pairs in the second position, found none" + .to_string(), + syn.span, + None, + ) + })? { + // Standard let + ExprKind::List(l) => l.args, + // Named let + name @ ExprKind::Atom(_) => return parse_named_let(value_iter, syn, name), + _ => { + return Err(ParseError::SyntaxError( + "let expects a list of variable bindings pairs in the second position".to_string(), + syn.span, + None, + )); + } + }; + + let body_exprs: Vec<_> = value_iter.collect(); + + if body_exprs.is_empty() { + return Err(ParseError::SyntaxError( + "let expects an expression, found none".to_string(), + syn.span, + None, + )); + } + + let body = if body_exprs.len() == 1 { + body_exprs[0].clone() + } else { + ExprKind::Begin(Begin::new( + body_exprs, + SyntaxObject::default(TokenType::Begin), + )) + }; + + let mut arguments = Vec::with_capacity(let_pairs.len()); + + // insert args at the end + // put the function in the inside + let mut application_args = Vec::with_capacity(let_pairs.len()); + + for pair in let_pairs { + if let ExprKind::List(l) = pair { + let pair = l.args; + + if pair.len() != 2 { + return Err(ParseError::SyntaxError( + format!("let expected a list of variable binding pairs, found a pair with length {}", + pair.len()), + syn.span, None + )); + } + + let identifier = pair[0].clone(); + let application_arg = pair[1].clone(); + + arguments.push(identifier); + application_args.push(application_arg); + } else { + return Err(ParseError::SyntaxError( + "let expected a list of variable binding pairs".to_string(), + syn.span, + None, + )); + } + } + + let mut function: Vec = vec![LambdaFunction::new(arguments, body, syn).into()]; + + function.append(&mut application_args); + + Ok(ExprKind::List(List::new(function))) +} + +#[inline] +pub(crate) fn parse_single_argument( + mut value_iter: I, + syn: SyntaxObject, + name: &'static str, + constructor: fn(ExprKind, SyntaxObject) -> ExprKind, +) -> Result +where + I: Iterator, +{ + value_iter.next(); + + let func = value_iter.next().ok_or_else(|| { + ParseError::ArityMismatch( + format!("{name} expected one argument, found none"), + syn.span, + None, + ) + })?; + + if value_iter.next().is_some() { + Err(ParseError::SyntaxError( + format!("{name} expects only one argument"), + syn.span, + None, + )) + } else { + Ok(constructor(func, syn)) + } +} + +impl TryFrom> for ExprKind { + type Error = ParseError; + fn try_from(value: Vec) -> std::result::Result { + // let mut value = value.into_iter().peekable(); + + // TODO -> get rid of this clone on the first value + if let Some(f) = value.first().cloned() { + match f { + ExprKind::Atom(a) => { + // let value = value.into_iter(); + + match &a.syn.ty { + // Have this also match on the first argument being a TokenType::Identifier("if") + // Do the same for the rest of the arguments + TokenType::If => parse_if(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *IF => { + parse_if(value.into_iter(), a.syn.clone()) + } + + TokenType::Define => parse_define(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *DEFINE => { + parse_define(value.into_iter(), a.syn.clone()) + } + + TokenType::Let => parse_let(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *LET => { + parse_let(value.into_iter(), a.syn.clone()) + } + + // TODO: Deprecate + TokenType::TestLet => parse_new_let(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *PLAIN_LET => { + parse_new_let(value.into_iter(), a.syn.clone()) + } + + TokenType::Quote => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "quote", + |expr, syn| Quote::new(expr, syn).into(), + ), + TokenType::Identifier(expr) if *expr == *QUOTE => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "quote", + |expr, syn| Quote::new(expr, syn).into(), + ), + + TokenType::Return => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "return!", + |expr, syn| Return::new(expr, syn).into(), + ), + TokenType::Identifier(expr) if *expr == *RETURN => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "return!", + |expr, syn| Return::new(expr, syn).into(), + ), + + TokenType::Require => parse_require(&a, value), + TokenType::Identifier(expr) if *expr == *REQUIRE => { + parse_require(&a, value) + } + + TokenType::Set => parse_set(&a, value), + TokenType::Identifier(expr) if *expr == *SET => parse_set(&a, value), + + TokenType::Begin => parse_begin(&a, value), + TokenType::Identifier(expr) if *expr == *BEGIN => parse_begin(&a, value), + + TokenType::Lambda => parse_lambda(&a, value), + TokenType::Identifier(expr) + if *expr == *LAMBDA + || *expr == *LAMBDA_FN + || *expr == *LAMBDA_SYMBOL => + { + parse_lambda(&a, value) + } + + TokenType::DefineSyntax => { + let syn = a.syn.clone(); + + if value.len() < 3 { + return Err(ParseError::SyntaxError( + format!("define-syntax expects 2 arguments - the name of the macro and the syntax-rules, found {}", value.len()), syn.span, None + )); + } + + // println!("{}", value.iter().map(|x| x.to_pretty(60)).join("\n\n")); + + let mut value_iter = value.into_iter(); + value_iter.next(); + + let name = value_iter.next().unwrap(); + + let syntax = value_iter.next(); + + // println!("{:?}", syntax); + + let syntax_rules = if let Some(ExprKind::SyntaxRules(s)) = syntax { + s + } else { + return Err(ParseError::SyntaxError( + "define-syntax expected a syntax-rules object".to_string(), + syn.span, + None, + )); + }; + + Ok(ExprKind::Macro(Macro::new(name, syntax_rules, syn))) + } + TokenType::SyntaxRules => { + let syn = a.syn.clone(); + + if value.len() < 3 { + return Err(ParseError::SyntaxError( + format!("syntax-rules expects a list of introduced syntax, and at least one pattern-body pair, found {} arguments", value.len()), syn.span, None + )); + } + + let mut value_iter = value.into_iter(); + value_iter.next(); + + let syntax_vec = if let Some(ExprKind::List(l)) = value_iter.next() { + l.args + } else { + return Err(ParseError::SyntaxError( + "syntax-rules expects a list of new syntax forms used in the macro".to_string(), syn.span, None)); + }; + + let mut pairs = Vec::new(); + let rest: Vec<_> = value_iter.collect(); + + for pair in rest { + if let ExprKind::List(l) = pair { + if l.args.len() != 2 { + return Err(ParseError::SyntaxError( + "syntax-rules requires only one pattern to one body" + .to_string(), + syn.span, + None, + )); + } + + let mut pair_iter = l.args.into_iter(); + let pair_object = PatternPair::new( + pair_iter.next().unwrap(), + pair_iter.next().unwrap(), + ); + pairs.push(pair_object); + } else { + return Err(ParseError::SyntaxError( + "syntax-rules requires pattern to expressions to be in a list".to_string(), syn.span, None + )); + } + } + + Ok(ExprKind::SyntaxRules(SyntaxRules::new( + syntax_vec, pairs, syn, + ))) + } + _ => Ok(ExprKind::List(List::new(value))), + } + } + _ => Ok(ExprKind::List(List::new(value))), + } + } else { + Ok(ExprKind::List(List::new(vec![]))) + } + } +} + +pub(crate) fn parse_lambda(a: &Atom, value: Vec) -> Result { + let syn = a.syn.clone(); + if value.len() < 3 { + return Err(ParseError::SyntaxError( + format!( + "lambda expected at least 2 arguments - the bindings list and one or more expressions, found {} instead", + value.len() + ), + syn.span, None + )); + } + let mut value_iter = value.into_iter(); + value_iter.next(); + let arguments = value_iter.next(); + match arguments { + Some(ExprKind::List(l)) => { + let args = l.args; + + for arg in &args { + if let ExprKind::Atom(_) = arg { + continue; + } else { + return Err(ParseError::SyntaxError( + format!( + "lambda function expects a list of identifiers, found: {}", + List::new(args) + ), + syn.span, + None, + )); + } + } + + let body_exprs: Vec<_> = value_iter.collect(); + + let body = if body_exprs.len() == 1 { + body_exprs.into_iter().next().unwrap() + } else { + ExprKind::Begin(Begin::new( + body_exprs, + SyntaxObject::default(TokenType::Begin), + )) + }; + + Ok(ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + args, body, syn, + )))) + } + Some(ExprKind::Atom(a)) => { + let body_exprs: Vec<_> = value_iter.collect(); + + let body = if body_exprs.len() == 1 { + body_exprs.into_iter().next().unwrap() + } else { + ExprKind::Begin(Begin::new( + body_exprs, + SyntaxObject::default(TokenType::Begin), + )) + }; + + // (lambda x ...) => x is a rest arg, becomes a list at run time + Ok(ExprKind::LambdaFunction(Box::new( + LambdaFunction::new_with_rest_arg(vec![ExprKind::Atom(a)], body, syn), + ))) + } + _ => { + // TODO -> handle case like + // (lambda x 10) <- where x is immediately bound to be a rest arg + // This should be fairly trivial in this case since we can just put the + // first thing into a vec for the lambda node + Err(ParseError::SyntaxError( + format!("lambda function expected a list of identifiers, found: {arguments:?}"), + syn.span, + None, + )) + } + } +} + +pub(crate) fn parse_set(a: &Atom, value: Vec) -> Result { + let syn = a.syn.clone(); + if value.len() != 3 { + return Err(ParseError::ArityMismatch( + "set! expects an identifier and an expression".to_string(), + syn.span, + None, + )); + } + let mut value_iter = value.into_iter(); + value_iter.next(); + let identifier = value_iter.next().unwrap(); + let expression = value_iter.next().unwrap(); + Ok(ExprKind::Set(Box::new(Set::new( + identifier, expression, syn, + )))) +} + +pub(crate) fn parse_require(a: &Atom, value: Vec) -> Result { + let syn = a.syn.clone(); + if value.len() < 2 { + return Err(ParseError::ArityMismatch( + "require expects at least one identifier or string".to_string(), + syn.span, + None, + )); + } + let mut value_iter = value.into_iter(); + value_iter.next(); + let expressions = value_iter + .map(|x| { + match &x { + ExprKind::Atom(_) | ExprKind::List(_) => Ok(x), + _ => Err(ParseError::SyntaxError( + "require expects atoms".to_string(), + syn.span, + None, + )), + } + + // if let ExprKind::Atom(a) = x { + // Ok(a) + // } else { + + // } + }) + .collect::, ParseError>>()?; + Ok(ExprKind::Require(Require::new(expressions, syn))) +} + +pub(crate) fn parse_begin(a: &Atom, value: Vec) -> Result { + let syn = a.syn.clone(); + let mut value_iter = value.into_iter(); + value_iter.next(); + Ok(ExprKind::Begin(Begin::new(value_iter.collect(), syn))) +} + +#[cfg(test)] +mod display_tests { + + use super::*; + use crate::parser::{Parser, Result}; + + fn parse(expr: &str) -> ExprKind { + let a: Result> = Parser::new(expr, None).collect(); + + a.unwrap()[0].clone() + } + + #[test] + fn display_lambda_quote() { + let expression = "(lambda (x) (quote x))"; + let parsed_expr = parse(expression); + let expected = "(lambda (x) (quote x))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_list() { + let expression = "(list 1 2 3 4)"; + let parsed_expr = parse(expression); + let expected = "(list 1 2 3 4)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_lambda() { + let expression = "(lambda (x) (+ x 10))"; + let parsed_expr = parse(expression); + let expected = "(lambda (x) (+ x 10))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_set() { + let expression = "(set! x 10)"; + let parsed_expr = parse(expression); + let expected = "(set! x 10)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_panic() { + let expression = "(panic! 12345)"; + let parsed_expr = parse(expression); + let expected = "(panic! 12345)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_begin() { + let expression = "(begin 1 2 3 4 5)"; + let parsed_expr = parse(expression); + let expected = "(begin 1 2 3 4 5)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_define_normal() { + let expression = "(define a 10)"; + let parsed_expr = parse(expression); + let expected = "(define a 10)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_define_function() { + let expression = "(define (applesauce x y z) (+ x y z))"; + let parsed_expr = parse(expression); + let expected = "(define applesauce (lambda (x y z) (+ x y z)))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_let() { + let expression = "(let ((x 10)) (+ x 10))"; + let parsed_expr = parse(expression); + let expected = "((lambda (x) (+ x 10)) 10)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_apply() { + let expression = "(apply + (list 1 2 3 4))"; + let parsed_expr = parse(expression); + let expected = "(apply + (list 1 2 3 4))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_transduce() { + let expression = "(transduce 1 2 3 4)"; + let parsed_expr = parse(expression); + let expected = "(transduce 1 2 3 4)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_execute_two_args() { + let expression = "(execute 1 2)"; + let parsed_expr = parse(expression); + let expected = "(execute 1 2)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_execute_three_args() { + let expression = "(execute 1 2 3)"; + let parsed_expr = parse(expression); + let expected = "(execute 1 2 3)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_if() { + let expression = "(if 1 2 3)"; + let parsed_expr = parse(expression); + let expected = "(if 1 2 3)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_quote() { + let expression = "'(1 2 3 4)"; + let parsed_expr = parse(expression); + let expected = "(quote (1 2 3 4))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_read() { + let expression = "(read '(1 2 3 4))"; + let parsed_expr = parse(expression); + let expected = "(read (quote (1 2 3 4)))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_return() { + let expression = "(return! 10)"; + let parsed_expr = parse(expression); + let expected = "(return! 10)"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_struct() { + let expression = "(struct Apple (a b c))"; + let parsed_expr = parse(expression); + let expected = "(struct Apple (a b c))"; + assert_eq!(parsed_expr.to_string(), expected); + } + + #[test] + fn display_eval() { + let expression = "(eval 'a)"; + let parsed_expr = parse(expression); + let expected = "(eval (quote a))"; + assert_eq!(parsed_expr.to_string(), expected); + } +} + +#[cfg(test)] +mod pretty_print_tests { + use super::*; + use crate::parser::{Parser, Result}; + + fn parse(expr: &str) -> ExprKind { + let a: Result> = Parser::new(expr, None).collect(); + + a.unwrap()[0].clone() + } + + #[test] + fn pretty_set() { + let expression = r#" + (define test-function + (lambda (a b c) + (begin + (set! bananas 10) + (if applesauce 100 #f) + (if applesauce 100 (if applesauce 100 #f)))))"#; + let parsed_expr = parse(expression); + let _output = parsed_expr.to_pretty(45); + + assert!(true) + } +} diff --git a/crates/steel-parser/src/interner.rs b/crates/steel-parser/src/interner.rs new file mode 100644 index 000000000..5c8d66a9b --- /dev/null +++ b/crates/steel-parser/src/interner.rs @@ -0,0 +1,151 @@ +use lasso::Key; +use lasso::Spur; +use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; +use std::{fmt, sync::Arc}; + +// TODO: Serialize and Deserialize should resolve() -> Otherwise we're in for deep trouble +// trying to serialize and deserialize this +#[derive(Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Hash)] +#[repr(transparent)] +pub struct InternedString(Spur); + +impl Serialize for InternedString { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + self.resolve().serialize(serializer) + } +} + +impl<'de> Deserialize<'de> for InternedString { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let key = <&str>::deserialize(deserializer)?; + + Ok(InternedString::from(key)) + } +} + +impl InternedString { + pub fn from_static(ident: &'static str) -> Self { + Self( + INTERNER + .get_or_init(|| Arc::new(ThreadedRodeo::new())) + .get_or_intern_static(ident), + ) + } + + pub fn from_string(ident: String) -> Self { + Self( + INTERNER + .get_or_init(|| Arc::new(ThreadedRodeo::new())) + .get_or_intern(ident), + ) + } + + pub fn new(key: usize) -> Self { + Self(Spur::try_from_usize(key).unwrap()) + } + + pub fn get(self) -> Spur { + self.0 + } + + pub fn try_get(ident: &str) -> Option { + INTERNER.get().unwrap().get(ident).map(InternedString) + } + + #[doc(hidden)] + pub fn as_u32(self) -> u32 { + self.get().into_usize() as u32 + } + + pub fn resolve(&self) -> &str { + resolve(&self.0) + } +} + +impl From<&str> for InternedString { + fn from(ident: &str) -> Self { + Self( + INTERNER + .get_or_init(|| Arc::new(ThreadedRodeo::new())) + .get_or_intern(ident), + ) + } +} + +impl From for InternedString { + fn from(ident: String) -> Self { + Self::from_string(ident) + } +} + +impl From for InternedString { + fn from(spur: Spur) -> Self { + Self(spur) + } +} + +impl fmt::Debug for InternedString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.get().into_usize()) + } +} + +impl fmt::Display for InternedString { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.resolve()) + } +} + +// impl Serialize for InternedString { +// fn serialize(&self, serializer: S) -> Result +// where +// S: Serializer, +// { +// serializer.serialize_str(self.resolve()) +// } +// } + +use lasso::ThreadedRodeo; + +static INTERNER: OnceCell> = OnceCell::new(); + +pub fn take_interner() -> Arc { + Arc::clone(INTERNER.get().unwrap()) +} + +pub fn initialize_with(interner: Arc) -> Result<(), Arc> { + INTERNER.set(interner) +} + +pub fn get_interner() -> Option<&'static Arc> { + INTERNER.get() +} + +pub fn add_interner(interner: Arc) { + let guard = INTERNER.get().unwrap(); + + for key in interner.strings() { + guard.get_or_intern(key); + } +} + +#[test] +fn test_initialization() { + INTERNER.get_or_init(|| Arc::new(ThreadedRodeo::new())); + let key = INTERNER.get().unwrap().get_or_intern_static("hello world"); + + let resolved_string = INTERNER.get().unwrap().resolve(&key); + + println!("resolved string: {resolved_string:?}"); +} + +fn resolve(key: &Spur) -> &str { + INTERNER.get().unwrap().resolve(key) +} diff --git a/crates/steel-parser/src/lib.rs b/crates/steel-parser/src/lib.rs index 915008d5b..ded4b447c 100644 --- a/crates/steel-parser/src/lib.rs +++ b/crates/steel-parser/src/lib.rs @@ -1,3 +1,5 @@ +pub mod ast; +pub mod interner; pub mod lexer; pub mod parser; pub mod span; diff --git a/crates/steel-parser/src/parser.rs b/crates/steel-parser/src/parser.rs index 51be859a2..6ddb75331 100644 --- a/crates/steel-parser/src/parser.rs +++ b/crates/steel-parser/src/parser.rs @@ -1,7 +1,2413 @@ +use std::{ + path::PathBuf, + rc::Rc, + result, + sync::atomic::{AtomicUsize, Ordering}, +}; + use serde::{Deserialize, Serialize}; +use crate::{ + ast::{ + self, parse_begin, parse_define, parse_if, parse_lambda, parse_let, parse_new_let, + parse_require, parse_set, parse_single_argument, Atom, ExprKind, List, Macro, PatternPair, + SyntaxRules, BEGIN, DEFINE, IF, LAMBDA, LAMBDA_FN, LAMBDA_SYMBOL, LET, PLAIN_LET, + QUASIQUOTE, QUOTE, RAW_UNQUOTE, RAW_UNQUOTE_SPLICING, REQUIRE, RETURN, SET, UNQUOTE, + UNQUOTE_SPLICING, + }, + interner::InternedString, + lexer::{OwnedTokenStream, ToOwnedString, TokenStream}, + span::Span, + tokens::{Token, TokenType}, +}; + #[derive( Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, )] #[repr(C)] pub struct SourceId(pub usize); + +// TODO: Fix the visibility here +pub static SYNTAX_OBJECT_ID: AtomicUsize = AtomicUsize::new(0); + +#[derive( + Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, +)] +pub struct SyntaxObjectId(pub usize); + +impl SyntaxObjectId { + #[inline] + pub fn fresh() -> Self { + SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)) + } +} + +impl From for usize { + fn from(value: SyntaxObjectId) -> Self { + value.0 + } +} + +impl std::fmt::Display for SyntaxObjectId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self:?}") + } +} + +#[derive( + Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, +)] +pub struct ListId(usize); + +#[derive( + Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Default, Debug, Ord, PartialOrd, +)] +pub struct FunctionId(usize); + +/// A syntax object that can hold anything as the syntax +/// In this case, we're using the token type emitted by logos +/// +/// This should open the door to interning our strings to make +/// parsing (and optimizations later) faster +#[derive(Serialize, Deserialize)] +pub struct RawSyntaxObject { + pub ty: T, + pub span: Span, + pub syntax_object_id: SyntaxObjectId, +} + +impl Clone for RawSyntaxObject { + fn clone(&self) -> Self { + Self { + ty: self.ty.clone(), + span: self.span, + syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), + } + } +} + +impl std::fmt::Debug for RawSyntaxObject { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RawSyntaxObject") + .field("ty", &self.ty) + .field("span", &self.span) + .finish() + } +} + +// Implementing hash here just on the token type - we dont want the span included +// For determining the hash here +impl std::hash::Hash for RawSyntaxObject { + fn hash(&self, state: &mut H) { + self.ty.hash(state); + self.span.hash(state); + } +} + +pub type SyntaxObject = RawSyntaxObject>; + +impl PartialEq for SyntaxObject { + fn eq(&self, other: &Self) -> bool { + self.ty == other.ty + } +} + +impl SyntaxObject { + pub fn new(ty: TokenType, span: Span) -> Self { + SyntaxObject { + ty, + span, + // source: None, + syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), + } + } + + pub fn default(ty: TokenType) -> Self { + SyntaxObject { + ty, + span: Span::new(0, 0, None), + // source: None, + syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), + } + } + + pub fn set_span(&mut self, span: Span) { + self.span = span + } + + pub fn from_token_with_source( + val: &Token<'_, InternedString>, + _source: &Option>, + ) -> Self { + SyntaxObject { + ty: val.ty.clone(), + span: val.span, + // source: source.as_ref().map(Rc::clone), + syntax_object_id: SyntaxObjectId(SYNTAX_OBJECT_ID.fetch_add(1, Ordering::Relaxed)), + } + } +} + +impl From<&Token<'_, InternedString>> for SyntaxObject { + fn from(val: &Token<'_, InternedString>) -> SyntaxObject { + SyntaxObject::new(val.ty.clone(), val.span) + } +} + +#[derive(Clone, Debug, PartialEq)] +pub enum ParseError { + Unexpected(TokenType, Option>), + UnexpectedEOF(Option>), + UnexpectedChar(char, Span, Option>), + IncompleteString(String, Span, Option>), + SyntaxError(String, Span, Option>), + ArityMismatch(String, Span, Option>), +} + +impl std::fmt::Display for ParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ParseError::Unexpected(l, _) => write!(f, "Parse: Unexpected token: {:?}", l), + ParseError::UnexpectedEOF(_) => write!(f, "Parse: Unexpected EOF"), + ParseError::UnexpectedChar(l, _, _) => { + write!(f, "Parse: Unexpected character: {:?}", l) + } + ParseError::IncompleteString(l, _, _) => write!(f, "Parse: Incomplete String: {}", l), + ParseError::SyntaxError(l, _, _) => write!(f, "Parse: Syntax Error: {}", l), + ParseError::ArityMismatch(l, _, _) => write!(f, "Parse: Arity mismatch: {}", l), + } + } +} + +impl std::error::Error for ParseError {} + +impl ParseError { + pub fn span(&self) -> Option { + match self { + // ParseError::TokenError(_) => None, + ParseError::Unexpected(_, _) => None, + ParseError::UnexpectedEOF(_) => None, + ParseError::UnexpectedChar(_, s, _) => Some(*s), + ParseError::IncompleteString(_, s, _) => Some(*s), + ParseError::SyntaxError(_, s, _) => Some(*s), + ParseError::ArityMismatch(_, s, _) => Some(*s), + } + } + + pub fn set_source(self, source: Option>) -> Self { + use ParseError::*; + match self { + ParseError::Unexpected(l, _) => Unexpected(l, source), + ParseError::UnexpectedEOF(_) => UnexpectedEOF(source), + ParseError::UnexpectedChar(l, s, _) => UnexpectedChar(l, s, source), + ParseError::IncompleteString(l, s, _) => IncompleteString(l, s, source), + ParseError::SyntaxError(l, s, _) => SyntaxError(l, s, source), + ParseError::ArityMismatch(l, s, _) => ArityMismatch(l, s, source), + } + } +} + +pub struct InternString; + +impl ToOwnedString for InternString { + fn own(&self, s: &str) -> InternedString { + s.into() + } +} + +// #[derive(Debug)] +pub struct Parser<'a> { + tokenizer: OwnedTokenStream<'a, InternedString, InternString>, + quote_stack: Vec, + quasiquote_depth: isize, + quote_context: bool, + shorthand_quote_stack: Vec, + source_name: Option>, + context: Vec, + comment_buffer: Vec<&'a str>, + collecting_comments: bool, + keep_lists: bool, +} + +#[derive(Debug, Copy, Clone, PartialEq)] +enum ParsingContext { + // Inside of a quote. Expressions should be parsed without being coerced into a typed variant of the AST + Quote(usize), + // Shortened version of a quote + QuoteTick(usize), + // Inside of an unquote - expressions should actually be parsed as usual + Unquote(usize), + // Shortened version of unquote + UnquoteTick(usize), + // Treat this like a normal quote + Quasiquote(usize), + // Shortened version of quasiquote + QuasiquoteTick(usize), + // expressions should parsed as normal + UnquoteSplicing(usize), + // Shorted version of Unquote Splicing + UnquoteSplicingTick(usize), +} + +impl<'a> Parser<'a> { + pub fn parse(expr: &str) -> Result> { + Parser::new(expr, None).collect() + } + + pub fn parse_without_lowering(expr: &str) -> Result> { + Parser::new(expr, None).without_lowering().collect() + } + + pub fn offset(&self) -> usize { + self.tokenizer.offset() + } +} + +pub type Result = result::Result; + +fn tokentype_error_to_parse_error(t: &Token<'_, InternedString>) -> ParseError { + if let TokenType::Error = t.ty { + // println!("Found an error: {}", t); + + if t.source.starts_with('\"') { + ParseError::IncompleteString(t.source.to_string(), t.span, None) + } else { + ParseError::UnexpectedChar(t.source.chars().next().unwrap(), t.span, None) + } + } else { + ParseError::UnexpectedEOF(None) + } +} + +impl<'a> Parser<'a> { + pub fn new(input: &'a str, source_id: Option) -> Self { + Parser { + tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), + quote_stack: Vec::new(), + quasiquote_depth: 0, + quote_context: false, + shorthand_quote_stack: Vec::new(), + source_name: None, + context: Vec::new(), + comment_buffer: Vec::new(), + collecting_comments: false, + keep_lists: false, + } + } + + pub fn without_lowering(mut self) -> Self { + self.keep_lists = true; + self + } + + pub fn new_flat(input: &'a str, source_id: Option) -> Self { + Parser { + tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), + quote_stack: Vec::new(), + quasiquote_depth: 0, + quote_context: false, + shorthand_quote_stack: Vec::new(), + source_name: None, + context: Vec::new(), + comment_buffer: Vec::new(), + collecting_comments: false, + keep_lists: true, + } + } + + pub fn new_from_source( + input: &'a str, + source_name: PathBuf, + source_id: Option, + ) -> Self { + Parser { + tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), + quote_stack: Vec::new(), + quasiquote_depth: 0, + quote_context: false, + shorthand_quote_stack: Vec::new(), + source_name: Some(Rc::from(source_name)), + context: Vec::new(), + comment_buffer: Vec::new(), + collecting_comments: false, + keep_lists: false, + } + } + + // Attach comments! + pub fn doc_comment_parser(input: &'a str, source_id: Option) -> Self { + Parser { + tokenizer: TokenStream::new(input, false, source_id).into_owned(InternString), + quote_stack: Vec::new(), + quasiquote_depth: 0, + quote_context: false, + shorthand_quote_stack: Vec::new(), + source_name: None, + context: Vec::new(), + comment_buffer: Vec::new(), + collecting_comments: false, + keep_lists: false, + } + } + + fn construct_quote(&mut self, val: ExprKind, span: Span) -> ExprKind { + ExprKind::Quote(Box::new(ast::Quote::new( + val, + SyntaxObject::new(TokenType::Quote, span), + ))) + } + + fn _expand_reader_macro( + &mut self, + token: TokenType, + val: ExprKind, + span: Span, + ) -> ExprKind { + let q = ExprKind::Atom(Atom::new(SyntaxObject::new(token, span))); + + ExprKind::List(List::new(vec![q, val])) + } + + fn construct_quote_vec(&mut self, val: ExprKind, span: Span) -> Vec { + // println!("Inside construct quote vec with: {:?}", val); + + let q = { + let rc_val = TokenType::Quote; + ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) + // let val = ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))); + // // self.intern.insert("quote".to_string(), rc_val); + // val + }; + + vec![q, val] + } + + // Reader macro for ` + fn construct_quasiquote(&mut self, val: ExprKind, span: Span) -> ExprKind { + let q = { + let rc_val = TokenType::Identifier(*QUASIQUOTE); + ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) + }; + + ExprKind::List(List::new(vec![q, val])) + } + + // Reader macro for , + fn construct_unquote(&mut self, val: ExprKind, span: Span) -> ExprKind { + let q = { + let rc_val = TokenType::Identifier(*UNQUOTE); + ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) + }; + + ExprKind::List(List::new(vec![q, val])) + } + + fn construct_raw_unquote(&mut self, val: ExprKind, span: Span) -> ExprKind { + let q = { + // let rc_val = TokenType::Identifier(*UNQUOTE); + let rc_val = TokenType::Identifier(*RAW_UNQUOTE); + ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) + }; + + ExprKind::List(List::new(vec![q, val])) + } + // Reader macro for ,@ + fn construct_unquote_splicing(&mut self, val: ExprKind, span: Span) -> ExprKind { + let q = { + let rc_val = TokenType::Identifier(*UNQUOTE_SPLICING); + ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) + }; + + ExprKind::List(List::new(vec![q, val])) + } + + // Reader macro for ,@ + fn construct_raw_unquote_splicing(&mut self, val: ExprKind, span: Span) -> ExprKind { + let q = { + let rc_val = TokenType::Identifier(*RAW_UNQUOTE_SPLICING); + ExprKind::Atom(Atom::new(SyntaxObject::new(rc_val, span))) + }; + + ExprKind::List(List::new(vec![q, val])) + } + + fn increment_quasiquote_context_if_not_in_quote_context(&mut self) { + // println!("INCREMENTING"); + if !self.quote_context { + self.quasiquote_depth += 1; + } + } + + fn decrement_quasiquote_context_if_not_in_quote_context(&mut self) { + // println!("DECREMENTING"); + if !self.quote_context { + self.quasiquote_depth -= 1; + } + } + + fn maybe_lower(&self, expr: Vec) -> Result { + if self.keep_lists { + Ok(ExprKind::List(List::new(expr))) + } else { + ExprKind::try_from(expr) + } + } + + fn read_from_tokens(&mut self) -> Result { + let mut stack: Vec> = Vec::new(); + let mut current_frame: Vec = Vec::new(); + + self.quote_stack = Vec::new(); + + // println!("READING FROM TOKENS"); + // self.quasiquote_depth = 0; + + loop { + match self.tokenizer.next() { + Some(token) => { + match token.ty { + TokenType::Comment => { + // println!("Found a comment!"); + // Internal comments, we're gonna skip for now + continue; + } + TokenType::Error => return Err(tokentype_error_to_parse_error(&token)), // TODO + TokenType::QuoteTick => { + // quote_count += 1; + // self.quote_stack.push(current_frame.len()); + self.shorthand_quote_stack.push(stack.len()); + + let last_context = self.quote_context; + + if self.quasiquote_depth == 0 { + self.quote_context = true; + } + + // println!("Entering context: Quote Tick in read from tokens"); + + self.context.push(ParsingContext::QuoteTick(stack.len())); + + let quote_inner = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| { + // if self.quasiquote_depth == 0 { + self.construct_quote(x, token.span) + // } else { + // self.construct_fake_quote(x, token.span) + // } + }); + // self.quote_stack.pop(); + self.shorthand_quote_stack.pop(); + + self.quote_context = last_context; + + // println!( + // "Exiting Context: {:?} in read from tokens", + // self.context.pop() + // ); + + // self.context.pop(); + + let popped_value = self.context.pop(); + + if let Some(popped) = popped_value { + // dbg!(&popped); + debug_assert!(matches!(popped, ParsingContext::QuoteTick(_))) + } + + current_frame.push(quote_inner?); + } + TokenType::Unquote => { + // println!("Entering context: Unquote"); + + // This could underflow and panic - if its negative then we have a problem. Maybe just use an isize and let it underflow? + self.decrement_quasiquote_context_if_not_in_quote_context(); + + self.context.push(ParsingContext::UnquoteTick(stack.len())); + + let quote_inner = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| { + // dbg!(self.quasiquote_depth); + // dbg!(self.quote_context); + if self.quasiquote_depth == 0 && !self.quote_context { + self.construct_raw_unquote(x, token.span) + } else { + self.construct_unquote(x, token.span) + } + }); + + let popped_value = self.context.pop(); + + self.increment_quasiquote_context_if_not_in_quote_context(); + + if let Some(popped) = popped_value { + debug_assert!(matches!(popped, ParsingContext::UnquoteTick(_))) + } + // println!("Exiting Context: {:?}", self.context.pop()); + current_frame.push(quote_inner?); + } + TokenType::QuasiQuote => { + // println!("Entering context: Quasiquote"); + + self.increment_quasiquote_context_if_not_in_quote_context(); + + self.context + .push(ParsingContext::QuasiquoteTick(stack.len())); + + let quote_inner = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| self.construct_quasiquote(x, token.span)); + + // self.context.pop(); + // println!( + // ">>>>>>>>>>>>>>>>>>> Exiting Context: {:?}", + // self.context.pop() + // ); + + let popped_value = self.context.pop(); + + self.decrement_quasiquote_context_if_not_in_quote_context(); + + if let Some(popped) = popped_value { + // println!("Popped: {:?}", popped); + debug_assert!(matches!(popped, ParsingContext::QuasiquoteTick(_))) + } + + current_frame.push(quote_inner?); + } + TokenType::UnquoteSplice => { + // println!("Entering context: UnquoteSplicing"); + + self.decrement_quasiquote_context_if_not_in_quote_context(); + + self.context + .push(ParsingContext::UnquoteSplicingTick(stack.len())); + + let quote_inner = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| { + if self.quasiquote_depth == 0 && !self.quote_context { + self.construct_raw_unquote_splicing(x, token.span) + } else { + self.construct_unquote_splicing(x, token.span) + } + }); + + // self.context.pop(); + + let popped_value = self.context.pop(); + + self.increment_quasiquote_context_if_not_in_quote_context(); + + if let Some(popped) = popped_value { + debug_assert!(matches!( + popped, + ParsingContext::UnquoteSplicingTick(_) + )) + } + + // println!("Exiting Context: {:?}", self.context.pop()); + current_frame.push(quote_inner?); + } + TokenType::OpenParen => { + stack.push(current_frame); + current_frame = Vec::new(); + } + TokenType::CloseParen => { + // This is the match that we'll want to move inside the below stack.pop() match statement + // As we close the current context, we check what our current state is - + + if let Some(mut prev_frame) = stack.pop() { + match prev_frame.first_mut().and_then(|x| x.atom_identifier_mut()) { + Some(ident) if *ident == *UNQUOTE => { + // self.increment_quasiquote_context_if_not_in_quote_context(); + if self.quasiquote_depth == 0 && !self.quote_context { + *ident = *RAW_UNQUOTE; + } + self.increment_quasiquote_context_if_not_in_quote_context(); + + // println!("Exiting unquote"); + } + Some(ident) if *ident == *QUASIQUOTE => { + self.decrement_quasiquote_context_if_not_in_quote_context(); + + // println!("Exiting quasiquote"); + } + Some(ident) if *ident == *UNQUOTE_SPLICING => { + // self.increment_quasiquote_context_if_not_in_quote_context(); + + if self.quasiquote_depth == 0 && !self.quote_context { + *ident = *RAW_UNQUOTE_SPLICING; + } + self.increment_quasiquote_context_if_not_in_quote_context(); + + // println!("Exiting unquote"); + } + _ => {} + } + + match self.context.last().cloned() { + // TODO: Change this -> This should really be just Some(ParsingContext::Quote) + // If we have _anything_ then we should check if we need to parse it differently. If we're at the last_quote_index, + // then we can pop it off inside there. + Some(ParsingContext::Quote(last_quote_index)) + | Some(ParsingContext::Quasiquote(last_quote_index)) => { + if stack.len() <= last_quote_index { + self.context.pop(); + } + + match current_frame.first() { + Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Quote, + .. + }, + })) => match self.context.last() { + Some( + ParsingContext::Quasiquote(_) + | ParsingContext::QuasiquoteTick(_) + | ParsingContext::Quote(_) + | ParsingContext::QuoteTick(_), + ) => prev_frame + .push(ExprKind::List(List::new(current_frame))), + _ => { + prev_frame.push( + self.maybe_lower(current_frame).map_err( + |x| { + x.set_source( + self.source_name.clone(), + ) + }, + )?, + ); + } + }, + _ => { + // println!("Converting to list"); + // println!("Context here: {:?}", self.context); + prev_frame + .push(ExprKind::List(List::new(current_frame))) + } + } + } + + Some(ParsingContext::QuoteTick(_)) + | Some(ParsingContext::QuasiquoteTick(_)) => { + match current_frame.first() { + Some(ExprKind::Atom(Atom { + syn: + SyntaxObject { + ty: TokenType::Quote, + .. + }, + })) => { + // println!("Converting to quote inside quote tick"); + prev_frame.push( + self.maybe_lower(current_frame).map_err( + |x| x.set_source(self.source_name.clone()), + )?, + ); + } + _ => { + // if let Some(ParsingContext::QuasiquoteTick(_)) = + // self.context.last() + // { + // self.decrement_quasiquote_context_if_not_in_quote_context(); + // } + + // println!("Converting to list inside quote tick"); + prev_frame + .push(ExprKind::List(List::new(current_frame))) + } + } + } + + // If we're in the short hand reader world, just ignore popping off the stack + // but still treat it as a normal expression + Some(ParsingContext::UnquoteTick(_)) + | Some(ParsingContext::UnquoteSplicingTick(_)) => { + // self.quasiquote_depth += 1; + + // self.increment_quasiquote_context_if_not_in_quote_context(); + + // println!( + // "UQ/UQS: Stack length: {:?}, last_quote_index: {:?}", + // stack.len(), + // last_quote_index + // ); + + // if stack.len() <= *last_quote_index { + // // println!("Exiting Context: {:?}", self.context.pop()); + // self.context.pop(); + // } + + prev_frame.push( + self.maybe_lower(current_frame).map_err(|x| { + x.set_source(self.source_name.clone()) + })?, + ); + } + + Some(ParsingContext::Unquote(last_quote_index)) + | Some(ParsingContext::UnquoteSplicing(last_quote_index)) => { + // self.quasiquote_depth += 1; + + // self.increment_quasiquote_context_if_not_in_quote_context(); + + // println!( + // "UQ/UQS: Stack length: {:?}, last_quote_index: {:?}", + // stack.len(), + // last_quote_index + // ); + + if stack.len() <= last_quote_index { + // println!("{} - {}", stack.len(), last_quote_index); + // println!("Exiting Context: {:?}", self.context.pop()); + self.context.pop(); + } + + prev_frame.push( + self.maybe_lower(current_frame).map_err(|x| { + x.set_source(self.source_name.clone()) + })?, + ); + } + + // Else case, just go ahead and assume it is a normal frame + _ => prev_frame.push( + self.maybe_lower(current_frame) + .map_err(|x| x.set_source(self.source_name.clone()))?, + ), + } + + // Reinitialize current frame here + current_frame = prev_frame; + } else { + // println!("Else case: {:?}", current_frame); + // println!("Context: {:?}", self.context); + + // dbg!(&self.quote_stack); + // dbg!(&self.context); + // dbg!(&self.shorthand_quote_stack); + match self.context.last() { + Some(ParsingContext::QuoteTick(_)) + | Some(ParsingContext::QuasiquoteTick(_)) => { + // | Some(ParsingContext::Quote(d)) && d > 0 => { + return Ok(ExprKind::List(List::new(current_frame))); + } + Some(ParsingContext::Quote(x)) if *x > 0 => { + self.context.pop(); + + return Ok(ExprKind::List(List::new(current_frame))); + } + Some(ParsingContext::Quote(0)) => { + self.context.pop(); + + return self + .maybe_lower(current_frame) + .map_err(|x| x.set_source(self.source_name.clone())); + } + _ => { + // dbg!(self.quasiquote_depth); + // println!("=> {}", List::new(current_frame.clone())); + // println!("----------------------------------------"); + + if self.quasiquote_depth > 0 { + // TODO/HACK - @Matt + // If we're in a define syntax situation, go ahead and just return a normal one + if current_frame + .first() + .map(|x| x.define_syntax_ident()) + .unwrap_or_default() + { + return self.maybe_lower(current_frame).map_err( + |x| x.set_source(self.source_name.clone()), + ); + } + + // println!("Should still be quoted here"); + + return Ok(ExprKind::List(List::new(current_frame))); + } + + return self + .maybe_lower(current_frame) + .map_err(|x| x.set_source(self.source_name.clone())); + } + } + } + } + + _ => { + if let TokenType::Quote = &token.ty { + // self.quote_stack.push(current_frame.len()); + self.quote_stack.push(stack.len()); + } + + // dbg!(&self.context); + + // Mark what context we're inside with the context stack: + // This only works when its the first argument - check the function call in open paren? + if current_frame.is_empty() { + match &token.ty { + TokenType::Quote => { + if self.context == &[ParsingContext::QuoteTick(0)] { + self.context.push(ParsingContext::Quote(1)) + } else { + self.context.push(ParsingContext::Quote(stack.len())) + } + + // self.context.push(ParsingContext::Quote(stack.len())) + } + TokenType::Identifier(ident) if *ident == *UNQUOTE => { + // println!("Entering unquote"); + + self.context.push(ParsingContext::Unquote(stack.len())); + self.decrement_quasiquote_context_if_not_in_quote_context(); + } + TokenType::Identifier(ident) if *ident == *QUASIQUOTE => { + // println!("Entering quasiquote"); + + self.context.push(ParsingContext::Quasiquote(stack.len())); + self.increment_quasiquote_context_if_not_in_quote_context(); + } + TokenType::Identifier(ident) if *ident == *UNQUOTE_SPLICING => { + self.context + .push(ParsingContext::UnquoteSplicing(stack.len())); + self.decrement_quasiquote_context_if_not_in_quote_context(); + } + _ => {} + } + + // println!("Context on application: {:?}", self.context); + } + + // println!("{}", token); + + current_frame.push(ExprKind::Atom(Atom::new( + SyntaxObject::from_token_with_source( + &token, + &self.source_name.clone(), + ), + ))) + } + } + } + + None => return Err(ParseError::UnexpectedEOF(self.source_name.clone())), + } + } + } +} + +fn wrap_in_doc_function(expr: ExprKind, comment: String) -> ExprKind { + // println!("Found comment : {} for expr {}", comment, expr); + + ExprKind::List(List::new(vec![ + ExprKind::ident("@doc"), + ExprKind::string_lit(comment), + expr, + ])) +} + +impl<'a> Parser<'a> { + fn get_next_and_maybe_wrap_in_doc(&mut self) -> Option> { + let mut next; + + loop { + next = self.tokenizer.next(); + + if let Some(res) = next { + match res.ty { + TokenType::Comment => { + if self.comment_buffer.is_empty() && !self.collecting_comments { + if res.source().trim_start_matches(';').starts_with("@doc") { + self.collecting_comments = true; + + continue; + } + } + + if self.collecting_comments { + let doc_line = res.source().trim_start_matches(';'); + + // If we hit another comment, clear it + if doc_line.starts_with("@doc") { + // println!("Clearing buffer"); + + self.comment_buffer.clear(); + continue; + } + + // println!("Collecting line: {}", doc_line); + + self.comment_buffer.push(doc_line.trim_start()); + } + + continue; + } + + TokenType::QuoteTick => { + // See if this does the job + self.shorthand_quote_stack.push(0); + + let last = self.quote_context; + + if self.quasiquote_depth == 0 { + self.quote_context = true; + } + + // self.quote_context = true; + + // println!("Entering Context: Quote Tick"); + self.context.push(ParsingContext::QuoteTick(0)); + + let value = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| self.construct_quote_vec(x, res.span)); + + self.shorthand_quote_stack.pop(); + + let popped_value = self.context.pop(); + + if let Some(popped) = popped_value { + // dbg!(&popped); + debug_assert!(matches!(popped, ParsingContext::QuoteTick(_))) + } + + self.quote_context = last; + + // println!("Exiting context: {:?}", self.context.pop()); + // println!("Result: {:?}", value); + + // println!("{}", List::new(value.clone().unwrap())); + + return Some(match value { + Ok(v) => { + // Ok(ExprKind::List(List::new(v))) + + self.maybe_lower(v) + } + Err(e) => Err(e), + }); + } + + TokenType::Unquote => { + // println!("Entering Context: Unquote"); + self.context.push(ParsingContext::UnquoteTick(0)); + + self.decrement_quasiquote_context_if_not_in_quote_context(); + + let value = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| { + // dbg!(&self.quasiquote_depth); + if self.quasiquote_depth == 0 && !self.quote_context { + self.construct_raw_unquote(x, res.span) + } else { + self.construct_unquote(x, res.span) + } + }); + + let popped_value = self.context.pop(); + + self.increment_quasiquote_context_if_not_in_quote_context(); + + if let Some(popped) = popped_value { + debug_assert!(matches!(popped, ParsingContext::UnquoteTick(_))) + } + // println!("Exiting context: {:?}", self.context.pop()); + + return Some(value); + } + + TokenType::UnquoteSplice => { + // println!("Entering Context: Unquotesplicing"); + self.context.push(ParsingContext::UnquoteSplicingTick(0)); + + self.decrement_quasiquote_context_if_not_in_quote_context(); + + let value = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| { + if self.quasiquote_depth == 0 && !self.quote_context { + self.construct_raw_unquote_splicing(x, res.span) + } else { + self.construct_unquote_splicing(x, res.span) + } + }); + + let popped_value = self.context.pop(); + + self.increment_quasiquote_context_if_not_in_quote_context(); + + if let Some(popped) = popped_value { + debug_assert!(matches!(popped, ParsingContext::UnquoteSplicingTick(_))) + } + + // println!("Exiting context: {:?}", self.context.pop()); + + return Some(value); + } + + TokenType::QuasiQuote => { + // println!("Entering Context: Quasiquote - top level"); + self.context.push(ParsingContext::QuasiquoteTick(0)); + + self.increment_quasiquote_context_if_not_in_quote_context(); + + let value = self + .next() + .unwrap_or(Err(ParseError::UnexpectedEOF(self.source_name.clone()))) + .map(|x| self.construct_quasiquote(x, res.span)); + + // println!("{:?}", self.context.pop()); + + // println!("Top level Context: {:?}", self.context); + + let popped_value = self.context.pop(); + + if let Some(popped) = popped_value { + debug_assert!(matches!(popped, ParsingContext::QuasiquoteTick(_))) + } + + self.decrement_quasiquote_context_if_not_in_quote_context(); + + // println!("Exiting context: {:?}", self.context.pop()); + + return Some(value); + } + + TokenType::OpenParen => return Some(self.read_from_tokens()), + TokenType::CloseParen => { + return Some(Err(ParseError::Unexpected( + TokenType::CloseParen, + self.source_name.clone(), + ))) + } + TokenType::Error => return Some(Err(tokentype_error_to_parse_error(&res))), + _ => return Some(Ok(ExprKind::Atom(Atom::new(SyntaxObject::from(&res))))), + }; + } else { + // We're done consuming input + return None; + } + } + } +} + +impl<'a> Iterator for Parser<'a> { + type Item = Result; + + // TODO -> put the + fn next(&mut self) -> Option { + if self.quote_stack.is_empty() + && self.shorthand_quote_stack.is_empty() + && self.context.is_empty() + { + self.quasiquote_depth = 0; + self.comment_buffer.clear(); + } + + self.get_next_and_maybe_wrap_in_doc().map(|res| { + if self.comment_buffer.is_empty() { + res + } else { + // Reset the comment collection until next @doc statement + self.collecting_comments = false; + res.map(|x| { + wrap_in_doc_function(x, self.comment_buffer.join("\n").drain(..).collect()) + }) + } + }) + } +} + +// Lower the syntax rules down from the list representation +pub fn lower_syntax_rules(expr: ExprKind) -> Result { + let mut value_iter = expr.into_list().into_iter(); + let syn = value_iter + .next() + .unwrap() + .into_atom_syntax_object() + .unwrap(); + + let syntax_vec = if let Some(ExprKind::List(l)) = value_iter.next() { + l.args + } else { + return Err(ParseError::SyntaxError( + "syntax-rules expects a list of new syntax forms used in the macro".to_string(), + syn.span, + None, + )); + }; + + let mut pairs = Vec::new(); + let rest: Vec<_> = value_iter.collect(); + + for pair in rest { + if let ExprKind::List(l) = pair { + if l.args.len() != 2 { + return Err(ParseError::SyntaxError( + "syntax-rules requires only one pattern to one body".to_string(), + syn.span, + None, + )); + } + + let mut pair_iter = l.args.into_iter(); + let pair_object = + PatternPair::new(pair_iter.next().unwrap(), pair_iter.next().unwrap()); + pairs.push(pair_object); + } else { + return Err(ParseError::SyntaxError( + "syntax-rules requires pattern to expressions to be in a list".to_string(), + syn.span, + None, + )); + } + } + + Ok(SyntaxRules::new(syntax_vec, pairs, syn)) +} + +// Lower define-syntax down from the list representation +pub fn lower_macro_and_require_definitions(expr: ExprKind) -> Result { + let as_list = expr.list(); + + // If this qualifies as + if as_list.map(List::is_define_syntax).unwrap_or_default() + && as_list + .unwrap() + .get(2) + .and_then(ExprKind::list) + .map(List::is_syntax_rules) + .unwrap_or_default() + { + let mut value_iter = expr.into_list().into_iter(); + + let define_syntax = value_iter.next().unwrap(); + + let name = value_iter.next().unwrap(); + let syntax = lower_syntax_rules(value_iter.next().unwrap())?; + + return Ok(ExprKind::Macro(Macro::new( + name, + syntax, + define_syntax.into_atom_syntax_object().unwrap(), + ))); + } + + if as_list.map(List::is_require).unwrap_or_default() { + let mut raw = expr.into_list().args; + + let syn = raw.remove(0).into_atom_syntax_object().unwrap(); + + if raw.is_empty() { + return Err(ParseError::ArityMismatch( + "require expects at least one identifier or string".to_string(), + syn.span, + None, + )); + } + + return Ok(ExprKind::Require(ast::Require::new(raw, syn))); + } + + Ok(expr) +} + +struct ASTLowerPass { + quote_depth: usize, +} + +impl ASTLowerPass { + // TODO: Make this mutable references, otherwise we'll be re-boxing everything for now reason + fn lower(&mut self, expr: ExprKind) -> Result { + match expr { + ExprKind::List(mut value) => { + if value.is_quote() { + // println!("Found quote"); + self.quote_depth += 1; + } + + // Visit the children first, on the way back up, assign into the + // correct AST node + value.args = value + .args + .into_iter() + .map(|x| self.lower(x)) + .collect::>()?; + + if value.is_quote() { + self.quote_depth -= 1; + } + + if let Some(f) = value.first().cloned() { + match f { + ExprKind::Atom(a) if self.quote_depth == 0 && value.is_quote() => { + match &a.syn.ty { + TokenType::Quote => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "quote", + |expr, syn| ast::Quote::new(expr, syn).into(), + ), + _ => unreachable!(), + } + } + ExprKind::Atom(a) if self.quote_depth == 0 => { + match &a.syn.ty { + // Have this also match on the first argument being a TokenType::Identifier("if") + // Do the same for the rest of the arguments + TokenType::If => parse_if(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *IF => { + parse_if(value.into_iter(), a.syn.clone()) + } + + TokenType::Define => parse_define(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *DEFINE => { + parse_define(value.into_iter(), a.syn.clone()) + } + + TokenType::Let => parse_let(value.into_iter(), a.syn.clone()), + TokenType::Identifier(expr) if *expr == *LET => { + parse_let(value.into_iter(), a.syn.clone()) + } + + // TODO: Deprecate + TokenType::TestLet => { + parse_new_let(value.into_iter(), a.syn.clone()) + } + TokenType::Identifier(expr) if *expr == *PLAIN_LET => { + parse_new_let(value.into_iter(), a.syn.clone()) + } + + TokenType::Quote => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "quote", + |expr, syn| ast::Quote::new(expr, syn).into(), + ), + TokenType::Identifier(expr) if *expr == *QUOTE => { + parse_single_argument( + value.into_iter(), + a.syn.clone(), + "quote", + |expr, syn| ast::Quote::new(expr, syn).into(), + ) + } + + TokenType::Return => parse_single_argument( + value.into_iter(), + a.syn.clone(), + "return!", + |expr, syn| ast::Return::new(expr, syn).into(), + ), + TokenType::Identifier(expr) if *expr == *RETURN => { + parse_single_argument( + value.into_iter(), + a.syn.clone(), + "return!", + |expr, syn| ast::Return::new(expr, syn).into(), + ) + } + + TokenType::Require => parse_require(&a, value.args), + TokenType::Identifier(expr) if *expr == *REQUIRE => { + parse_require(&a, value.args) + } + + TokenType::Set => parse_set(&a, value.args), + TokenType::Identifier(expr) if *expr == *SET => { + parse_set(&a, value.args) + } + + TokenType::Begin => parse_begin(&a, value.args), + TokenType::Identifier(expr) if *expr == *BEGIN => { + parse_begin(&a, value.args) + } + + TokenType::Lambda => parse_lambda(&a, value.args), + TokenType::Identifier(expr) + if *expr == *LAMBDA + || *expr == *LAMBDA_FN + || *expr == *LAMBDA_SYMBOL => + { + parse_lambda(&a, value.args) + } + + _ => Ok(ExprKind::List(value)), + } + } + _ => Ok(ExprKind::List(value)), + } + } else { + Ok(ExprKind::List(List::new(vec![]))) + } + } + ExprKind::Atom(_) => Ok(expr), + ExprKind::If(iff) => Ok(ExprKind::If(Box::new(ast::If::new( + self.lower(iff.test_expr)?, + self.lower(iff.then_expr)?, + self.lower(iff.else_expr)?, + iff.location, + )))), + ExprKind::Let(l) => Ok(ExprKind::Let(Box::new(ast::Let::new( + l.bindings + .into_iter() + .map(|(a, b)| Ok((self.lower(a)?, self.lower(b)?))) + .collect::>()?, + self.lower(l.body_expr)?, + l.location, + )))), + ExprKind::Define(d) => Ok(ExprKind::Define(Box::new(ast::Define::new( + self.lower(d.name)?, + self.lower(d.body)?, + d.location, + )))), + ExprKind::LambdaFunction(f) => Ok(ExprKind::LambdaFunction(Box::new( + ast::LambdaFunction::new_maybe_rest( + f.args + .into_iter() + .map(|x| self.lower(x)) + .collect::>()?, + self.lower(f.body)?, + f.location, + f.rest, + ), + ))), + ExprKind::Begin(b) => Ok(ExprKind::Begin(ast::Begin::new( + b.exprs + .into_iter() + .map(|x| self.lower(x)) + .collect::>()?, + b.location, + ))), + ExprKind::Return(r) => Ok(ExprKind::Return(Box::new(ast::Return::new( + self.lower(r.expr)?, + r.location, + )))), + ExprKind::Quote(_) => Ok(expr), + ExprKind::Macro(_) => Ok(expr), + ExprKind::SyntaxRules(_) => Ok(expr), + ExprKind::Set(s) => Ok(ExprKind::Set(Box::new(ast::Set::new( + self.lower(s.variable)?, + self.lower(s.expr)?, + s.location, + )))), + ExprKind::Require(_) => Ok(expr), // _ => Ok(expr), + } + } +} + +// TODO: Lower the rest of the AST post expansion, such that +pub fn lower_entire_ast(expr: ExprKind) -> Result { + ASTLowerPass { quote_depth: 0 }.lower(expr) +} + +#[cfg(test)] +mod parser_tests { + // use super::TokenType::*; + use super::*; + use crate::parser::ast::{Begin, Define, If, LambdaFunction, Quote, Return}; + use crate::{parser::ast::ExprKind, tokens::MaybeBigInt}; + + fn atom(ident: &str) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::Identifier( + ident.into(), + )))) + } + + fn int(num: isize) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::IntegerLiteral( + MaybeBigInt::Small(num), + )))) + } + + fn character(c: char) -> ExprKind { + ExprKind::Atom(Atom::new(SyntaxObject::default( + TokenType::CharacterLiteral(c), + ))) + } + + #[test] + fn check_quote_parsing() { + println!("{:?}", Parser::parse("'(a b 'c)")); + } + + fn parses(s: &str) { + let a: Result> = Parser::new(s, None).collect(); + a.unwrap(); + } + + fn assert_parse(s: &str, result: &[ExprKind]) { + let a: Result> = Parser::new(s, None).collect(); + let a = a.unwrap(); + assert_eq!(a.as_slice(), result); + } + + fn assert_parse_err(s: &str, err: ParseError) { + let a: Result> = Parser::new(s, None).collect(); + assert_eq!(a, Err(err)); + } + + fn assert_parse_is_err(s: &str) { + let a: Result> = Parser::new(s, None).collect(); + assert!(a.is_err()); + } + + #[test] + fn check_resulting_parsing() { + let expr = r#"`(a `(b ,(+ 1 2) ,(foo ,(+ 1 3) d) e) f)"#; + + let a: Result> = Parser::new(expr, None).collect(); + let a = a.unwrap(); + + println!("{}", a[0]); + } + + #[test] + fn check_double_unquote_parsing() { + let expr = r#"(let ([name1 'x] [name2 'y]) `(a `(b ,,name1 ,',name2 d) e))"#; + + let a: Result> = Parser::new(expr, None).collect(); + let a = a.unwrap(); + + println!("{}", a[0]); + } + + #[test] + fn check_parser_with_doc_comments() { + let expr = r#" + ;;@doc + ;; This is a fancy cool comment, that I want to attach to a top level definition! + ;; This is the second line of the comment, I want this attached as well! + ;; Macro for creating a new struct, in the form of: + ;; `(struct (fields ...) options ...)` + ;; The options can consist of the following: + ;; + ;; Single variable options (those which their presence indicates #true) + ;; - #:mutable + ;; - #:transparent + ;; + ;; Other options must be presented as key value pairs, and will get stored + ;; in the struct instance. They will also be bound to the variable + ;; ___-options___ in the same lexical environment where the + ;; struct was defined. For example: + ;; + ;; (Applesauce (a b c) #:mutable #:transparent #:unrecognized-option 1234) + ;; + ;; Will result in the value `___Applesauce-options___` like so: + ;; (hash #:mutable #true #:transparent #true #:unrecognized-option 1234) + ;; + ;; By default, structs are immutable, which means setter functions will not + ;; be generated. Also by default, structs are not transparent, which means + ;; printing them will result in an opaque struct that does not list the fields + (define foo 12345) + "#; + + let parser = Parser::doc_comment_parser(expr, None); + + let result: Result> = parser.collect(); + + println!("{:?}", result.unwrap()); + } + + #[test] + fn parses_make_struct() { + parses("(define make-struct (lambda (struct-name fields) (map (lambda (field) (list (quote define) (concat-symbols struct-name field) (quote (lambda (this) (vector-ref this 0))))) fields)))") + } + + #[test] + fn parses_quasiquote() { + parses(r#"(quasiquote ((unquote x) xs ...)) "#); + } + + #[test] + fn parse_syntax_rules() { + parses( + r#" + (syntax-rules (unquote unquote-splicing) + ((quasiquote ((unquote x) xs ...)) (cons x (quasiquote (xs ...)))) + ((quasiquote ((unquote-splicing x))) (append (list x) '())) + ((quasiquote ((unquote-splicing x) xs ...)) (append x (quasiquote (xs ...)))) + ((quasiquote (unquote x)) x) + ((quasiquote (x)) '(x)) + ((quasiquote (x xs ...)) (cons (quasiquote x) (quasiquote (xs ...)))) + ((quasiquote x) 'x)) + "#, + ); + } + + #[test] + fn parse_define_syntax() { + parses( + r#" + (define-syntax quasiquote + (syntax-rules (unquote unquote-splicing) + ((quasiquote ((unquote x) xs ...)) (cons x (quasiquote (xs ...)))) + ((quasiquote ((unquote-splicing x))) (append (list x) '())) + ((quasiquote ((unquote-splicing x) xs ...)) (append x (quasiquote (xs ...)))) + ((quasiquote (unquote x)) x) + ((quasiquote (x)) '(x)) + ((quasiquote (x xs ...)) (cons (quasiquote x) (quasiquote (xs ...)))) + ((quasiquote x) 'x))) + "#, + ); + } + + #[test] + fn parse_quote() { + // parses("(displayln (match (quote (lambda y z)) '(x y z)))") + parses("(displayln (match '(lambda y z) '(x y z)))") + } + + #[test] + fn parse_unicode() { + assert_parse("#\\¡", &[character('¡')]); + assert_parse("#\\u{b}", &[character('\u{b}')]); + } + + #[test] + fn parse_more_unicode() { + assert_parse("#\\u{a0}", &[character('\u{a0}')]); + } + + #[test] + fn parse_strange_characters() { + assert_parse("#\\^", &[character('^')]); + } + + #[test] + fn parse_character_sequence() { + assert_parse( + "#\\¡ #\\SPACE #\\g", + &[character('¡'), character(' '), character('g')], + ) + } + + #[test] + fn parse_character_sequence_inside_if() { + assert_parse( + "(if #\\¡ #\\SPACE #\\g)", + &[ExprKind::If(Box::new(If::new( + character('¡'), + character(' '), + character('g'), + SyntaxObject::default(TokenType::If), + )))], + ) + } + + #[test] + fn parse_close_paren_character() { + assert_parse("#\\)", &[character(')')]); + assert_parse("#\\]", &[character(']')]) + } + + #[test] + fn parse_open_paren_character() { + assert_parse("#\\(", &[character('(')]) + } + + #[test] + fn test_error() { + assert_parse_err("(", ParseError::UnexpectedEOF(None)); + assert_parse_err("(abc", ParseError::UnexpectedEOF(None)); + assert_parse_err("(ab 1 2", ParseError::UnexpectedEOF(None)); + assert_parse_err("((((ab 1 2) (", ParseError::UnexpectedEOF(None)); + assert_parse_err("())", ParseError::Unexpected(TokenType::CloseParen, None)); + assert_parse_err("() ((((", ParseError::UnexpectedEOF(None)); + assert_parse_err("')", ParseError::Unexpected(TokenType::CloseParen, None)); + assert_parse_err("(')", ParseError::Unexpected(TokenType::CloseParen, None)); + assert_parse_err("('", ParseError::UnexpectedEOF(None)); + } + + #[test] + fn quote_multiple_args_should_err() { + assert_parse_is_err("(quote a b c)"); + } + + #[test] + fn test_let_should_err() { + assert_parse_is_err("(let)"); + assert_parse_is_err("(let (a) 10)"); + } + + #[test] + fn test_if_should_err() { + assert_parse_is_err("(if)"); + assert_parse_is_err("(if 1)"); + // assert_parse_is_err("(if 1 2)"); + assert_parse_is_err("(if 1 2 3 4)"); + } + + #[test] + fn test_define_should_err() { + assert_parse_is_err("(define)"); + assert_parse_is_err("(define blagh)"); + assert_parse_is_err("(define test 1 2)"); + assert_parse_is_err("(define () test"); + } + + #[test] + fn test_lambda_should_err() { + assert_parse_is_err("(lambda)"); + assert_parse_is_err("(lambda (x))"); + } + + #[test] + fn test_empty() { + assert_parse("", &[]); + assert_parse("()", &[ExprKind::List(List::new(vec![]))]); + } + + #[test] + fn test_empty_quote_inside_if() { + assert_parse( + "(if #\\¡ (quote ()) #\\g)", + &[ExprKind::If(Box::new(If::new( + character('¡'), + ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + character('g'), + SyntaxObject::default(TokenType::If), + )))], + ) + } + + #[test] + fn test_empty_quote() { + assert_parse( + "'()", + &[ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + )], + ) + } + + #[test] + fn test_empty_quote_nested() { + assert_parse( + "(list '())", + &[ExprKind::List(List::new(vec![ + atom("list"), + ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ]))], + ) + } + + #[test] + fn test_multi_parse_simple() { + assert_parse("a b +", &[atom("a"), atom("b"), atom("+")]); + } + + #[test] + fn test_multi_parse_complicated() { + assert_parse( + "a b (funcall 1 (+ 2 3.5))", + &[ + atom("a"), + atom("b"), + ExprKind::List(List::new(vec![ + atom("funcall"), + int(1), + ExprKind::List(List::new(vec![ + atom("+"), + int(2), + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::NumberLiteral( + 3.5, + )))), + ])), + ])), + ], + ) + } + #[test] + fn test_parse_simple() { + assert_parse( + "(+ 1 2 3) (- 4 3)", + &[ + ExprKind::List(List::new(vec![atom("+"), int(1), int(2), int(3)])), + ExprKind::List(List::new(vec![atom("-"), int(4), int(3)])), + ], + ); + } + + #[test] + fn test_parse_nested() { + assert_parse( + "(+ 1 (foo (bar 2 3)))", + &[ExprKind::List(List::new(vec![ + atom("+"), + int(1), + ExprKind::List(List::new(vec![ + atom("foo"), + ExprKind::List(List::new(vec![atom("bar"), int(2), int(3)])), + ])), + ]))], + ); + assert_parse( + "(+ 1 (+ 2 3) (foo (bar 2 3)))", + &[ExprKind::List(List::new(vec![ + atom("+"), + int(1), + ExprKind::List(List::new(vec![atom("+"), int(2), int(3)])), + ExprKind::List(List::new(vec![ + atom("foo"), + ExprKind::List(List::new(vec![atom("bar"), int(2), int(3)])), + ])), + ]))], + ); + } + + #[test] + fn test_if() { + assert_parse( + "(+ 1 (if 2 3 4) (foo (+ (bar 1 1) 3) 5))", + &[ExprKind::List(List::new(vec![ + atom("+"), + int(1), + ExprKind::If(Box::new(If::new( + int(2), + int(3), + int(4), + SyntaxObject::default(TokenType::If), + ))), + ExprKind::List(List::new(vec![ + atom("foo"), + ExprKind::List(List::new(vec![ + atom("+"), + ExprKind::List(List::new(vec![atom("bar"), int(1), int(1)])), + int(3), + ])), + int(5), + ])), + ]))], + ); + } + + #[test] + fn test_quote() { + assert_parse( + "(quote (if 1 2))", + &[ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + int(1), + int(2), + ])), + SyntaxObject::default(TokenType::Quote), + )))], + ) + } + + #[test] + fn test_quote_shorthand() { + assert_parse( + "'(if 1 2)", + &[ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + int(1), + int(2), + ])), + SyntaxObject::default(TokenType::Quote), + )))], + ) + } + + #[test] + fn test_quote_nested() { + assert_parse( + "(quote (if (if 1 2) 3))", + &[ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + int(1), + int(2), + ])), + int(3), + ])), + SyntaxObject::default(TokenType::Quote), + )))], + ) + } + + #[test] + fn test_quote_shorthand_nested() { + assert_parse( + "'(if (if 1 2) 3)", + &[ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + int(1), + int(2), + ])), + int(3), + ])), + SyntaxObject::default(TokenType::Quote), + )))], + ) + } + + #[test] + fn test_quote_shorthand_multiple_exprs() { + assert_parse( + "'(if (if 1 2) 3) (+ 1 (if 2 3 4) (foo (+ (bar 1 1) 3) 5))", + &[ + ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default(TokenType::If))), + int(1), + int(2), + ])), + int(3), + ])), + SyntaxObject::default(TokenType::Quote), + ))), + ExprKind::List(List::new(vec![ + atom("+"), + int(1), + ExprKind::If(Box::new(If::new( + int(2), + int(3), + int(4), + SyntaxObject::default(TokenType::If), + ))), + ExprKind::List(List::new(vec![ + atom("foo"), + ExprKind::List(List::new(vec![ + atom("+"), + ExprKind::List(List::new(vec![atom("bar"), int(1), int(1)])), + int(3), + ])), + int(5), + ])), + ])), + ], + ) + } + + #[test] + fn test_quote_inner() { + assert_parse( + "'(applesauce 'one)", + &[ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + atom("applesauce"), + ExprKind::Quote(Box::new(Quote::new( + atom("one"), + SyntaxObject::default(TokenType::Quote), + ))), + ])), + SyntaxObject::default(TokenType::Quote), + )))], + ) + } + + #[test] + fn test_quote_inner_without_shorthand() { + assert_parse( + "(quote (applesauce 'one))", + &[ExprKind::Quote(Box::new(Quote::new( + ExprKind::List(List::new(vec![ + atom("applesauce"), + ExprKind::Quote(Box::new(Quote::new( + atom("one"), + SyntaxObject::default(TokenType::Quote), + ))), + ])), + SyntaxObject::default(TokenType::Quote), + )))], + ) + } + + #[test] + fn test_quasiquote_shorthand() { + assert_parse( + "`(+ 1 2)", + &[ExprKind::List(List::new(vec![ + atom("quasiquote"), + ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), + ]))], + ) + } + + #[test] + fn test_quasiquote_normal() { + assert_parse( + "(quasiquote (+ 1 2))", + &[ExprKind::List(List::new(vec![ + atom("quasiquote"), + ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), + ]))], + ) + } + + #[test] + fn test_unquote_shorthand() { + assert_parse( + ",(+ 1 2)", + &[ExprKind::List(List::new(vec![ + atom("unquote"), + ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), + ]))], + ) + } + + #[test] + fn test_unquote_normal() { + assert_parse( + "(unquote (+ 1 2))", + &[ExprKind::List(List::new(vec![ + atom("unquote"), + ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), + ]))], + ) + } + + #[test] + fn test_unquote_splicing_shorthand() { + assert_parse( + ",@(+ 1 2)", + &[ExprKind::List(List::new(vec![ + atom("unquote-splicing"), + ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), + ]))], + ) + } + + #[test] + fn test_unquote_splicing_normal() { + assert_parse( + "(unquote-splicing (+ 1 2))", + &[ExprKind::List(List::new(vec![ + atom("unquote-splicing"), + ExprKind::List(List::new(vec![atom("+"), int(1), int(2)])), + ]))], + ) + } + + #[test] + fn test_define_simple() { + assert_parse( + "(define a 10)", + &[ExprKind::Define(Box::new(Define::new( + atom("a"), + int(10), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_define_func_simple() { + assert_parse( + "(define (foo x) (+ x 10))", + &[ExprKind::Define(Box::new(Define::new( + atom("foo"), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("x")], + ExprKind::List(List::new(vec![atom("+"), atom("x"), int(10)])), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_define_func_multiple_args() { + assert_parse( + "(define (foo x y z) (+ x 10))", + &[ExprKind::Define(Box::new(Define::new( + atom("foo"), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("x"), atom("y"), atom("z")], + ExprKind::List(List::new(vec![atom("+"), atom("x"), int(10)])), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_define_func_multiple_args_multiple_body_exprs() { + assert_parse( + "(define (foo x y z) (+ x 10) (+ y 20) (+ z 30))", + &[ExprKind::Define(Box::new(Define::new( + atom("foo"), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("x"), atom("y"), atom("z")], + ExprKind::Begin(Begin::new( + vec![ + ExprKind::List(List::new(vec![atom("+"), atom("x"), int(10)])), + ExprKind::List(List::new(vec![atom("+"), atom("y"), int(20)])), + ExprKind::List(List::new(vec![atom("+"), atom("z"), int(30)])), + ], + SyntaxObject::default(TokenType::Begin), + )), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_recursive_function() { + assert_parse( + "(define (test) (define (foo) (bar)) (define (bar) (foo)))", + &[ExprKind::Define(Box::new(Define::new( + atom("test"), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![], + ExprKind::Begin(Begin::new( + vec![ + ExprKind::Define(Box::new(Define::new( + atom("foo"), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![], + ExprKind::List(List::new(vec![atom("bar")])), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + ))), + ExprKind::Define(Box::new(Define::new( + atom("bar"), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![], + ExprKind::List(List::new(vec![atom("foo")])), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + ))), + ], + SyntaxObject::default(TokenType::Begin), + )), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_return_normal() { + assert_parse( + "(return! 10)", + &[ExprKind::Return(Box::new(Return::new( + int(10), + SyntaxObject::default(TokenType::Return), + )))], + ) + } + + #[test] + fn test_begin() { + assert_parse( + "(begin 1 2 3)", + &[ExprKind::Begin(Begin::new( + vec![int(1), int(2), int(3)], + SyntaxObject::default(TokenType::Begin), + ))], + ) + } + + #[test] + fn test_lambda_function() { + assert_parse( + "(lambda (x) 10)", + &[ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("x")], + int(10), + SyntaxObject::default(TokenType::Lambda), + )))], + ) + } + + #[test] + fn test_lambda_matches_let() { + assert_parse( + "((lambda (a) (+ a 20)) 10)", + &[ExprKind::List(List::new(vec![ + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("a")], + ExprKind::List(List::new(vec![atom("+"), atom("a"), int(20)])), + SyntaxObject::default(TokenType::Lambda), + ))), + int(10), + ]))], + ); + } + + #[test] + fn test_let() { + assert_parse( + "(let ([a 10]) (+ a 20))", + &[ExprKind::List(List::new(vec![ + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("a")], + ExprKind::List(List::new(vec![atom("+"), atom("a"), int(20)])), + SyntaxObject::default(TokenType::Let), + ))), + int(10), + ]))], + ) + } + + #[test] + fn test_quote_with_inner_nested() { + assert_parse( + "'(#f '())", + &[ExprKind::Quote( + Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default( + TokenType::BooleanLiteral(false), + ))), + ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ])), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + )], + ) + } + + #[test] + fn test_quote_with_inner_nested_sub_expr() { + assert_parse( + "(if (null? contents) + '(#f '()) + (list (car contents) (cdr contents)))", + &[ExprKind::If(Box::new(If::new( + ExprKind::List(List::new(vec![atom("null?"), atom("contents")])), + ExprKind::Quote( + Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default( + TokenType::BooleanLiteral(false), + ))), + ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ])), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ExprKind::List(List::new(vec![ + atom("list"), + ExprKind::List(List::new(vec![atom("car"), atom("contents")])), + ExprKind::List(List::new(vec![atom("cdr"), atom("contents")])), + ])), + SyntaxObject::default(TokenType::If), + )))], + ); + } + + #[test] + fn test_quote_normal_with_inner_nested_sub_expr() { + assert_parse( + "(if (null? contents) + (quote (#f '())) + (list (car contents) (cdr contents)))", + &[ExprKind::If(Box::new(If::new( + ExprKind::List(List::new(vec![atom("null?"), atom("contents")])), + ExprKind::Quote( + Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default( + TokenType::BooleanLiteral(false), + ))), + ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ])), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ExprKind::List(List::new(vec![ + atom("list"), + ExprKind::List(List::new(vec![atom("car"), atom("contents")])), + ExprKind::List(List::new(vec![atom("cdr"), atom("contents")])), + ])), + SyntaxObject::default(TokenType::If), + )))], + ); + } + + #[test] + fn test_quote_with_inner_sub_expr_even_more_nested() { + assert_parse( + "(list + (if (null? contents) + '(#f '()) + (list (car contents) (cdr contents))))", + &[ExprKind::List(List::new(vec![ + atom("list"), + ExprKind::If(Box::new(If::new( + ExprKind::List(List::new(vec![atom("null?"), atom("contents")])), + ExprKind::Quote( + Quote::new( + ExprKind::List(List::new(vec![ + ExprKind::Atom(Atom::new(SyntaxObject::default( + TokenType::BooleanLiteral(false), + ))), + ExprKind::Quote( + Quote::new( + List::new(vec![]).into(), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ])), + SyntaxObject::default(TokenType::Quote), + ) + .into(), + ), + ExprKind::List(List::new(vec![ + atom("list"), + ExprKind::List(List::new(vec![atom("car"), atom("contents")])), + ExprKind::List(List::new(vec![atom("cdr"), atom("contents")])), + ])), + SyntaxObject::default(TokenType::If), + ))), + ]))], + ); + } + + #[test] + fn test_define_with_datum_syntax_name() { + assert_parse( + "(define (datum->syntax var) (car ret-value))", + &[ExprKind::Define(Box::new(Define::new( + ExprKind::List(List::new(vec![atom("datum->syntax"), atom("var")])), + ExprKind::List(List::new(vec![atom("car"), atom("ret-value")])), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_define_with_datum_syntax_function_name() { + assert_parse( + "(define ((datum->syntax var) arg) 10)", + &[ExprKind::Define(Box::new(Define::new( + ExprKind::List(List::new(vec![atom("datum->syntax"), atom("var")])), + ExprKind::LambdaFunction(Box::new(LambdaFunction::new( + vec![atom("arg")], + int(10), + SyntaxObject::default(TokenType::Lambda), + ))), + SyntaxObject::default(TokenType::Define), + )))], + ) + } + + #[test] + fn test_parse_without_lowering_ast() { + let a: Result> = + Parser::new_flat("(define (quote a) 10) (require foo bar)", None) + .map(|x| x.and_then(lower_macro_and_require_definitions)) + .collect(); + + let a = a.unwrap(); + + println!("{:#?}", a); + } + + #[test] + fn test_delayed_lowering() { + let a: Result> = Parser::new_flat( + r#" + ;; (define foo (quote a)) (require foo bar) + + ;; (define (foo) (if 10 20 30)) + + ;; (define (foo) (quote (define 10 20))) + + ;; (define foo '()) + + (define-syntax + with-handler + (syntax-rules + () + ((with-handler handler expr) + (reset + (call-with-exception-handler + (λ (err) + (begin (handler err) (shift k (k void)))) + (λ () + expr)))) + ((with-handler handler expr ...) + (reset + (call-with-exception-handler + (λ (err) + (begin (handler err) (shift k (k void)))) + (λ () + (begin expr ...))))))) + + + + "#, + None, + ) + .map(|x| x.and_then(lower_macro_and_require_definitions)) + .map(|x| x.and_then(lower_entire_ast)) + .collect(); + + let a = a.unwrap(); + + println!("{:#?}", a); + } +} diff --git a/crates/steel-values/Cargo.toml b/crates/steel-values/Cargo.toml new file mode 100644 index 000000000..fd950a0b1 --- /dev/null +++ b/crates/steel-values/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "steel-values" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/crates/steel-values/src/lib.rs b/crates/steel-values/src/lib.rs new file mode 100644 index 000000000..7d12d9af8 --- /dev/null +++ b/crates/steel-values/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} diff --git a/crates/steel-webrequests/src/lib.rs b/crates/steel-webrequests/src/lib.rs index 4d460c31d..51662a897 100644 --- a/crates/steel-webrequests/src/lib.rs +++ b/crates/steel-webrequests/src/lib.rs @@ -15,8 +15,10 @@ fn create_module() -> FFIModule { .register_fn("get", get) .register_fn( "call", - |request: Request| -> Result { - Request::call(request).map(|x| x.into()) + |request: BlockingRequest| -> Result { + Request::call(request.0) + .map(|x| x.into()) + .map_err(BlockingError::Ureq) }, ) .register_fn("response->text", SteelResponse::into_text); @@ -24,6 +26,7 @@ fn create_module() -> FFIModule { module } +#[derive(Clone)] struct BlockingRequest(Request); struct BlockingResponse(Response); @@ -36,8 +39,8 @@ impl Custom for BlockingRequest {} impl Custom for BlockingResponse {} impl Custom for BlockingError {} -fn get(url: String) -> Request { - ureq::get(&url) +fn get(url: String) -> BlockingRequest { + BlockingRequest(ureq::get(&url)) } struct SteelResponse { diff --git a/crates/steel-websockets/src/lib.rs b/crates/steel-websockets/src/lib.rs index 9ca7b1b40..8ca2e5e43 100644 --- a/crates/steel-websockets/src/lib.rs +++ b/crates/steel-websockets/src/lib.rs @@ -105,12 +105,14 @@ fn create_module() -> FFIModule { .register_fn("ws/message-ping->pong", ping_to_pong) .register_fn("ws/message->text-payload", text_payload) .register_fn("ws/connect", |url: String| { - connect(url).map(|(socket, resp)| { - vec![ - SteelWebSocket(socket).into_ffi_val().unwrap(), - WebSocketResponse(resp).into_ffi_val().unwrap(), - ] - }) + connect(url) + .map(|(socket, resp)| { + vec![ + SteelWebSocket(socket).into_ffi_val().unwrap(), + WebSocketResponse(resp).into_ffi_val().unwrap(), + ] + }) + .map_err(WebSocketError) }) .register_fn("ws/read-message!", SteelWebSocket::read_message) .register_fn("ws/write-message!", SteelWebSocket::write_message);