From 5f669059a8466dc98ad35563e61fdf93b9bdf1df Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sat, 9 Nov 2024 15:01:55 +0700 Subject: [PATCH 01/31] skip trivial first tokens in parsing --- crates/parser/src/grammar.rs | 12 ++ crates/parser/src/input.rs | 48 ++++- crates/parser/src/parser.rs | 2 +- crates/parser/src/token_kind.rs | 4 +- crates/syntax/src/lib.rs | 2 + crates/syntax/src/syntax.rs | 330 ++++++++++++++++++----------- crates/syntax/src/test_programs.rs | 135 ++++++++++++ 7 files changed, 402 insertions(+), 131 deletions(-) create mode 100644 crates/syntax/src/test_programs.rs diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 63d498f..98832cc 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -18,10 +18,22 @@ mod template; pub mod entry { + use crate::token_kind::TokenKind; + use super::*; pub fn circom_program(p: &mut Parser) { let m = p.open(); + + while p.at_any(&[ + TokenKind::BlockComment, + TokenKind::CommentLine, + TokenKind::EndLine, + TokenKind::WhiteSpace, + ]) { + p.skip(); + } + pragma::pragma(p); while !p.eof() { match p.current() { diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index 0819515..c8e17c8 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -4,7 +4,7 @@ use logos::Lexer; use crate::token_kind::TokenKind; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct Input<'a> { kind: Vec, source: &'a str, @@ -71,7 +71,9 @@ impl<'a> Input<'a> { #[cfg(test)] mod tests { - use std::cmp::min; + // use std::cmp::min; + + use crate::token_kind::TokenKind; use super::Input; @@ -83,11 +85,45 @@ mod tests { "# .to_string(); + let expected_input = Input { + kind: vec![ + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::BlockComment, + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::Identifier, + TokenKind::WhiteSpace, + TokenKind::Add, + TokenKind::WhiteSpace, + TokenKind::Number, + TokenKind::EndLine, + TokenKind::WhiteSpace + ], + source: &source, + position: vec![ + {0..1}, + {1..9}, + {9..24}, + {24..25}, + {25..33}, + {33..34}, + {34..35}, + {35..36}, + {36..37}, + {37..39}, + {39..40}, + {40..44}, + ] + }; + let input = Input::new(&source); - for i in 0..min(input.size(), 10) { - println!("kind = {:?}", input.kind[i]); - println!("position {:?}", input.position[i]); - } + assert_eq!(expected_input, input, "Tokens extract from source code are not correct"); + + // for i in 0..min(input.size(), 10) { + // println!("kind = {:?}", input.kind[i]); + // println!("position {:?}", input.position[i]); + // } } } diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index e3d461c..23c4dde 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -113,7 +113,7 @@ impl<'a> Parser<'a> { let mut kind: TokenKind; loop { kind = self.input.kind_of(self.pos); - if !kind.is_travial() { + if !kind.is_trivial() { break; } diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs index 300028f..26145bf 100644 --- a/crates/parser/src/token_kind.rs +++ b/crates/parser/src/token_kind.rs @@ -248,10 +248,10 @@ impl TokenKind { pub fn is_declaration_kw(self) -> bool { matches!(self, Self::VarKw | Self::ComponentKw | Self::SignalKw) } - pub fn is_travial(self) -> bool { + pub fn is_trivial(self) -> bool { matches!( self, - Self::WhiteSpace | Self::EndLine | Self::CommentLine | Self::Error + Self::WhiteSpace | Self::EndLine | Self::CommentLine | Self::BlockComment | Self::Error ) } } diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index 7ca0f92..adeb8b0 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -2,3 +2,5 @@ pub mod syntax; pub mod syntax_node; pub mod abstract_syntax_tree; + +pub mod test_programs; \ No newline at end of file diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 78c0ab2..3483b61 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -45,8 +45,10 @@ impl<'a> SyntaxTreeBuilder<'a> { pub fn syntax_tree(source: &str) -> SyntaxNode { let input = Input::new(source); - let mut builder = SyntaxTreeBuilder::new(&input); + let output = Parser::parsing(&input); + + let mut builder = SyntaxTreeBuilder::new(&input); builder.build(output); let green = builder.finish(); SyntaxNode::new_root(green) @@ -55,35 +57,127 @@ impl<'a> SyntaxTreeBuilder<'a> { #[cfg(test)] mod tests { - + use parser::token_kind::TokenKind::{self, *}; use std::hash::{DefaultHasher, Hash, Hasher}; - use rowan::ast::AstNode; + use rowan::{ast::AstNode, TextRange}; - use crate::abstract_syntax_tree::AstCircomProgram; + use crate::{abstract_syntax_tree::AstCircomProgram, test_programs}; use super::SyntaxTreeBuilder; + fn generate_expected_token_kind(ast: &AstCircomProgram) { + let children = ast + .syntax() + .first_child() + .unwrap() + .siblings(rowan::Direction::Next); + + println!("vec!["); + for child in children { + println!("{:?},", child.kind()); + } + println!("];"); + } + + fn generate_expected_token_range(ast: &AstCircomProgram) { + let children = ast + .syntax() + .first_child() + .unwrap() + .siblings(rowan::Direction::Next); + + println!("vec!["); + for child in children { + println!( + "TextRange::new({:?}.into(), {:?}.into()), ", + child.text_range().start(), + child.text_range().end() + ); + } + println!("];"); + } + + fn check_ast_children( + ast: &AstCircomProgram, + expected_kinds: &Vec, + expected_ranges: &Vec, + ) { + let children = ast + .syntax() + .first_child() + .unwrap() + .siblings(rowan::Direction::Next); + + let mut kind_iterator = expected_kinds.iter(); + let mut range_iterator = expected_ranges.iter(); + + for child in children { + if let (Some(expected_kind), Some(expected_range)) = + (kind_iterator.next(), range_iterator.next()) + { + assert_eq!(child.kind(), *expected_kind); + assert_eq!(child.text_range(), *expected_range); + } else { + panic!("Mismatched number of children and expected values"); + } + } + println!(); + } + #[test] - fn other_parser_test() { - let source: String = r#"pragma circom 2.0.0; + fn parser_test_1() { + let source: &str = test_programs::PARSER_TEST_1; + + let expected_pragma = "pragma circom 2.0.0;".to_string(); + let expected_kinds = vec![ + Pragma, + EndLine, + EndLine, + WhiteSpace, + EndLine, + WhiteSpace, + TemplateDef, + EndLine, + WhiteSpace, + TemplateDef, + WhiteSpace, + EndLine, + WhiteSpace, + ]; + let expected_ranges = vec![ + TextRange::new(0.into(), 20.into()), + TextRange::new(20.into(), 21.into()), + TextRange::new(21.into(), 22.into()), + TextRange::new(22.into(), 26.into()), + TextRange::new(26.into(), 27.into()), + TextRange::new(27.into(), 31.into()), + TextRange::new(31.into(), 57.into()), + TextRange::new(57.into(), 58.into()), + TextRange::new(58.into(), 62.into()), + TextRange::new(62.into(), 88.into()), + TextRange::new(88.into(), 89.into()), + TextRange::new(89.into(), 90.into()), + TextRange::new(90.into(), 94.into()), + ]; + + let syntax = SyntaxTreeBuilder::syntax_tree(source); - template Multiplier2 () {} - template Multiplier2 () {} - "# - .to_string(); + if let Some(ast) = AstCircomProgram::cast(syntax) { + check_ast_children(&ast, &expected_kinds, &expected_ranges); - let syntax = SyntaxTreeBuilder::syntax_tree(&source); + // check pragma + let pragma = ast.pragma().unwrap().syntax().text().to_string(); + assert_eq!(pragma, expected_pragma, "Pragma is not correct!"); - if let Some(ast) = AstCircomProgram::cast(syntax) { + // check ast hash let mut hasher = DefaultHasher::default(); ast.syntax().hash(&mut hasher); - // println!("{:#?}", syntax); - println!("{:?}", hasher.finish()); + let _ast_hash = hasher.finish(); + // check template hash let mut h1 = DefaultHasher::default(); - let mut h2 = DefaultHasher::default(); let template = ast.template_list(); @@ -91,124 +185,116 @@ mod tests { template[0].syntax().hash(&mut h1); template[1].syntax().hash(&mut h2); - println!("{}", h1.finish()); - println!("{}", h2.finish()); - println!("{:?}", template[0].syntax().text()); - println!("{:?}", template[1].syntax().text()); - println!("{}", template[0].syntax() == template[0].syntax()); - println!( - "{}", - template[0].syntax().green() == template[1].syntax().green() + assert_ne!( + h1.finish(), + h2.finish(), + "Templates with same syntax should have different hashes!" + ); + + // check template syntax (text & green node) + assert_eq!( + template[0].syntax().text(), + template[1].syntax().text(), + "The syntax (as text) of template 1 and 2 must be the same!" + ); + assert_eq!( + template[0].syntax().green(), + template[1].syntax().green(), + "The syntax (as green node) of template 1 and 2 must be the same!!" ); } + } + + #[test] + fn parser_test_2() { + let source = test_programs::PARSER_TEST_2; + + let syntax = SyntaxTreeBuilder::syntax_tree(source); + + if let Some(ast) = AstCircomProgram::cast(syntax) { + // print_ast_children(&ast); + + println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); - // find token + print!("Templates: "); + let templates = ast.template_list(); + for template in templates.iter() { + // print!("{:?} ", template.name().unwrap().name().unwrap().syntax().text()); + print!("{:?} ", template.name().unwrap().syntax().text()); // leading whitespaces + // print!("{:?} ", template.syntax().text()); // leading whitespaces + } + println!(); + + print!("Functions: "); + let functions = ast.function_list(); + for function in functions.iter() { + print!("{:?} ", function.function_name().unwrap().syntax().text()); + // leading whitespaces + // print!("{:?} ", function.syntax().text()); // leading whitespaces + } + println!(); + } } #[test] - fn parser_test() { - let source = r#"/* - Copyright 2018 0KIMS association. - - This file is part of circom (Zero Knowledge Circuit Compiler). - - circom is a free software: you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - circom is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public - License for more details. - - You should have received a copy of the GNU General Public License - along with circom. If not, see . - */ - /* - - Binary Sum - ========== - - This component creates a binary sum componet of ops operands and n bits each operand. - - e is Number of carries: Depends on the number of operands in the input. - - Main Constraint: - in[0][0] * 2^0 + in[0][1] * 2^1 + ..... + in[0][n-1] * 2^(n-1) + - + in[1][0] * 2^0 + in[1][1] * 2^1 + ..... + in[1][n-1] * 2^(n-1) + - + .. - + in[ops-1][0] * 2^0 + in[ops-1][1] * 2^1 + ..... + in[ops-1][n-1] * 2^(n-1) + - === - out[0] * 2^0 + out[1] * 2^1 + + out[n+e-1] *2(n+e-1) - - To waranty binary outputs: - - out[0] * (out[0] - 1) === 0 - out[1] * (out[0] - 1) === 0 - . - . - . - out[n+e-1] * (out[n+e-1] - 1) == 0 - - */ - - - /* - This function calculates the number of extra bits in the output to do the full sum. - */ - pragma circom 2.0.0; - - function nbits(a) { - var n = 1; - var r = 0; - while (n-1> k) & 1; - - // Ensure out is binary - out[k] * (out[k] - 1) === 0; - - lout += out[k] * e2; - - e2 = e2+e2; + } + + #[test] + fn parser_test_5() { + let source = test_programs::PARSER_TEST_5; + + let syntax = SyntaxTreeBuilder::syntax_tree(source); + + if let Some(ast) = AstCircomProgram::cast(syntax) { + // print_ast_children(&ast); + + println!("{:?}", ast.pragma()); + // assert!(ast.pragma().is_none(), "No pragma in source code"); } - - // Ensure the sum; - - lin === lout; } - "#; - let _syntax = SyntaxTreeBuilder::syntax_tree(source); + #[test] + fn parser_test_6() { + let source = test_programs::PARSER_TEST_6; + + let syntax = SyntaxTreeBuilder::syntax_tree(source); + + if let Some(ast) = AstCircomProgram::cast(syntax) { + // print_ast_children(&ast); + + println!("{:?}", ast.pragma()); + // assert!(ast.pragma().is_none(), "No pragma in source code"); + } } } diff --git a/crates/syntax/src/test_programs.rs b/crates/syntax/src/test_programs.rs new file mode 100644 index 0000000..0879679 --- /dev/null +++ b/crates/syntax/src/test_programs.rs @@ -0,0 +1,135 @@ +pub const PARSER_TEST_1: &str = r#"pragma circom 2.0.0; + + + template Multiplier2 () {} + template Multiplier2 () {} + "#; + +pub const PARSER_TEST_2: &str = r#"/* + Copyright 2018 0KIMS association. + + This file is part of circom (Zero Knowledge Circuit Compiler). + + circom is a free software: you can redistribute it and/or modify it + under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + circom is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public + License for more details. + + You should have received a copy of the GNU General Public License + along with circom. If not, see . +*/ +/* + +Binary Sum +========== + +This component creates a binary sum componet of ops operands and n bits each operand. + +e is Number of carries: Depends on the number of operands in the input. + +Main Constraint: + in[0][0] * 2^0 + in[0][1] * 2^1 + ..... + in[0][n-1] * 2^(n-1) + + + in[1][0] * 2^0 + in[1][1] * 2^1 + ..... + in[1][n-1] * 2^(n-1) + + + .. + + in[ops-1][0] * 2^0 + in[ops-1][1] * 2^1 + ..... + in[ops-1][n-1] * 2^(n-1) + + === + out[0] * 2^0 + out[1] * 2^1 + + out[n+e-1] *2(n+e-1) + +To waranty binary outputs: + + out[0] * (out[0] - 1) === 0 + out[1] * (out[0] - 1) === 0 + . + . + . + out[n+e-1] * (out[n+e-1] - 1) == 0 + + */ + + +/* + This function calculates the number of extra bits in the output to do the full sum. + */ + pragma circom 2.0.0; + +function nbits(a) { + var n = 1; + var r = 0; + while (n-1> k) & 1; + + // Ensure out is binary + out[k] * (out[k] - 1) === 0; + + lout += out[k] * e2; + + e2 = e2+e2; + } + + // Ensure the sum; + + lin === lout; + } + "#; + +pub const PARSER_TEST_3: &str = r#" + +// comment :> + + pragma circom 2.0.0; + + "#; + +pub const PARSER_TEST_4: &str = r#" + +/* +comment +blocks +*/ +pragma circom 2.0.0; + "#; + +pub const PARSER_TEST_5: &str = r#" +// no pragma here + template Multiplier2 () {} + "#; + +pub const PARSER_TEST_6: &str = r#" +/* T _ T */ + template Multiplier2 () {} + "#; + \ No newline at end of file From 6913c55cf23105829d5cf8f4b956f38775d17546 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Wed, 13 Nov 2024 13:20:34 +0700 Subject: [PATCH 02/31] remove comment in input test --- crates/parser/src/input.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index c8e17c8..cdfd5d8 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -120,10 +120,5 @@ mod tests { let input = Input::new(&source); assert_eq!(expected_input, input, "Tokens extract from source code are not correct"); - - // for i in 0..min(input.size(), 10) { - // println!("kind = {:?}", input.kind[i]); - // println!("position {:?}", input.position[i]); - // } } } From caac481857e878c4dd3113c2ca2e8c1d345d1dd7 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Mon, 18 Nov 2024 19:55:59 +0700 Subject: [PATCH 03/31] manage out-of-bound case, update input test --- crates/parser/src/input.rs | 297 +++++++++++++++++++++---------------- 1 file changed, 173 insertions(+), 124 deletions(-) diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index cdfd5d8..bb37613 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -1,124 +1,173 @@ -use std::ops::Range; - -use logos::Lexer; - -use crate::token_kind::TokenKind; - -#[derive(Debug, PartialEq)] -pub struct Input<'a> { - kind: Vec, - source: &'a str, - position: Vec>, -} - -impl<'a> Input<'a> { - pub fn new(source: &'a str) -> Self { - let mut input = Input { - source, - kind: Vec::new(), - position: Vec::new(), - }; - - let mut lex = Lexer::::new(source); - - while let Some(tk) = lex.next() { - if tk == TokenKind::CommentBlockOpen { - let mut closed = false; - let mut join_span = lex.span(); - while let Some(t) = lex.next() { - join_span.end = lex.span().end; - if t == TokenKind::CommentBlockClose { - closed = true; - break; - } - } - - if closed { - input.kind.push(TokenKind::BlockComment); - } else { - input.kind.push(TokenKind::Error); - } - input.position.push(join_span); - } else { - input.kind.push(tk); - input.position.push(lex.span()); - } - } - - input - } - - pub fn token_value(&self, index: usize) -> &'a str { - &self.source[self.position[index].start..self.position[index].end] - } - - pub fn kind_of(&self, index: usize) -> TokenKind { - if index < self.kind.len() { - self.kind[index] - } else { - TokenKind::EOF - } - } - - pub fn position_of(&self, index: usize) -> Range { - self.position[index].clone() - } - - pub fn size(&self) -> usize { - self.kind.len() - } -} - -#[cfg(test)] -mod tests { - // use std::cmp::min; - - use crate::token_kind::TokenKind; - - use super::Input; - - #[test] - fn test_input() { - let source = r#" - /*a + b == 10*/ - a + 10 - "# - .to_string(); - - let expected_input = Input { - kind: vec![ - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::BlockComment, - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::Identifier, - TokenKind::WhiteSpace, - TokenKind::Add, - TokenKind::WhiteSpace, - TokenKind::Number, - TokenKind::EndLine, - TokenKind::WhiteSpace - ], - source: &source, - position: vec![ - {0..1}, - {1..9}, - {9..24}, - {24..25}, - {25..33}, - {33..34}, - {34..35}, - {35..36}, - {36..37}, - {37..39}, - {39..40}, - {40..44}, - ] - }; - - let input = Input::new(&source); - - assert_eq!(expected_input, input, "Tokens extract from source code are not correct"); - } -} +use std::ops::Range; + +use logos::Lexer; + +use crate::token_kind::TokenKind; + +#[derive(Debug, PartialEq)] +pub struct Input<'a> { + kind: Vec, + source: &'a str, + position: Vec>, +} + +impl<'a> Input<'a> { + pub fn new(source: &'a str) -> Self { + let mut input = Input { + source, + kind: Vec::new(), + position: Vec::new(), + }; + + let mut lex = Lexer::::new(source); + + while let Some(tk) = lex.next() { + if tk == TokenKind::CommentBlockOpen { + let mut closed = false; + let mut join_span = lex.span(); + while let Some(t) = lex.next() { + join_span.end = lex.span().end; + if t == TokenKind::CommentBlockClose { + closed = true; + break; + } + } + + if closed { + input.kind.push(TokenKind::BlockComment); + } else { + input.kind.push(TokenKind::Error); + } + input.position.push(join_span); + } else { + input.kind.push(tk); + input.position.push(lex.span()); + } + } + + input + } + + pub fn token_value(&self, index: usize) -> &'a str { + if index < self.kind.len() { + &self.source[self.position[index].start..self.position[index].end] + } else { + // return error for out of bound index + "" + } + } + + pub fn kind_of(&self, index: usize) -> TokenKind { + if index < self.kind.len() { + self.kind[index] + } else { + TokenKind::EOF + } + } + + pub fn position_of(&self, index: usize) -> Range { + if index < self.kind.len() { + self.position[index].clone() + } else { + // return error for out of bound index + 0..0 + } + + } + + pub fn size(&self) -> usize { + self.kind.len() + } +} + +#[cfg(test)] +mod tests { + use crate::token_kind::TokenKind; + + use super::Input; + + #[test] + fn test_input() { + let source = r#" + /*a + b == 10*/ + a + 10 + "# + .to_string(); + + let expected_input = Input { + kind: vec![ + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::BlockComment, + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::Identifier, + TokenKind::WhiteSpace, + TokenKind::Add, + TokenKind::WhiteSpace, + TokenKind::Number, + TokenKind::EndLine, + TokenKind::WhiteSpace + ], + source: &source, + position: vec![ + {0..1}, + {1..9}, + {9..24}, + {24..25}, + {25..33}, + {33..34}, + {34..35}, + {35..36}, + {36..37}, + {37..39}, + {39..40}, + {40..44}, + ] + }; + + let input = Input::new(&source); + + assert_eq!(expected_input, input, "Tokens extract from source code are not correct"); + + // test size method + let expected_size = input.kind.len(); + let size = input.size(); + assert_eq!(expected_size, size, "size method failed"); + + // test methods with index out of bound + let index = input.kind.len(); + + let expected_token_value = ""; + let token_value = input.token_value(index); + assert_eq!(expected_token_value, token_value, "token_value failed (case: index out of bound)"); + + let expected_kind = TokenKind::EOF; + let kind = input.kind_of(index); + assert_eq!(expected_kind, kind, "kind_of failed (case: index out of bound)"); + + let expected_position = 0..0; + let position = input.position_of(index); + assert_eq!(expected_position, position, "position_of failed (case: index out of bound)"); + + // test methods with index in bound + if input.size() == 0 { + return; + } + + let index = input.size() / 2; // a valid index if input size > 0 + + let expected_token_value = &input.source[input.position[index].clone()]; + let token_value = input.token_value(index); + assert_eq!(expected_token_value, token_value, "token_value failed"); + + let expected_kind = input.kind[index]; + let kind = input.kind_of(index); + assert_eq!(expected_kind, kind, "kind_of failed"); + + let expected_position = input.position[index].clone(); + let position = input.position_of(index); + assert_eq!(expected_position, position, "position_of failed"); + + } +} From 8efa4f70d57a52e24b92ed4c5002ec339ed77095 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Mon, 18 Nov 2024 21:30:38 +0700 Subject: [PATCH 04/31] update input test --- crates/parser/src/input.rs | 145 +++++++++++++++++++++++++++++++------ 1 file changed, 123 insertions(+), 22 deletions(-) diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index bb37613..ebc1b9e 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -72,7 +72,6 @@ impl<'a> Input<'a> { // return error for out of bound index 0..0 } - } pub fn size(&self) -> usize { @@ -87,7 +86,7 @@ mod tests { use super::Input; #[test] - fn test_input() { + fn test_input_1() { let source = r#" /*a + b == 10*/ a + 10 @@ -107,27 +106,121 @@ mod tests { TokenKind::WhiteSpace, TokenKind::Number, TokenKind::EndLine, - TokenKind::WhiteSpace + TokenKind::WhiteSpace, ], source: &source, position: vec![ - {0..1}, - {1..9}, - {9..24}, - {24..25}, - {25..33}, - {33..34}, - {34..35}, - {35..36}, - {36..37}, - {37..39}, - {39..40}, - {40..44}, - ] + { 0..1 }, + { 1..9 }, + { 9..24 }, + { 24..25 }, + { 25..33 }, + { 33..34 }, + { 34..35 }, + { 35..36 }, + { 36..37 }, + { 37..39 }, + { 39..40 }, + { 40..44 }, + ], }; let input = Input::new(&source); + assert_eq!( + expected_input, input, + "Tokens extract from source code are not correct" + ); + + // test size method + let expected_size = input.kind.len(); + let size = input.size(); + assert_eq!(expected_size, size, "size method failed"); + + // test methods with index out of bound + let index = input.kind.len(); + + let expected_token_value = ""; + let token_value = input.token_value(index); + assert_eq!( + expected_token_value, token_value, + "token_value failed (case: index out of bound)" + ); + + let expected_kind = TokenKind::EOF; + let kind = input.kind_of(index); + assert_eq!( + expected_kind, kind, + "kind_of failed (case: index out of bound)" + ); + + let expected_position = 0..0; + let position = input.position_of(index); + assert_eq!( + expected_position, position, + "position_of failed (case: index out of bound)" + ); + + // test methods with index in bound + if input.size() == 0 { + return; + } + + let index = input.size() / 2; // a valid index if input size > 0 + + let expected_token_value = &input.source[input.position[index].clone()]; + let token_value = input.token_value(index); + assert_eq!(expected_token_value, token_value, "token_value failed"); + + let expected_kind = input.kind[index]; + let kind = input.kind_of(index); + assert_eq!(expected_kind, kind, "kind_of failed"); + + let expected_position = input.position[index].clone(); + let position = input.position_of(index); + assert_eq!(expected_position, position, "position_of failed"); + } + + #[test] + fn test_input_2() { + let source = r#" + pragma 2.1.1; + /*a + b == 10* + a + 10 + template + + /* + "# + .to_string(); + + let expected_input = Input { + kind: vec![ + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::Pragma, + TokenKind::WhiteSpace, + TokenKind::Version, + TokenKind::Semicolon, + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::Error, + ], + source: &source, + position: vec![ + 0..1, + 1..9, + 9..15, + 15..16, + 16..21, + 21..22, + 22..23, + 23..31, + 31..94, + ], + }; + + let input = Input::new(&source); + assert_eq!(expected_input, input, "Tokens extract from source code are not correct"); // test size method @@ -140,15 +233,24 @@ mod tests { let expected_token_value = ""; let token_value = input.token_value(index); - assert_eq!(expected_token_value, token_value, "token_value failed (case: index out of bound)"); + assert_eq!( + expected_token_value, token_value, + "token_value failed (case: index out of bound)" + ); let expected_kind = TokenKind::EOF; let kind = input.kind_of(index); - assert_eq!(expected_kind, kind, "kind_of failed (case: index out of bound)"); + assert_eq!( + expected_kind, kind, + "kind_of failed (case: index out of bound)" + ); let expected_position = 0..0; let position = input.position_of(index); - assert_eq!(expected_position, position, "position_of failed (case: index out of bound)"); + assert_eq!( + expected_position, position, + "position_of failed (case: index out of bound)" + ); // test methods with index in bound if input.size() == 0 { @@ -156,7 +258,7 @@ mod tests { } let index = input.size() / 2; // a valid index if input size > 0 - + let expected_token_value = &input.source[input.position[index].clone()]; let token_value = input.token_value(index); assert_eq!(expected_token_value, token_value, "token_value failed"); @@ -164,10 +266,9 @@ mod tests { let expected_kind = input.kind[index]; let kind = input.kind_of(index); assert_eq!(expected_kind, kind, "kind_of failed"); - + let expected_position = input.position[index].clone(); let position = input.position_of(index); assert_eq!(expected_position, position, "position_of failed"); - } } From c78761e2ae8378241608491964f93f81ac160b9b Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 6 Dec 2024 21:25:09 +0700 Subject: [PATCH 05/31] update input test --- crates/parser/src/input.rs | 285 ++++++++++++++++++++++---------- crates/parser/src/token_kind.rs | 3 + crates/syntax/src/syntax.rs | 12 +- 3 files changed, 209 insertions(+), 91 deletions(-) diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index ebc1b9e..4b1cc55 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -81,50 +81,11 @@ impl<'a> Input<'a> { #[cfg(test)] mod tests { - use crate::token_kind::TokenKind; + use crate::token_kind::TokenKind::{self, *}; use super::Input; - #[test] - fn test_input_1() { - let source = r#" - /*a + b == 10*/ - a + 10 - "# - .to_string(); - - let expected_input = Input { - kind: vec![ - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::BlockComment, - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::Identifier, - TokenKind::WhiteSpace, - TokenKind::Add, - TokenKind::WhiteSpace, - TokenKind::Number, - TokenKind::EndLine, - TokenKind::WhiteSpace, - ], - source: &source, - position: vec![ - { 0..1 }, - { 1..9 }, - { 9..24 }, - { 24..25 }, - { 25..33 }, - { 33..34 }, - { 34..35 }, - { 35..36 }, - { 36..37 }, - { 37..39 }, - { 39..40 }, - { 40..44 }, - ], - }; - + fn test(source: &str, expected_input: Input) { let input = Input::new(&source); assert_eq!( @@ -182,7 +143,49 @@ mod tests { } #[test] - fn test_input_2() { + fn test_comment_block() { + let source = r#" + /*a + b == 10*/ + a + 10 + "#; + + let expected_input = Input { + kind: vec![ + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::BlockComment, + TokenKind::EndLine, + TokenKind::WhiteSpace, + TokenKind::Identifier, + TokenKind::WhiteSpace, + TokenKind::Add, + TokenKind::WhiteSpace, + TokenKind::Number, + TokenKind::EndLine, + TokenKind::WhiteSpace, + ], + source: &source, + position: vec![ + { 0..1 }, + { 1..9 }, + { 9..24 }, + { 24..25 }, + { 25..33 }, + { 33..34 }, + { 34..35 }, + { 35..36 }, + { 36..37 }, + { 37..39 }, + { 39..40 }, + { 40..44 }, + ], + }; + + test(source, expected_input); + } + + #[test] + fn test_comment_error() { let source = r#" pragma 2.1.1; /*a + b == 10* @@ -190,8 +193,7 @@ mod tests { template /* - "# - .to_string(); + "#; let expected_input = Input { kind: vec![ @@ -219,56 +221,169 @@ mod tests { ], }; - let input = Input::new(&source); + test(source, expected_input); + } - assert_eq!(expected_input, input, "Tokens extract from source code are not correct"); + #[test] + fn test_pragma() { + let source = r#" + /* test pragma token kinds */ - // test size method - let expected_size = input.kind.len(); - let size = input.size(); - assert_eq!(expected_size, size, "size method failed"); + pragma circom 2.0.0; - // test methods with index out of bound - let index = input.kind.len(); + "#; - let expected_token_value = ""; - let token_value = input.token_value(index); - assert_eq!( - expected_token_value, token_value, - "token_value failed (case: index out of bound)" - ); - - let expected_kind = TokenKind::EOF; - let kind = input.kind_of(index); - assert_eq!( - expected_kind, kind, - "kind_of failed (case: index out of bound)" - ); + let expected_input = Input { + kind: vec![ + EndLine, + WhiteSpace, + BlockComment, + EndLine, + EndLine, + WhiteSpace, + Pragma, + WhiteSpace, + Circom, + WhiteSpace, + Version, + Semicolon, + EndLine, + EndLine, + WhiteSpace, + ], + source: &source, + position: vec![ + 0..1, + 1..9, + 9..38, + 38..39, + 39..40, + 40..44, + 44..50, + 50..51, + 51..57, + 57..58, + 58..63, + 63..64, + 64..65, + 65..66, + 66..70, + ], + }; - let expected_position = 0..0; - let position = input.position_of(index); - assert_eq!( - expected_position, position, - "position_of failed (case: index out of bound)" - ); + test(source, expected_input); + } - // test methods with index in bound - if input.size() == 0 { - return; + #[test] + fn test_function() { + let source = r#" + function nbits(a) { + var n = 1; + var r = 0; + while (n-1 0 + let expected_input = Input { + kind: vec![ + EndLine, WhiteSpace, FunctionKw, WhiteSpace, Identifier, LParen, Identifier, + RParen, WhiteSpace, LCurly, EndLine, WhiteSpace, VarKw, WhiteSpace, Identifier, + WhiteSpace, Assign, WhiteSpace, Number, Semicolon, EndLine, WhiteSpace, VarKw, + WhiteSpace, Identifier, WhiteSpace, Assign, WhiteSpace, Number, Semicolon, EndLine, + WhiteSpace, WhileKw, WhiteSpace, LParen, Identifier, Sub, Number, LessThan, + Identifier, RParen, WhiteSpace, LCurly, EndLine, WhiteSpace, Identifier, Add, Add, + Semicolon, EndLine, WhiteSpace, Identifier, WhiteSpace, Mul, Assign, WhiteSpace, + Number, Semicolon, EndLine, WhiteSpace, RCurly, EndLine, WhiteSpace, ReturnKw, + WhiteSpace, Identifier, Semicolon, EndLine, WhiteSpace, RCurly, + ], + source: &source, + position: vec![ + 0..1, + 1..5, + 5..13, + 13..14, + 14..19, + 19..20, + 20..21, + 21..22, + 22..23, + 23..24, + 24..25, + 25..33, + 33..36, + 36..37, + 37..38, + 38..39, + 39..40, + 40..41, + 41..42, + 42..43, + 43..44, + 44..52, + 52..55, + 55..56, + 56..57, + 57..58, + 58..59, + 59..60, + 60..61, + 61..62, + 62..63, + 63..71, + 71..76, + 76..77, + 77..78, + 78..79, + 79..80, + 80..81, + 81..82, + 82..83, + 83..84, + 84..85, + 85..86, + 86..87, + 87..99, + 99..100, + 100..101, + 101..102, + 102..103, + 103..104, + 104..116, + 116..117, + 117..118, + 118..119, + 119..120, + 120..121, + 121..122, + 122..123, + 123..124, + 124..132, + 132..133, + 133..134, + 134..142, + 142..148, + 148..149, + 149..150, + 150..151, + 151..152, + 152..156, + 156..157, + ], + }; - let expected_token_value = &input.source[input.position[index].clone()]; - let token_value = input.token_value(index); - assert_eq!(expected_token_value, token_value, "token_value failed"); + test(source, expected_input); + } - let expected_kind = input.kind[index]; - let kind = input.kind_of(index); - assert_eq!(expected_kind, kind, "kind_of failed"); + // #[test] + // fn test_gen() { + // let source = r#" + // "#; - let expected_position = input.position[index].clone(); - let position = input.position_of(index); - assert_eq!(expected_position, position, "position_of failed"); - } + // let input = Input::new(&source); + // println!("{:?}", input.kind); + // println!("{:?}", input.position); + // } } diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs index 26145bf..779191f 100644 --- a/crates/parser/src/token_kind.rs +++ b/crates/parser/src/token_kind.rs @@ -229,6 +229,7 @@ impl TokenKind { _ => None, } } + pub fn prefix(self) -> Option { match self { Self::Sub => Some(100), @@ -245,9 +246,11 @@ impl TokenKind { _ => None, } } + pub fn is_declaration_kw(self) -> bool { matches!(self, Self::VarKw | Self::ComponentKw | Self::SignalKw) } + pub fn is_trivial(self) -> bool { matches!( self, diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 3483b61..21b9734 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -126,7 +126,7 @@ mod tests { } #[test] - fn parser_test_1() { + fn syntax_test_1() { let source: &str = test_programs::PARSER_TEST_1; let expected_pragma = "pragma circom 2.0.0;".to_string(); @@ -206,7 +206,7 @@ mod tests { } #[test] - fn parser_test_2() { + fn syntax_test_2() { let source = test_programs::PARSER_TEST_2; let syntax = SyntaxTreeBuilder::syntax_tree(source); @@ -237,7 +237,7 @@ mod tests { } #[test] - fn parser_test_3() { + fn syntax_test_3() { let source = test_programs::PARSER_TEST_3; let syntax = SyntaxTreeBuilder::syntax_tree(source); @@ -254,7 +254,7 @@ mod tests { } #[test] - fn parser_test_4() { + fn syntax_test_4() { let source = test_programs::PARSER_TEST_4; let syntax = SyntaxTreeBuilder::syntax_tree(source); @@ -271,7 +271,7 @@ mod tests { } #[test] - fn parser_test_5() { + fn syntax_test_5() { let source = test_programs::PARSER_TEST_5; let syntax = SyntaxTreeBuilder::syntax_tree(source); @@ -285,7 +285,7 @@ mod tests { } #[test] - fn parser_test_6() { + fn syntax_test_6() { let source = test_programs::PARSER_TEST_6; let syntax = SyntaxTreeBuilder::syntax_tree(source); From bfd1f1519bba52be08a0b9032bdaa28edad5dff5 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sun, 8 Dec 2024 15:38:34 +0700 Subject: [PATCH 06/31] make test programs private --- crates/syntax/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index adeb8b0..d1300bb 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -3,4 +3,4 @@ pub mod syntax_node; pub mod abstract_syntax_tree; -pub mod test_programs; \ No newline at end of file +mod test_programs; \ No newline at end of file From e054b574fa54e0862006da06e0072a11789aa780 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sun, 8 Dec 2024 15:41:47 +0700 Subject: [PATCH 07/31] remove comments in syntax tests --- crates/syntax/src/syntax.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 21b9734..58d7194 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -277,9 +277,8 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - // print_ast_children(&ast); - - println!("{:?}", ast.pragma()); + println!("pragma: {:?}", ast.pragma()); + println!("template list: {:?}", ast.template_list()); // assert!(ast.pragma().is_none(), "No pragma in source code"); } } @@ -294,6 +293,8 @@ mod tests { // print_ast_children(&ast); println!("{:?}", ast.pragma()); + + println!("template list: {:?}", ast.template_list()); // assert!(ast.pragma().is_none(), "No pragma in source code"); } } From 15c9e5f7dfb801952f358f8e2d97036cb7928a67 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sun, 8 Dec 2024 15:45:28 +0700 Subject: [PATCH 08/31] format --- crates/parser/src/grammar.rs | 5 +++-- crates/parser/src/grammar/block.rs | 1 + crates/parser/src/grammar/declaration.rs | 3 +++ crates/parser/src/grammar/expression.rs | 1 + crates/parser/src/grammar/function.rs | 8 +++++--- crates/parser/src/grammar/include.rs | 2 +- crates/parser/src/grammar/list_identity.rs | 1 + crates/parser/src/grammar/main_component.rs | 7 +++++++ crates/parser/src/grammar/template.rs | 2 +- crates/parser/src/output.rs | 1 + crates/parser/src/token_kind.rs | 2 +- crates/syntax/src/syntax.rs | 10 ---------- 12 files changed, 25 insertions(+), 18 deletions(-) diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 98832cc..6b3791a 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -35,14 +35,15 @@ pub mod entry { } pragma::pragma(p); + while !p.eof() { match p.current() { TemplateKw => { template::template(p); - } + }, IncludeKw => { include::include(p); - } + }, ComponentKw => main_component::main_component(p), FunctionKw => function::function_parse(p), _ => { diff --git a/crates/parser/src/grammar/block.rs b/crates/parser/src/grammar/block.rs index 7b0f89a..5a40865 100644 --- a/crates/parser/src/grammar/block.rs +++ b/crates/parser/src/grammar/block.rs @@ -2,6 +2,7 @@ use super::*; pub fn block(p: &mut Parser) { p.inc_rcurly(); + if !p.at(LCurly) { p.advance_with_error("Miss {"); } else { diff --git a/crates/parser/src/grammar/declaration.rs b/crates/parser/src/grammar/declaration.rs index 1ebbb70..8022f81 100644 --- a/crates/parser/src/grammar/declaration.rs +++ b/crates/parser/src/grammar/declaration.rs @@ -3,6 +3,9 @@ use super::{ *, }; +// "signal" --> None +// "signal input" --> Some(true) +// "signal output" --> Some(false) fn signal_header(p: &mut Parser) -> Option { let mut res = None; let m = p.open(); diff --git a/crates/parser/src/grammar/expression.rs b/crates/parser/src/grammar/expression.rs index 8932828..8c7fad6 100644 --- a/crates/parser/src/grammar/expression.rs +++ b/crates/parser/src/grammar/expression.rs @@ -1,6 +1,7 @@ use crate::parser::Marker; use super::*; + pub(super) fn expression(p: &mut Parser) { let m = p.open(); circom_expression(p); diff --git a/crates/parser/src/grammar/function.rs b/crates/parser/src/grammar/function.rs index 24b60d3..fad1d44 100644 --- a/crates/parser/src/grammar/function.rs +++ b/crates/parser/src/grammar/function.rs @@ -1,12 +1,15 @@ use crate::grammar::*; +// fucntion name() pub fn function_parse(p: &mut Parser) { let m = p.open(); + p.expect(FunctionKw); - let fn_name_marker = p.open(); + let fn_name_marker = p.open(); p.expect(Identifier); p.close(fn_name_marker, FunctionName); + p.expect(LParen); let arg_marker = p.open(); while !p.at(RParen) && !p.eof() { @@ -15,11 +18,10 @@ pub fn function_parse(p: &mut Parser) { p.expect(Comma); } } - p.close(arg_marker, ParameterList); - p.expect(RParen); block::block(p); + p.close(m, FunctionDef); } diff --git a/crates/parser/src/grammar/include.rs b/crates/parser/src/grammar/include.rs index 7269995..d8f0ac7 100644 --- a/crates/parser/src/grammar/include.rs +++ b/crates/parser/src/grammar/include.rs @@ -2,7 +2,7 @@ use super::*; pub(super) fn include(p: &mut Parser) { // assert!(p.at(IncludeKw)); - + let m = p.open(); p.expect(IncludeKw); p.expect(CircomString); diff --git a/crates/parser/src/grammar/list_identity.rs b/crates/parser/src/grammar/list_identity.rs index 6a4effa..73f85c9 100644 --- a/crates/parser/src/grammar/list_identity.rs +++ b/crates/parser/src/grammar/list_identity.rs @@ -1,5 +1,6 @@ use super::*; +// a, b, c, d pub fn parse(p: &mut Parser) { while p.at(Identifier) && !p.eof() { p.expect(Identifier); diff --git a/crates/parser/src/grammar/main_component.rs b/crates/parser/src/grammar/main_component.rs index 538a73b..3f6f862 100644 --- a/crates/parser/src/grammar/main_component.rs +++ b/crates/parser/src/grammar/main_component.rs @@ -1,13 +1,20 @@ use super::*; +/* +component main {public [signal_list]} = tempid(v1,...,vn); + +{public [signal_list]} is optional +*/ pub fn main_component(p: &mut Parser) { p.expect(ComponentKw); p.expect(MainKw); + p.expect(LCurly); p.expect(PublicKw); p.expect(LBracket); list_identity::parse(p); p.expect(RBracket); + p.expect(Assign); expression::expression(p); } diff --git a/crates/parser/src/grammar/template.rs b/crates/parser/src/grammar/template.rs index 9973366..9249b1f 100644 --- a/crates/parser/src/grammar/template.rs +++ b/crates/parser/src/grammar/template.rs @@ -1,7 +1,7 @@ use crate::grammar::*; /** * template Identifier() {content} - * + * template Identifier( param_1, ... , param_n ) { content } */ pub fn template(p: &mut Parser) { // assert!(p.at(TemplateKw)); diff --git a/crates/parser/src/output.rs b/crates/parser/src/output.rs index 0a4e07d..271c2c3 100644 --- a/crates/parser/src/output.rs +++ b/crates/parser/src/output.rs @@ -30,6 +30,7 @@ impl Output { &self.children } } + impl From> for Output { fn from(events: Vec) -> Self { let mut stack = Vec::new(); diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs index 779191f..9ebba15 100644 --- a/crates/parser/src/token_kind.rs +++ b/crates/parser/src/token_kind.rs @@ -229,7 +229,7 @@ impl TokenKind { _ => None, } } - + pub fn prefix(self) -> Option { match self { Self::Sub => Some(100), diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 58d7194..f44b88c 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -212,14 +212,11 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - // print_ast_children(&ast); - println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); print!("Templates: "); let templates = ast.template_list(); for template in templates.iter() { - // print!("{:?} ", template.name().unwrap().name().unwrap().syntax().text()); print!("{:?} ", template.name().unwrap().syntax().text()); // leading whitespaces // print!("{:?} ", template.syntax().text()); // leading whitespaces } @@ -243,8 +240,6 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - // print_ast_children(&ast); - println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); println!( "Pragma version: {:?}", @@ -260,8 +255,6 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - // print_ast_children(&ast); - println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); println!( "Pragma version: {:?}", @@ -290,10 +283,7 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - // print_ast_children(&ast); - println!("{:?}", ast.pragma()); - println!("template list: {:?}", ast.template_list()); // assert!(ast.pragma().is_none(), "No pragma in source code"); } From c76246b5251954ffb6e31f34294754eb5efcd327 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sun, 8 Dec 2024 16:04:32 +0700 Subject: [PATCH 09/31] make Pragma optional, remove ROOT --- crates/parser/src/grammar.rs | 31 +++++++------------------------ 1 file changed, 7 insertions(+), 24 deletions(-) diff --git a/crates/parser/src/grammar.rs b/crates/parser/src/grammar.rs index 6b3791a..a0c9f53 100644 --- a/crates/parser/src/grammar.rs +++ b/crates/parser/src/grammar.rs @@ -34,21 +34,14 @@ pub mod entry { p.skip(); } - pragma::pragma(p); - while !p.eof() { match p.current() { - TemplateKw => { - template::template(p); - }, - IncludeKw => { - include::include(p); - }, + Pragma => pragma::pragma(p), + TemplateKw => template::template(p), + IncludeKw => include::include(p), ComponentKw => main_component::main_component(p), FunctionKw => function::function_parse(p), - _ => { - p.advance_with_error("invalid token"); - } + _ => p.advance_with_error("invalid token"), } } p.close(m, CircomProgram); @@ -64,20 +57,10 @@ pub mod entry { impl Scope { pub fn parse(self, p: &mut Parser) { match self { - Self::Block => { - let m = p.open(); - block::block(p); - p.close(m, ROOT); - } + Self::Block => block::block(p), Self::CircomProgram => circom_program(p), - Self::Pragma => { - let m = p.open(); - pragma::pragma(p); - p.close(m, ROOT); - } - Self::Template => { - template::template(p); - } + Self::Pragma => pragma::pragma(p), + Self::Template => template::template(p), } } } From cd125abe3dee9663b84f010eeeff6775c10b08a9 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sun, 8 Dec 2024 16:07:16 +0700 Subject: [PATCH 10/31] rename close() params, use advance() in eat(), fix typo scope --- crates/parser/src/parser.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 23c4dde..a4f0efe 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -52,8 +52,8 @@ impl<'a> Parser<'a> { } } - pub fn close(&mut self, marker_close: Marker, kind: TokenKind) -> Marker { - match marker_close { + pub fn close(&mut self, marker_open: Marker, kind: TokenKind) -> Marker { + match marker_open { Marker::Open(index) => { self.events[index] = Event::Open { kind }; self.events.push(Event::Close); @@ -159,8 +159,7 @@ impl<'a> Parser<'a> { pub fn eat(&mut self, kind: TokenKind) -> bool { if self.at(kind) { - self.events.push(Event::TokenPosition(self.pos)); - self.skip(); + self.advance(); return true; } false @@ -193,7 +192,7 @@ impl<'a> Parser<'a> { } impl Parser<'_> { - pub fn parsing_with_scrope(input: &Input, scope: Scope) -> Output { + pub fn parsing_with_scope(input: &Input, scope: Scope) -> Output { let mut p = Parser::new(input); scope.parse(&mut p); Output::from(p.events) @@ -201,6 +200,6 @@ impl Parser<'_> { pub fn parsing(input: &Input) -> Output { let c = Scope::CircomProgram; - Parser::parsing_with_scrope(input, c) + Parser::parsing_with_scope(input, c) } } From 15470b1035bf20a855ae0be0d83f74ed275e2311 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sun, 8 Dec 2024 16:08:12 +0700 Subject: [PATCH 11/31] parse params using list_identity --- crates/parser/src/grammar/function.rs | 7 +------ crates/parser/src/grammar/template.rs | 12 +++++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/crates/parser/src/grammar/function.rs b/crates/parser/src/grammar/function.rs index fad1d44..bca7018 100644 --- a/crates/parser/src/grammar/function.rs +++ b/crates/parser/src/grammar/function.rs @@ -12,12 +12,7 @@ pub fn function_parse(p: &mut Parser) { p.expect(LParen); let arg_marker = p.open(); - while !p.at(RParen) && !p.eof() { - p.expect(Identifier); - if p.at(Comma) { - p.expect(Comma); - } - } + list_identity::parse(p); p.close(arg_marker, ParameterList); p.expect(RParen); diff --git a/crates/parser/src/grammar/template.rs b/crates/parser/src/grammar/template.rs index 9249b1f..be5f17e 100644 --- a/crates/parser/src/grammar/template.rs +++ b/crates/parser/src/grammar/template.rs @@ -6,23 +6,21 @@ use crate::grammar::*; pub fn template(p: &mut Parser) { // assert!(p.at(TemplateKw)); let m = p.open(); + p.expect(TemplateKw); + let name_marker = p.open(); p.expect(Identifier); p.close(name_marker, TemplateName); p.expect(LParen); let arg_marker = p.open(); - while !p.at(RParen) && !p.eof() { - p.expect(Identifier); - if p.at(Comma) { - p.expect(Comma); - } - } - + list_identity::parse(p); p.close(arg_marker, ParameterList); p.expect(RParen); + block::block(p); + p.close(m, TemplateDef); } From 6dbb51a333ccc075c49a609895bce1a32b5c7041 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 10 Dec 2024 22:40:00 +0700 Subject: [PATCH 12/31] return Option in token_value() and position_of() --- crates/parser/src/input.rs | 22 +++++++++++----------- crates/syntax/src/syntax.rs | 3 ++- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index 4b1cc55..8ddf602 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -48,12 +48,12 @@ impl<'a> Input<'a> { input } - pub fn token_value(&self, index: usize) -> &'a str { + pub fn token_value(&self, index: usize) -> Option<&'a str> { if index < self.kind.len() { - &self.source[self.position[index].start..self.position[index].end] + Some(&self.source[self.position[index].start..self.position[index].end]) } else { - // return error for out of bound index - "" + // return None for out of bound index + None } } @@ -65,12 +65,12 @@ impl<'a> Input<'a> { } } - pub fn position_of(&self, index: usize) -> Range { + pub fn position_of(&self, index: usize) -> Option> { if index < self.kind.len() { - self.position[index].clone() + Some(self.position[index].clone()) } else { // return error for out of bound index - 0..0 + None } } @@ -101,7 +101,7 @@ mod tests { // test methods with index out of bound let index = input.kind.len(); - let expected_token_value = ""; + let expected_token_value = None; let token_value = input.token_value(index); assert_eq!( expected_token_value, token_value, @@ -115,7 +115,7 @@ mod tests { "kind_of failed (case: index out of bound)" ); - let expected_position = 0..0; + let expected_position = None; let position = input.position_of(index); assert_eq!( expected_position, position, @@ -130,7 +130,7 @@ mod tests { let index = input.size() / 2; // a valid index if input size > 0 let expected_token_value = &input.source[input.position[index].clone()]; - let token_value = input.token_value(index); + let token_value = input.token_value(index).unwrap(); assert_eq!(expected_token_value, token_value, "token_value failed"); let expected_kind = input.kind[index]; @@ -138,7 +138,7 @@ mod tests { assert_eq!(expected_kind, kind, "kind_of failed"); let expected_position = input.position[index].clone(); - let position = input.position_of(index); + let position = input.position_of(index).unwrap(); assert_eq!(expected_position, position, "position_of failed"); } diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index f44b88c..32a273b 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -23,7 +23,8 @@ impl<'a> SyntaxTreeBuilder<'a> { match child { Child::Token(token_id) => { let token_kind = self.input.kind_of(*token_id); - let token_value = self.input.token_value(*token_id); + // TODO: return Error to replace .unwrap() + let token_value = self.input.token_value(*token_id).unwrap(); self.builder.start_node(token_kind.into()); self.builder.token(token_kind.into(), token_value); self.builder.finish_node(); From 6d5f0c6d44d8d8d15e8bfa66b68659dc4644552a Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 10 Dec 2024 22:51:11 +0700 Subject: [PATCH 13/31] do not allow <--, <== in var declaration --- crates/parser/src/grammar/declaration.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/parser/src/grammar/declaration.rs b/crates/parser/src/grammar/declaration.rs index 8022f81..c37630c 100644 --- a/crates/parser/src/grammar/declaration.rs +++ b/crates/parser/src/grammar/declaration.rs @@ -38,7 +38,7 @@ pub(super) fn var_declaration(p: &mut Parser) { if p.at(LParen) { tuple(p); - if p.at_any(&[Assign, RAssignSignal, RAssignConstraintSignal]) { + if p.at(Assign) { tuple_init(p); } } else { From fbdc5b9857cd6944c9b0df9cb892d44c00ece8dc Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 10 Dec 2024 22:54:53 +0700 Subject: [PATCH 14/31] make public signal optional in main component --- crates/parser/src/grammar/main_component.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/parser/src/grammar/main_component.rs b/crates/parser/src/grammar/main_component.rs index 3f6f862..d9c176d 100644 --- a/crates/parser/src/grammar/main_component.rs +++ b/crates/parser/src/grammar/main_component.rs @@ -9,11 +9,13 @@ pub fn main_component(p: &mut Parser) { p.expect(ComponentKw); p.expect(MainKw); - p.expect(LCurly); - p.expect(PublicKw); - p.expect(LBracket); - list_identity::parse(p); - p.expect(RBracket); + if p.at(LCurly) { + p.expect(LCurly); + p.expect(PublicKw); + p.expect(LBracket); + list_identity::parse(p); + p.expect(RBracket); + } p.expect(Assign); expression::expression(p); From 07c4171540a9fbb51df57caa9e2f09671b12b9f6 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 13 Dec 2024 14:32:06 +0700 Subject: [PATCH 15/31] fix format before merge --- crates/parser/src/grammar/block.rs | 2 +- crates/parser/src/grammar/function.rs | 2 +- crates/parser/src/grammar/include.rs | 2 +- crates/parser/src/grammar/main_component.rs | 2 +- crates/parser/src/grammar/template.rs | 2 +- crates/syntax/src/lib.rs | 5 ++--- crates/syntax/src/syntax.rs | 3 +-- crates/syntax/src/test_programs.rs | 1 - 8 files changed, 8 insertions(+), 11 deletions(-) diff --git a/crates/parser/src/grammar/block.rs b/crates/parser/src/grammar/block.rs index 5a40865..03f7ed2 100644 --- a/crates/parser/src/grammar/block.rs +++ b/crates/parser/src/grammar/block.rs @@ -2,7 +2,7 @@ use super::*; pub fn block(p: &mut Parser) { p.inc_rcurly(); - + if !p.at(LCurly) { p.advance_with_error("Miss {"); } else { diff --git a/crates/parser/src/grammar/function.rs b/crates/parser/src/grammar/function.rs index bca7018..4da6276 100644 --- a/crates/parser/src/grammar/function.rs +++ b/crates/parser/src/grammar/function.rs @@ -3,7 +3,7 @@ use crate::grammar::*; // fucntion name() pub fn function_parse(p: &mut Parser) { let m = p.open(); - + p.expect(FunctionKw); let fn_name_marker = p.open(); diff --git a/crates/parser/src/grammar/include.rs b/crates/parser/src/grammar/include.rs index d8f0ac7..7269995 100644 --- a/crates/parser/src/grammar/include.rs +++ b/crates/parser/src/grammar/include.rs @@ -2,7 +2,7 @@ use super::*; pub(super) fn include(p: &mut Parser) { // assert!(p.at(IncludeKw)); - + let m = p.open(); p.expect(IncludeKw); p.expect(CircomString); diff --git a/crates/parser/src/grammar/main_component.rs b/crates/parser/src/grammar/main_component.rs index d9c176d..5129310 100644 --- a/crates/parser/src/grammar/main_component.rs +++ b/crates/parser/src/grammar/main_component.rs @@ -8,7 +8,7 @@ component main {public [signal_list]} = tempid(v1,...,vn); pub fn main_component(p: &mut Parser) { p.expect(ComponentKw); p.expect(MainKw); - + if p.at(LCurly) { p.expect(LCurly); p.expect(PublicKw); diff --git a/crates/parser/src/grammar/template.rs b/crates/parser/src/grammar/template.rs index be5f17e..693fe68 100644 --- a/crates/parser/src/grammar/template.rs +++ b/crates/parser/src/grammar/template.rs @@ -8,7 +8,7 @@ pub fn template(p: &mut Parser) { let m = p.open(); p.expect(TemplateKw); - + let name_marker = p.open(); p.expect(Identifier); p.close(name_marker, TemplateName); diff --git a/crates/syntax/src/lib.rs b/crates/syntax/src/lib.rs index d1300bb..7f4b53c 100644 --- a/crates/syntax/src/lib.rs +++ b/crates/syntax/src/lib.rs @@ -1,6 +1,5 @@ +pub mod abstract_syntax_tree; pub mod syntax; pub mod syntax_node; -pub mod abstract_syntax_tree; - -mod test_programs; \ No newline at end of file +mod test_programs; diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 32a273b..9b9202f 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -164,8 +164,7 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); - - if let Some(ast) = AstCircomProgram::cast(syntax) { + if let Some(ast) = AstCircomProgram::cast(syntax) { check_ast_children(&ast, &expected_kinds, &expected_ranges); // check pragma diff --git a/crates/syntax/src/test_programs.rs b/crates/syntax/src/test_programs.rs index 0879679..655222f 100644 --- a/crates/syntax/src/test_programs.rs +++ b/crates/syntax/src/test_programs.rs @@ -132,4 +132,3 @@ pub const PARSER_TEST_6: &str = r#" /* T _ T */ template Multiplier2 () {} "#; - \ No newline at end of file From 0bf3a04a0f476adf43cda2d39136b01592ab9bf1 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 17 Dec 2024 15:59:27 +0700 Subject: [PATCH 16/31] remove empty test --- crates/parser/src/grammar/declaration.rs | 8 +---- crates/parser/src/grammar/expression.rs | 24 +------------ crates/parser/src/grammar/pragma.rs | 23 ------------ crates/parser/src/grammar/statement.rs | 46 ++++++++++++++---------- crates/parser/src/grammar/template.rs | 41 +-------------------- 5 files changed, 31 insertions(+), 111 deletions(-) diff --git a/crates/parser/src/grammar/declaration.rs b/crates/parser/src/grammar/declaration.rs index c37630c..27471b3 100644 --- a/crates/parser/src/grammar/declaration.rs +++ b/crates/parser/src/grammar/declaration.rs @@ -127,10 +127,4 @@ pub(super) fn declaration(p: &mut Parser) { ComponentKw => component_declaration(p), _ => unreachable!(), } -} - -#[cfg(test)] -mod declar_tests { - #[test] - fn signal_with_tag() {} -} +} \ No newline at end of file diff --git a/crates/parser/src/grammar/expression.rs b/crates/parser/src/grammar/expression.rs index 8c7fad6..96c85e0 100644 --- a/crates/parser/src/grammar/expression.rs +++ b/crates/parser/src/grammar/expression.rs @@ -155,26 +155,4 @@ fn circom_expression(p: &mut Parser) { p.close(m, TenaryConditional); } } -} -// #[cfg(test)] -// mod tests { - -// use rowan::SyntaxNode; - -// use crate::{syntax_node::CircomLang}; - -// use super::{entry::Scope, Parser}; - -// #[test] -// fn test_expression() { -// let source = r#" -// { -// a.tmp <== 100; -// b[1].c <== 10; -// } -// "#; -// let green = Parser::parse_scope(source, Scope::Block); -// let node = SyntaxNode::::new_root(green); -// println!("{:#?}", node); -// } -// } +} \ No newline at end of file diff --git a/crates/parser/src/grammar/pragma.rs b/crates/parser/src/grammar/pragma.rs index bacf839..9ce3eaf 100644 --- a/crates/parser/src/grammar/pragma.rs +++ b/crates/parser/src/grammar/pragma.rs @@ -14,26 +14,3 @@ pub fn pragma(p: &mut Parser) { p.expect(Semicolon); p.close(m, Pragma); } - -// #[cfg(test)] -// mod tests { -// #[test] -// fn pragam_test() { -// use crate::{ -// ast::{AstNode, AstPragma}, -// syntax_node::SyntaxNode, -// token_kind::TokenKind, -// }; - -// use super::{entry::Scope, Parser}; - -// let source: String = r#"pragma circom 2.0.1;"#.to_string(); - -// let green_node = Parser::parse_scope(&source, Scope::Pragma); -// let node = SyntaxNode::new_root(green_node); - -// let pragma = AstPragma::cast(node.last_child().unwrap()).unwrap(); - -// assert!(pragma.version().unwrap().syntax().kind() == TokenKind::Version); -// } -// } diff --git a/crates/parser/src/grammar/statement.rs b/crates/parser/src/grammar/statement.rs index a4ee698..d8b75fa 100644 --- a/crates/parser/src/grammar/statement.rs +++ b/crates/parser/src/grammar/statement.rs @@ -9,6 +9,12 @@ pub(super) fn statement(p: &mut Parser) { p.close(m, Statement); } +/* +if (expr) + +else + +*/ fn if_statement(p: &mut Parser) { let m = p.open(); p.expect(IfKw); @@ -25,6 +31,7 @@ fn if_statement(p: &mut Parser) { /** * no if condition here. + * for/while/return/assert... */ fn statement_no_condition(p: &mut Parser) { match p.current() { @@ -50,6 +57,10 @@ fn statement_no_condition(p: &mut Parser) { } } +/* +for (/; ; ) + +*/ fn for_statement(p: &mut Parser) { let m = p.open(); p.expect(ForKw); @@ -70,6 +81,10 @@ fn for_statement(p: &mut Parser) { p.close(m, ForLoop); } +/* +while () + +*/ fn while_statement(p: &mut Parser) { p.expect(WhileKw); p.expect(LParen); @@ -78,6 +93,9 @@ fn while_statement(p: &mut Parser) { statement(p); } +/* +assert() +*/ fn assert_statement(p: &mut Parser) { let m = p.open(); p.expect(AssertKw); @@ -87,6 +105,9 @@ fn assert_statement(p: &mut Parser) { p.close(m, AssertKw); } +/* +log() +*/ fn log_statement(p: &mut Parser) { let m = p.open(); p.expect(LogKw); @@ -109,6 +130,9 @@ fn log_statement(p: &mut Parser) { p.close(m, LogKw); } +/* +return +*/ fn return_statement(p: &mut Parser) { let m = p.open(); p.expect(ReturnKw); @@ -116,6 +140,9 @@ fn return_statement(p: &mut Parser) { p.close(m, ReturnKw); } +/* + +*/ fn assignment_statement(p: &mut Parser) { let m = p.open(); @@ -154,21 +181,4 @@ fn assignment_statement(p: &mut Parser) { } else { p.close(m, Error); } -} - -#[cfg(test)] -mod tests { - - #[test] - fn if_statement_test() { - let _source = r#" - assert(1 == 2); - "#; - // let mut parser = Parser::new(source); - - // statement(&mut parser); - // let cst = parser.build_tree().ok().unwrap(); - - // println!("{:?}", cst); - } -} +} \ No newline at end of file diff --git a/crates/parser/src/grammar/template.rs b/crates/parser/src/grammar/template.rs index 693fe68..359d774 100644 --- a/crates/parser/src/grammar/template.rs +++ b/crates/parser/src/grammar/template.rs @@ -22,43 +22,4 @@ pub fn template(p: &mut Parser) { block::block(p); p.close(m, TemplateDef); -} - -// #[cfg(test)] -// mod tests { -// use crate::ast::AstTemplateDef; - -// #[test] -// fn template_parse_test() { -// use crate::{ast::AstNode, syntax_node::SyntaxNode}; - -// use super::{entry::Scope, Parser}; - -// let source: String = r#" -// template Multiplier2 (a, b, c) { - -// // Declaration of signals. -// signal input a; -// signal input b; -// signal output c; - -// // Constraints. -// c <== a * b; -// } - -// "# -// .to_string(); - -// let green_node = ::parse_scope(&source, Scope::Template); -// let node = SyntaxNode::new_root(green_node); - -// let ast_template = AstTemplateDef::cast(node); - -// if let Some(ast_internal) = ast_template { -// println!( -// "name {:?}", -// ast_internal.template_name().unwrap().syntax().text() -// ); -// } -// } -// } +} \ No newline at end of file From f1b0c3967be3941287c76bb69b90a9290d9db586 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 17 Dec 2024 16:00:07 +0700 Subject: [PATCH 17/31] replace expect by eat in block --- crates/parser/src/grammar/block.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/parser/src/grammar/block.rs b/crates/parser/src/grammar/block.rs index 03f7ed2..f01b515 100644 --- a/crates/parser/src/grammar/block.rs +++ b/crates/parser/src/grammar/block.rs @@ -1,5 +1,13 @@ use super::*; +/* +{ + / + / + .... + / +} +*/ pub fn block(p: &mut Parser) { p.inc_rcurly(); @@ -30,7 +38,7 @@ pub fn block(p: &mut Parser) { p.close(stmt_marker, StatementList); - p.expect(RCurly); + p.eat(RCurly); p.close(m, Block); From a10be252a1d9661aa5183883546fd5ccc6a531b4 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 17 Dec 2024 16:00:36 +0700 Subject: [PATCH 18/31] add scope parsing test --- crates/parser/Cargo.toml | 1 - crates/syntax/src/syntax.rs | 213 ++++++++++++++++++++++++++++++++++++ 2 files changed, 213 insertions(+), 1 deletion(-) diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 5021d30..1ce7c59 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -12,6 +12,5 @@ rowan = "0.15.15" num-traits = "0.2" num-derive = "0.2" - [profile.dev] debug = 2 \ No newline at end of file diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 9b9202f..6cde0b1 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -289,3 +289,216 @@ mod tests { } } } + +#[cfg(test)] +mod grammar_tests { + use parser::{grammar::entry::Scope, input::Input, parser::Parser, token_kind::TokenKind}; + use rowan::ast::AstNode; + + use crate::{ + abstract_syntax_tree::{AstBlock, AstPragma, AstTemplateDef}, + syntax::SyntaxTreeBuilder, + syntax_node::SyntaxNode, + }; + + #[test] + fn pragma_happy_test() { + // parse source (string) into output tree + let version = r#"2.0.1"#; + let source = format!(r#"pragma circom {};"#, version); + let input = Input::new(&source); + let output = Parser::parsing_with_scope(&input, Scope::Pragma); + + // output is a tree whose node is index of token, no content of token + // convert output into green node + let mut builder = SyntaxTreeBuilder::new(&input); + builder.build(output); + let green = builder.finish(); + + // then cast green node into syntax node + let syntax = SyntaxNode::new_root(green); + + // cast syntax node into ast node to retrieve more information + let pragma = AstPragma::cast(syntax).expect("Can not cast syntax node into ast pragma"); + + // finally, assert with expect value + assert!(pragma.version().unwrap().syntax().kind() == TokenKind::Version); + assert!(pragma.version().unwrap().syntax().text() == version); + } + + #[test] + fn template_happy_test() { + // SOURCE & EXPECTED RESULT + const SOURCE: &str = r#"template MultiplierN (N, P, QQ) { + //Declaration of signals and components. + signal input in[N]; + signal output out; + component comp[N-1]; + + //Statements. + for(var i = 0; i < N-1; i++){ + comp[i] = Multiplier2(); + } + + // ... some more code (see below) + + }"#; + let expected_statements: Vec<&str> = vec![ + "signal input in[N];", + "signal output out;", + "component comp[N-1];", + "for(var i = 0; i < N-1; i++){ + comp[i] = Multiplier2(); + }", + ]; + let expected_name = "MultiplierN"; + let expected_first_param = "N"; + let expected_last_param = "QQ"; + + // parse source (string) into output tree + let input = Input::new(&SOURCE); + let output = Parser::parsing_with_scope(&input, Scope::Template); + + // output is a tree whose node is index of token, no content of token + // convert output into green node + let mut builder = SyntaxTreeBuilder::new(&input); + builder.build(output); + let green = builder.finish(); + + // then cast green node into syntax node + let syntax = SyntaxNode::new_root(green); + + // cast syntax node into ast node to retrieve more information + let template = + AstTemplateDef::cast(syntax).expect("Can not cast syntax node into ast template"); + + // finally, assert with expect value + + // name + let name = template + .name() + .expect("Can not extract template name") + .syntax() + .text(); + assert_eq!(expected_name, name); + + // parameter list + let first_param = template + .parameter_list() + .expect("Can not detect parameter list") + .syntax() + .first_child() + .unwrap() + .text(); + assert_eq!(expected_first_param, first_param); + let last_param = template + .parameter_list() + .expect("Can not detect parameter list") + .syntax() + .last_child() + .unwrap() + .text(); + assert_eq!(expected_last_param, last_param); + + // statements + let statements = template.statements().unwrap().statement_list(); + let statements: Vec = statements + .into_iter() + .map(|statement| statement.syntax().text().to_string()) + .collect(); + assert_eq!( + expected_statements.len(), + statements.len(), + "Number of statements is not match" + ); + + for id in 0..statements.len() { + assert_eq!(expected_statements[id].to_string(), statements[id]); + } + + // // input signal + // println!("find_input_signal: {:?}", template.find_input_signal()); + + // // output signal + // println!("find_output_signal: {:?}", template.find_output_signal()); + + // // internal signal + // println!("find_internal_signal: {:?}", template.find_internal_signal()); + + // // component + // println!("find_component: {:?}", template.find_component()); + } + + #[test] + fn block_happy_test() { + // SOURCE & EXPECTED RESULT + let source = r#"{ + //Declaration of signals. + signal input in[N]; + signal output out; + component comp[N-1]; + + //Statements. + for(var i = 0; i < N-1; i++){ + comp[i] = Multiplier2(); + } + comp[0].in1 <== in[0]; + comp[0].in2 <== in[1]; + for(var i = 0; i < N-2; i++){ + comp[i+1].in1 <== comp[i].out; + comp[i+1].in2 <== in[i+2]; + + } + out <== comp[N-2].out; +}"#; + let expected_statements = vec![ + "signal input in[N];", + "signal output out;", + "component comp[N-1];", + "for(var i = 0; i < N-1; i++){ + comp[i] = Multiplier2(); + }", + "comp[0].in1 <== in[0];", + "comp[0].in2 <== in[1];", + "for(var i = 0; i < N-2; i++){ + comp[i+1].in1 <== comp[i].out; + comp[i+1].in2 <== in[i+2]; + + }", + "out <== comp[N-2].out;", + ]; + + + // parse source (string) into output tree + let input = Input::new(&source); + let output = Parser::parsing_with_scope(&input, Scope::Block); + + // output is a tree whose node is index of token, no content of token + // convert output into green node + let mut builder = SyntaxTreeBuilder::new(&input); + builder.build(output); + let green = builder.finish(); + + // then cast green node into syntax node + let syntax = SyntaxNode::new_root(green); + + // cast syntax node into ast node to retrieve more information + let block = AstBlock::cast(syntax).expect("Can not cast syntax node into ast block"); + + // finally, assert with expect statements + let statements = block.statement_list().unwrap().statement_list(); + let statements: Vec = statements + .into_iter() + .map(|statement| statement.syntax().text().to_string()) + .collect(); + assert_eq!( + expected_statements.len(), + statements.len(), + "Number of statements is not match" + ); + + for id in 0..statements.len() { + assert_eq!(expected_statements[id].to_string(), statements[id]); + } + } +} From 47df964daedb4bca28ccead785dc348614fe9e88 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Tue, 17 Dec 2024 16:04:50 +0700 Subject: [PATCH 19/31] fix space in scope parsing test --- crates/syntax/src/syntax.rs | 49 ++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 6cde0b1..defc696 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -348,8 +348,8 @@ mod grammar_tests { "signal output out;", "component comp[N-1];", "for(var i = 0; i < N-1; i++){ - comp[i] = Multiplier2(); - }", + comp[i] = Multiplier2(); + }", ]; let expected_name = "MultiplierN"; let expected_first_param = "N"; @@ -433,42 +433,41 @@ mod grammar_tests { fn block_happy_test() { // SOURCE & EXPECTED RESULT let source = r#"{ - //Declaration of signals. - signal input in[N]; - signal output out; - component comp[N-1]; - - //Statements. - for(var i = 0; i < N-1; i++){ - comp[i] = Multiplier2(); - } - comp[0].in1 <== in[0]; - comp[0].in2 <== in[1]; - for(var i = 0; i < N-2; i++){ - comp[i+1].in1 <== comp[i].out; - comp[i+1].in2 <== in[i+2]; + //Declaration of signals. + signal input in[N]; + signal output out; + component comp[N-1]; - } - out <== comp[N-2].out; -}"#; + //Statements. + for(var i = 0; i < N-1; i++){ + comp[i] = Multiplier2(); + } + comp[0].in1 <== in[0]; + comp[0].in2 <== in[1]; + for(var i = 0; i < N-2; i++){ + comp[i+1].in1 <== comp[i].out; + comp[i+1].in2 <== in[i+2]; + + } + out <== comp[N-2].out; + }"#; let expected_statements = vec![ "signal input in[N];", "signal output out;", "component comp[N-1];", "for(var i = 0; i < N-1; i++){ - comp[i] = Multiplier2(); - }", + comp[i] = Multiplier2(); + }", "comp[0].in1 <== in[0];", "comp[0].in2 <== in[1];", "for(var i = 0; i < N-2; i++){ - comp[i+1].in1 <== comp[i].out; - comp[i+1].in2 <== in[i+2]; + comp[i+1].in1 <== comp[i].out; + comp[i+1].in2 <== in[i+2]; - }", + }", "out <== comp[N-2].out;", ]; - // parse source (string) into output tree let input = Input::new(&source); let output = Parser::parsing_with_scope(&input, Scope::Block); From 5c6cb56fe53f636b4ea1e82d4d0d1a52badb74af Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Thu, 19 Dec 2024 18:10:15 +0700 Subject: [PATCH 20/31] update snapshot test for syntax --- Cargo.toml | 1 + crates/parser/Cargo.toml | 1 + crates/parser/src/token_kind.rs | 3 +- crates/syntax/Cargo.toml | 10 +- ...ar_tests__block_happy_test_statements.snap | 12 + ...tests__template_happy_test_statements.snap | 8 + ...syntax__tests__syntax_test_1_children.snap | 17 ++ ...yntax__tests__syntax_test_2_functions.snap | 5 + ...yntax__tests__syntax_test_2_templates.snap | 5 + ...yntax__tests__syntax_test_5_templates.snap | 5 + ...yntax__tests__syntax_test_6_templates.snap | 5 + crates/syntax/src/syntax.rs | 252 +++++++----------- 12 files changed, 163 insertions(+), 161 deletions(-) create mode 100644 crates/syntax/src/snapshots/syntax__syntax__grammar_tests__block_happy_test_statements.snap create mode 100644 crates/syntax/src/snapshots/syntax__syntax__grammar_tests__template_happy_test_statements.snap create mode 100644 crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_1_children.snap create mode 100644 crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap create mode 100644 crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap create mode 100644 crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap create mode 100644 crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap diff --git a/Cargo.toml b/Cargo.toml index 5dc248b..bfbc8e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,5 +10,6 @@ syntax = {path = './crates/syntax', version = "0.1.0"} circom-lsp = {path = './crates/lsp', version = "*"} common = { path = './crates/common', version = "*"} database = {path = "./crates/database", version = "*"} + [workspace.package] rust-version = "1.71" diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 1ce7c59..9070855 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -11,6 +11,7 @@ lsp-types = {version = "0.94.1", features = ["proposed"]} rowan = "0.15.15" num-traits = "0.2" num-derive = "0.2" +serde = "1.0.216" [profile.dev] debug = 2 \ No newline at end of file diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs index 9ebba15..9eba89c 100644 --- a/crates/parser/src/token_kind.rs +++ b/crates/parser/src/token_kind.rs @@ -1,6 +1,7 @@ use logos::Logos; +use serde::Serialize; -#[derive(Logos, Debug, PartialEq, Clone, Copy, Eq, PartialOrd, Ord, Hash)] +#[derive(Logos, Debug, PartialEq, Clone, Copy, Eq, PartialOrd, Ord, Hash, Serialize)] #[allow(non_camel_case_types)] #[repr(u16)] pub enum TokenKind { diff --git a/crates/syntax/Cargo.toml b/crates/syntax/Cargo.toml index d6a3b47..795f0b1 100644 --- a/crates/syntax/Cargo.toml +++ b/crates/syntax/Cargo.toml @@ -9,5 +9,13 @@ rust-version.workspace = true [dependencies] rowan = "0.15.13" parser.workspace = true - lsp-types = {version = "0.94.1", features = ["proposed"]} + +[dev-dependencies] +# for snapshot testing, yaml format +insta = { version = "1.41.1", features = ["yaml"] } + +[profile.dev.package] +# compile slightly slower once, but use less memory, have faster diffs +insta.opt-level = 3 +similar.opt-level = 3 \ No newline at end of file diff --git a/crates/syntax/src/snapshots/syntax__syntax__grammar_tests__block_happy_test_statements.snap b/crates/syntax/src/snapshots/syntax__syntax__grammar_tests__block_happy_test_statements.snap new file mode 100644 index 0000000..5db6859 --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__grammar_tests__block_happy_test_statements.snap @@ -0,0 +1,12 @@ +--- +source: crates/syntax/src/syntax.rs +expression: statements +--- +- "signal input in[N];" +- "signal output out;" +- "component comp[N-1];" +- "for(var i = 0; i < N-1; i++){\n comp[i] = Multiplier2();\n }" +- "comp[0].in1 <== in[0];" +- "comp[0].in2 <== in[1];" +- "for(var i = 0; i < N-2; i++){\n comp[i+1].in1 <== comp[i].out;\n comp[i+1].in2 <== in[i+2];\n\n }" +- "out <== comp[N-2].out;" \ No newline at end of file diff --git a/crates/syntax/src/snapshots/syntax__syntax__grammar_tests__template_happy_test_statements.snap b/crates/syntax/src/snapshots/syntax__syntax__grammar_tests__template_happy_test_statements.snap new file mode 100644 index 0000000..74cf89f --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__grammar_tests__template_happy_test_statements.snap @@ -0,0 +1,8 @@ +--- +source: crates/syntax/src/syntax.rs +expression: statements +--- +- "signal input in[N];" +- "signal output out;" +- "component comp[N-1];" +- "for(var i = 0; i < N-1; i++){\n comp[i] = Multiplier2();\n }" diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_1_children.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_1_children.snap new file mode 100644 index 0000000..b1feddb --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_1_children.snap @@ -0,0 +1,17 @@ +--- +source: crates/syntax/src/syntax.rs +expression: children_string +--- +- pragma circom 2.0.0; +- "\n" +- "\n" +- " " +- "\n" +- " " +- "template Multiplier2 () {}" +- "\n" +- " " +- "template Multiplier2 () {}" +- " " +- "\n" +- " " diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap new file mode 100644 index 0000000..49ea709 --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap @@ -0,0 +1,5 @@ +--- +source: crates/syntax/src/syntax.rs +expression: function_names +--- +- "nbits" diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap new file mode 100644 index 0000000..03a1b69 --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap @@ -0,0 +1,5 @@ +--- +source: crates/syntax/src/syntax.rs +expression: template_names +--- +- "BinSum" diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap new file mode 100644 index 0000000..53cde17 --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap @@ -0,0 +1,5 @@ +--- +source: crates/syntax/src/syntax.rs +expression: template_names +--- +- "Multiplier2" diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap new file mode 100644 index 0000000..53cde17 --- /dev/null +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap @@ -0,0 +1,5 @@ +--- +source: crates/syntax/src/syntax.rs +expression: template_names +--- +- "Multiplier2" diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index defc696..dcb98d0 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -58,118 +58,37 @@ impl<'a> SyntaxTreeBuilder<'a> { #[cfg(test)] mod tests { - use parser::token_kind::TokenKind::{self, *}; use std::hash::{DefaultHasher, Hash, Hasher}; - use rowan::{ast::AstNode, TextRange}; + use rowan::ast::AstNode; use crate::{abstract_syntax_tree::AstCircomProgram, test_programs}; use super::SyntaxTreeBuilder; - fn generate_expected_token_kind(ast: &AstCircomProgram) { - let children = ast - .syntax() - .first_child() - .unwrap() - .siblings(rowan::Direction::Next); - - println!("vec!["); - for child in children { - println!("{:?},", child.kind()); - } - println!("];"); - } - - fn generate_expected_token_range(ast: &AstCircomProgram) { - let children = ast - .syntax() - .first_child() - .unwrap() - .siblings(rowan::Direction::Next); - - println!("vec!["); - for child in children { - println!( - "TextRange::new({:?}.into(), {:?}.into()), ", - child.text_range().start(), - child.text_range().end() - ); - } - println!("];"); - } - - fn check_ast_children( - ast: &AstCircomProgram, - expected_kinds: &Vec, - expected_ranges: &Vec, - ) { - let children = ast - .syntax() - .first_child() - .unwrap() - .siblings(rowan::Direction::Next); - - let mut kind_iterator = expected_kinds.iter(); - let mut range_iterator = expected_ranges.iter(); - - for child in children { - if let (Some(expected_kind), Some(expected_range)) = - (kind_iterator.next(), range_iterator.next()) - { - assert_eq!(child.kind(), *expected_kind); - assert_eq!(child.text_range(), *expected_range); - } else { - panic!("Mismatched number of children and expected values"); - } - } - println!(); - } - #[test] fn syntax_test_1() { let source: &str = test_programs::PARSER_TEST_1; - let expected_pragma = "pragma circom 2.0.0;".to_string(); - let expected_kinds = vec![ - Pragma, - EndLine, - EndLine, - WhiteSpace, - EndLine, - WhiteSpace, - TemplateDef, - EndLine, - WhiteSpace, - TemplateDef, - WhiteSpace, - EndLine, - WhiteSpace, - ]; - let expected_ranges = vec![ - TextRange::new(0.into(), 20.into()), - TextRange::new(20.into(), 21.into()), - TextRange::new(21.into(), 22.into()), - TextRange::new(22.into(), 26.into()), - TextRange::new(26.into(), 27.into()), - TextRange::new(27.into(), 31.into()), - TextRange::new(31.into(), 57.into()), - TextRange::new(57.into(), 58.into()), - TextRange::new(58.into(), 62.into()), - TextRange::new(62.into(), 88.into()), - TextRange::new(88.into(), 89.into()), - TextRange::new(89.into(), 90.into()), - TextRange::new(90.into(), 94.into()), - ]; - let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - check_ast_children(&ast, &expected_kinds, &expected_ranges); + // check_ast_children + let children = ast + .syntax() + .first_child() + .unwrap() + .siblings(rowan::Direction::Next); + let mut children_string = Vec::new(); + + for child in children.into_iter() { + children_string.push(child.text().to_string()); + } + insta::assert_yaml_snapshot!("syntax_test_1_children", children_string); // check pragma let pragma = ast.pragma().unwrap().syntax().text().to_string(); - assert_eq!(pragma, expected_pragma, "Pragma is not correct!"); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); // check ast hash let mut hasher = DefaultHasher::default(); @@ -212,24 +131,29 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); + let pragma = ast.pragma().unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - print!("Templates: "); let templates = ast.template_list(); + let mut template_names = Vec::new(); for template in templates.iter() { - print!("{:?} ", template.name().unwrap().syntax().text()); // leading whitespaces - // print!("{:?} ", template.syntax().text()); // leading whitespaces + let template_name = template.name().unwrap().syntax().text().to_string(); + template_names.push(template_name); } - println!(); + insta::assert_yaml_snapshot!("syntax_test_2_templates", template_names); - print!("Functions: "); let functions = ast.function_list(); + let mut function_names = Vec::new(); for function in functions.iter() { - print!("{:?} ", function.function_name().unwrap().syntax().text()); - // leading whitespaces - // print!("{:?} ", function.syntax().text()); // leading whitespaces + let function_name = function + .function_name() + .unwrap() + .syntax() + .text() + .to_string(); + function_names.push(function_name); } - println!(); + insta::assert_yaml_snapshot!("syntax_test_2_functions", function_names); } } @@ -240,11 +164,18 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); - println!( - "Pragma version: {:?}", - ast.pragma().unwrap().version().unwrap().syntax().text() - ); + let pragma = ast.pragma().unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); + + let pragma_version = ast + .pragma() + .unwrap() + .version() + .unwrap() + .syntax() + .text() + .to_string(); + insta::assert_yaml_snapshot!(pragma_version, @"2.0.0"); } } @@ -255,11 +186,18 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - println!("Pragma: {:?}", ast.pragma().unwrap().syntax().text()); - println!( - "Pragma version: {:?}", - ast.pragma().unwrap().version().unwrap().syntax().text() - ); + let pragma = ast.pragma().unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); + + let pragma_version = ast + .pragma() + .unwrap() + .version() + .unwrap() + .syntax() + .text() + .to_string(); + insta::assert_yaml_snapshot!(pragma_version, @"2.0.0"); } } @@ -270,9 +208,16 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - println!("pragma: {:?}", ast.pragma()); - println!("template list: {:?}", ast.template_list()); - // assert!(ast.pragma().is_none(), "No pragma in source code"); + let pragma = ast.pragma().is_none(); + insta::assert_yaml_snapshot!(pragma, @"true"); + + let templates = ast.template_list(); + let mut template_names = Vec::new(); + for template in templates.iter() { + let template_name = template.name().unwrap().syntax().text().to_string(); + template_names.push(template_name); + } + insta::assert_yaml_snapshot!("syntax_test_5_templates", template_names); } } @@ -283,16 +228,23 @@ mod tests { let syntax = SyntaxTreeBuilder::syntax_tree(source); if let Some(ast) = AstCircomProgram::cast(syntax) { - println!("{:?}", ast.pragma()); - println!("template list: {:?}", ast.template_list()); - // assert!(ast.pragma().is_none(), "No pragma in source code"); + let pragma = ast.pragma().is_none(); + insta::assert_yaml_snapshot!(pragma, @"true"); + + let templates = ast.template_list(); + let mut template_names = Vec::new(); + for template in templates.iter() { + let template_name = template.name().unwrap().syntax().text().to_string(); + template_names.push(template_name); + } + insta::assert_yaml_snapshot!("syntax_test_6_templates", template_names); } } } #[cfg(test)] mod grammar_tests { - use parser::{grammar::entry::Scope, input::Input, parser::Parser, token_kind::TokenKind}; + use parser::{grammar::entry::Scope, input::Input, parser::Parser}; use rowan::ast::AstNode; use crate::{ @@ -322,8 +274,11 @@ mod grammar_tests { let pragma = AstPragma::cast(syntax).expect("Can not cast syntax node into ast pragma"); // finally, assert with expect value - assert!(pragma.version().unwrap().syntax().kind() == TokenKind::Version); - assert!(pragma.version().unwrap().syntax().text() == version); + let pragma_versison_kind = pragma.version().unwrap().syntax().kind(); + insta::assert_yaml_snapshot!(pragma_versison_kind, @"Version"); + + let pragma_versison_text = pragma.version().unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(pragma_versison_text, @"2.0.1"); } #[test] @@ -343,17 +298,6 @@ mod grammar_tests { // ... some more code (see below) }"#; - let expected_statements: Vec<&str> = vec![ - "signal input in[N];", - "signal output out;", - "component comp[N-1];", - "for(var i = 0; i < N-1; i++){ - comp[i] = Multiplier2(); - }", - ]; - let expected_name = "MultiplierN"; - let expected_first_param = "N"; - let expected_last_param = "QQ"; // parse source (string) into output tree let input = Input::new(&SOURCE); @@ -379,8 +323,9 @@ mod grammar_tests { .name() .expect("Can not extract template name") .syntax() - .text(); - assert_eq!(expected_name, name); + .text() + .to_string(); + insta::assert_yaml_snapshot!(name, @"MultiplierN"); // parameter list let first_param = template @@ -389,16 +334,19 @@ mod grammar_tests { .syntax() .first_child() .unwrap() - .text(); - assert_eq!(expected_first_param, first_param); + .text() + .to_string(); + insta::assert_yaml_snapshot!(first_param, @"N"); + let last_param = template .parameter_list() .expect("Can not detect parameter list") .syntax() .last_child() .unwrap() - .text(); - assert_eq!(expected_last_param, last_param); + .text() + .to_string(); + insta::assert_yaml_snapshot!(last_param, @"QQ"); // statements let statements = template.statements().unwrap().statement_list(); @@ -406,15 +354,7 @@ mod grammar_tests { .into_iter() .map(|statement| statement.syntax().text().to_string()) .collect(); - assert_eq!( - expected_statements.len(), - statements.len(), - "Number of statements is not match" - ); - - for id in 0..statements.len() { - assert_eq!(expected_statements[id].to_string(), statements[id]); - } + insta::assert_yaml_snapshot!("template_happy_test_statements", statements); // // input signal // println!("find_input_signal: {:?}", template.find_input_signal()); @@ -451,6 +391,7 @@ mod grammar_tests { } out <== comp[N-2].out; }"#; + /* let expected_statements = vec![ "signal input in[N];", "signal output out;", @@ -467,6 +408,7 @@ mod grammar_tests { }", "out <== comp[N-2].out;", ]; + */ // parse source (string) into output tree let input = Input::new(&source); @@ -490,14 +432,6 @@ mod grammar_tests { .into_iter() .map(|statement| statement.syntax().text().to_string()) .collect(); - assert_eq!( - expected_statements.len(), - statements.len(), - "Number of statements is not match" - ); - - for id in 0..statements.len() { - assert_eq!(expected_statements[id].to_string(), statements[id]); - } + insta::assert_yaml_snapshot!("block_happy_test_statements", statements); } } From 372efd3d6a6510b100305b9d8c9d751f2d21c723 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Thu, 19 Dec 2024 18:39:51 +0700 Subject: [PATCH 21/31] refactor syntax test --- .gitignore | 4 +- crates/syntax/src/syntax.rs | 331 +++++++++++++++--------------------- 2 files changed, 144 insertions(+), 191 deletions(-) diff --git a/.gitignore b/.gitignore index 6f429f5..165e8e1 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,6 @@ node_modules .vscode-test /target Cargo.lock -assets \ No newline at end of file +assets +*.new +*.pending-snap \ No newline at end of file diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index dcb98d0..0c1d1b8 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -66,198 +66,178 @@ mod tests { use super::SyntaxTreeBuilder; - #[test] - fn syntax_test_1() { - let source: &str = test_programs::PARSER_TEST_1; - + fn ast_from_source(source: &str) -> AstCircomProgram { let syntax = SyntaxTreeBuilder::syntax_tree(source); + AstCircomProgram::cast(syntax).unwrap() + } - if let Some(ast) = AstCircomProgram::cast(syntax) { - // check_ast_children - let children = ast - .syntax() - .first_child() - .unwrap() - .siblings(rowan::Direction::Next); - let mut children_string = Vec::new(); - - for child in children.into_iter() { - children_string.push(child.text().to_string()); - } - insta::assert_yaml_snapshot!("syntax_test_1_children", children_string); - - // check pragma - let pragma = ast.pragma().unwrap().syntax().text().to_string(); - insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - - // check ast hash - let mut hasher = DefaultHasher::default(); - ast.syntax().hash(&mut hasher); - let _ast_hash = hasher.finish(); - - // check template hash - let mut h1 = DefaultHasher::default(); - let mut h2 = DefaultHasher::default(); - - let template = ast.template_list(); - - template[0].syntax().hash(&mut h1); - template[1].syntax().hash(&mut h2); - - assert_ne!( - h1.finish(), - h2.finish(), - "Templates with same syntax should have different hashes!" - ); - - // check template syntax (text & green node) - assert_eq!( - template[0].syntax().text(), - template[1].syntax().text(), - "The syntax (as text) of template 1 and 2 must be the same!" - ); - assert_eq!( - template[0].syntax().green(), - template[1].syntax().green(), - "The syntax (as green node) of template 1 and 2 must be the same!!" - ); - } + fn children_from_ast(ast: &AstCircomProgram) -> Vec { + let children = ast + .syntax() + .first_child() + .unwrap() + .siblings(rowan::Direction::Next) + .into_iter() + .map(|child| child.text().to_string()) + .collect(); + + children } - #[test] - fn syntax_test_2() { - let source = test_programs::PARSER_TEST_2; + fn pragma_string_from_ast(ast: &AstCircomProgram) -> String { + ast.pragma().unwrap().syntax().text().to_string() + } - let syntax = SyntaxTreeBuilder::syntax_tree(source); + fn pragma_version_from_ast(ast: &AstCircomProgram) -> String { + ast.pragma() + .unwrap() + .version() + .unwrap() + .syntax() + .text() + .to_string() + } - if let Some(ast) = AstCircomProgram::cast(syntax) { - let pragma = ast.pragma().unwrap().syntax().text().to_string(); - insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); + fn template_names_from_ast(ast: &AstCircomProgram) -> Vec { + let templates = ast + .template_list() + .iter() + .map(|template| template.name().unwrap().syntax().text().to_string()) + .collect(); - let templates = ast.template_list(); - let mut template_names = Vec::new(); - for template in templates.iter() { - let template_name = template.name().unwrap().syntax().text().to_string(); - template_names.push(template_name); - } - insta::assert_yaml_snapshot!("syntax_test_2_templates", template_names); + templates + } - let functions = ast.function_list(); - let mut function_names = Vec::new(); - for function in functions.iter() { - let function_name = function + fn function_names_from_ast(ast: &AstCircomProgram) -> Vec { + let functions = ast + .function_list() + .iter() + .map(|function| { + function .function_name() .unwrap() .syntax() .text() - .to_string(); - function_names.push(function_name); - } - insta::assert_yaml_snapshot!("syntax_test_2_functions", function_names); - } + .to_string() + }) + .collect(); + + functions } #[test] - fn syntax_test_3() { - let source = test_programs::PARSER_TEST_3; - - let syntax = SyntaxTreeBuilder::syntax_tree(source); - - if let Some(ast) = AstCircomProgram::cast(syntax) { - let pragma = ast.pragma().unwrap().syntax().text().to_string(); - insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - - let pragma_version = ast - .pragma() - .unwrap() - .version() - .unwrap() - .syntax() - .text() - .to_string(); - insta::assert_yaml_snapshot!(pragma_version, @"2.0.0"); - } + fn syntax_test_1() { + let ast = ast_from_source(test_programs::PARSER_TEST_1); + + // check_ast_children + let children = children_from_ast(&ast); + insta::assert_yaml_snapshot!("syntax_test_1_children", children); + + // check pragma + let pragma = pragma_string_from_ast(&ast); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); + + // check ast hash + let mut hasher = DefaultHasher::default(); + ast.syntax().hash(&mut hasher); + let _ast_hash = hasher.finish(); + + // check template hash + let mut h1 = DefaultHasher::default(); + let mut h2 = DefaultHasher::default(); + + let template = ast.template_list(); + + template[0].syntax().hash(&mut h1); + template[1].syntax().hash(&mut h2); + + assert_ne!( + h1.finish(), + h2.finish(), + "Templates with same syntax should have different hashes!" + ); + + // check template syntax (text & green node) + assert_eq!( + template[0].syntax().text(), + template[1].syntax().text(), + "The syntax (as text) of template 1 and 2 must be the same!" + ); + assert_eq!( + template[0].syntax().green(), + template[1].syntax().green(), + "The syntax (as green node) of template 1 and 2 must be the same!!" + ); } #[test] - fn syntax_test_4() { - let source = test_programs::PARSER_TEST_4; + fn syntax_test_2() { + let ast = ast_from_source(test_programs::PARSER_TEST_2); - let syntax = SyntaxTreeBuilder::syntax_tree(source); + let pragma = pragma_string_from_ast(&ast); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - if let Some(ast) = AstCircomProgram::cast(syntax) { - let pragma = ast.pragma().unwrap().syntax().text().to_string(); - insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - - let pragma_version = ast - .pragma() - .unwrap() - .version() - .unwrap() - .syntax() - .text() - .to_string(); - insta::assert_yaml_snapshot!(pragma_version, @"2.0.0"); - } + let template_names = template_names_from_ast(&ast); + insta::assert_yaml_snapshot!("syntax_test_2_templates", template_names); + + let function_names = function_names_from_ast(&ast); + insta::assert_yaml_snapshot!("syntax_test_2_functions", function_names); } #[test] - fn syntax_test_5() { - let source = test_programs::PARSER_TEST_5; + fn syntax_test_3() { + let ast = ast_from_source(test_programs::PARSER_TEST_3); + let pragma = pragma_string_from_ast(&ast); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - let syntax = SyntaxTreeBuilder::syntax_tree(source); + let pragma_version = pragma_version_from_ast(&ast); + insta::assert_yaml_snapshot!(pragma_version, @"2.0.0"); + } - if let Some(ast) = AstCircomProgram::cast(syntax) { - let pragma = ast.pragma().is_none(); - insta::assert_yaml_snapshot!(pragma, @"true"); + #[test] + fn syntax_test_4() { + let ast = ast_from_source(test_programs::PARSER_TEST_4); + let pragma = pragma_string_from_ast(&ast); + insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - let templates = ast.template_list(); - let mut template_names = Vec::new(); - for template in templates.iter() { - let template_name = template.name().unwrap().syntax().text().to_string(); - template_names.push(template_name); - } - insta::assert_yaml_snapshot!("syntax_test_5_templates", template_names); - } + let pragma_version = pragma_version_from_ast(&ast); + insta::assert_yaml_snapshot!(pragma_version, @"2.0.0"); } #[test] - fn syntax_test_6() { - let source = test_programs::PARSER_TEST_6; + fn syntax_test_5() { + let ast = ast_from_source(test_programs::PARSER_TEST_5); + let pragma = ast.pragma().is_none(); + insta::assert_yaml_snapshot!(pragma, @"true"); - let syntax = SyntaxTreeBuilder::syntax_tree(source); + let template_names = template_names_from_ast(&ast); + insta::assert_yaml_snapshot!("syntax_test_5_templates", template_names); + } - if let Some(ast) = AstCircomProgram::cast(syntax) { - let pragma = ast.pragma().is_none(); - insta::assert_yaml_snapshot!(pragma, @"true"); + #[test] + fn syntax_test_6() { + let ast = ast_from_source(test_programs::PARSER_TEST_6); + let pragma = ast.pragma().is_none(); + insta::assert_yaml_snapshot!(pragma, @"true"); - let templates = ast.template_list(); - let mut template_names = Vec::new(); - for template in templates.iter() { - let template_name = template.name().unwrap().syntax().text().to_string(); - template_names.push(template_name); - } - insta::assert_yaml_snapshot!("syntax_test_6_templates", template_names); - } + let template_names = template_names_from_ast(&ast); + insta::assert_yaml_snapshot!("syntax_test_6_templates", template_names); } } #[cfg(test)] mod grammar_tests { + use parser::{grammar::entry::Scope, input::Input, parser::Parser}; - use rowan::ast::AstNode; + use rowan::{ast::AstNode, SyntaxNode}; use crate::{ abstract_syntax_tree::{AstBlock, AstPragma, AstTemplateDef}, syntax::SyntaxTreeBuilder, - syntax_node::SyntaxNode, + syntax_node::CircomLanguage, }; - #[test] - fn pragma_happy_test() { - // parse source (string) into output tree - let version = r#"2.0.1"#; - let source = format!(r#"pragma circom {};"#, version); + fn syntax_node_from_source(source: &str) -> SyntaxNode { let input = Input::new(&source); let output = Parser::parsing_with_scope(&input, Scope::Pragma); @@ -270,6 +250,17 @@ mod grammar_tests { // then cast green node into syntax node let syntax = SyntaxNode::new_root(green); + syntax + } + + #[test] + fn pragma_happy_test() { + // parse source (string) into output tree + let version = r#"2.0.1"#; + let source = format!(r#"pragma circom {};"#, version); + + let syntax = syntax_node_from_source(&source); + // cast syntax node into ast node to retrieve more information let pragma = AstPragma::cast(syntax).expect("Can not cast syntax node into ast pragma"); @@ -299,18 +290,7 @@ mod grammar_tests { }"#; - // parse source (string) into output tree - let input = Input::new(&SOURCE); - let output = Parser::parsing_with_scope(&input, Scope::Template); - - // output is a tree whose node is index of token, no content of token - // convert output into green node - let mut builder = SyntaxTreeBuilder::new(&input); - builder.build(output); - let green = builder.finish(); - - // then cast green node into syntax node - let syntax = SyntaxNode::new_root(green); + let syntax = syntax_node_from_source(&SOURCE); // cast syntax node into ast node to retrieve more information let template = @@ -391,37 +371,8 @@ mod grammar_tests { } out <== comp[N-2].out; }"#; - /* - let expected_statements = vec![ - "signal input in[N];", - "signal output out;", - "component comp[N-1];", - "for(var i = 0; i < N-1; i++){ - comp[i] = Multiplier2(); - }", - "comp[0].in1 <== in[0];", - "comp[0].in2 <== in[1];", - "for(var i = 0; i < N-2; i++){ - comp[i+1].in1 <== comp[i].out; - comp[i+1].in2 <== in[i+2]; - }", - "out <== comp[N-2].out;", - ]; - */ - - // parse source (string) into output tree - let input = Input::new(&source); - let output = Parser::parsing_with_scope(&input, Scope::Block); - - // output is a tree whose node is index of token, no content of token - // convert output into green node - let mut builder = SyntaxTreeBuilder::new(&input); - builder.build(output); - let green = builder.finish(); - - // then cast green node into syntax node - let syntax = SyntaxNode::new_root(green); + let syntax = syntax_node_from_source(&source); // cast syntax node into ast node to retrieve more information let block = AstBlock::cast(syntax).expect("Can not cast syntax node into ast block"); From 46929c2d65db1c28ddf66117c815285af4ca8fca Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 20 Dec 2024 12:34:32 +0700 Subject: [PATCH 22/31] update input tests with snapshot --- crates/parser/Cargo.toml | 11 +- crates/parser/src/input.rs | 265 +----------------- ...ser__input__tests__test_comment_block.snap | 43 +++ ...ser__input__tests__test_comment_error.snap | 34 +++ .../parser__input__tests__test_function.snap | 217 ++++++++++++++ .../parser__input__tests__test_pragma.snap | 52 ++++ 6 files changed, 364 insertions(+), 258 deletions(-) create mode 100644 crates/parser/src/snapshots/parser__input__tests__test_comment_block.snap create mode 100644 crates/parser/src/snapshots/parser__input__tests__test_comment_error.snap create mode 100644 crates/parser/src/snapshots/parser__input__tests__test_function.snap create mode 100644 crates/parser/src/snapshots/parser__input__tests__test_pragma.snap diff --git a/crates/parser/Cargo.toml b/crates/parser/Cargo.toml index 9070855..0173d54 100644 --- a/crates/parser/Cargo.toml +++ b/crates/parser/Cargo.toml @@ -14,4 +14,13 @@ num-derive = "0.2" serde = "1.0.216" [profile.dev] -debug = 2 \ No newline at end of file +debug = 2 + +[dev-dependencies] +# for snapshot testing, yaml format +insta = { version = "1.41.1", features = ["yaml"] } + +[profile.dev.package] +# compile slightly slower once, but use less memory, have faster diffs +insta.opt-level = 3 +similar.opt-level = 3 \ No newline at end of file diff --git a/crates/parser/src/input.rs b/crates/parser/src/input.rs index 8ddf602..a37579e 100644 --- a/crates/parser/src/input.rs +++ b/crates/parser/src/input.rs @@ -1,10 +1,11 @@ use std::ops::Range; use logos::Lexer; +use serde::Serialize; use crate::token_kind::TokenKind; -#[derive(Debug, PartialEq)] +#[derive(Debug, PartialEq, Serialize)] pub struct Input<'a> { kind: Vec, source: &'a str, @@ -81,65 +82,12 @@ impl<'a> Input<'a> { #[cfg(test)] mod tests { - use crate::token_kind::TokenKind::{self, *}; - use super::Input; - fn test(source: &str, expected_input: Input) { + fn test(source: &str, snapshot_name: &str) { let input = Input::new(&source); - assert_eq!( - expected_input, input, - "Tokens extract from source code are not correct" - ); - - // test size method - let expected_size = input.kind.len(); - let size = input.size(); - assert_eq!(expected_size, size, "size method failed"); - - // test methods with index out of bound - let index = input.kind.len(); - - let expected_token_value = None; - let token_value = input.token_value(index); - assert_eq!( - expected_token_value, token_value, - "token_value failed (case: index out of bound)" - ); - - let expected_kind = TokenKind::EOF; - let kind = input.kind_of(index); - assert_eq!( - expected_kind, kind, - "kind_of failed (case: index out of bound)" - ); - - let expected_position = None; - let position = input.position_of(index); - assert_eq!( - expected_position, position, - "position_of failed (case: index out of bound)" - ); - - // test methods with index in bound - if input.size() == 0 { - return; - } - - let index = input.size() / 2; // a valid index if input size > 0 - - let expected_token_value = &input.source[input.position[index].clone()]; - let token_value = input.token_value(index).unwrap(); - assert_eq!(expected_token_value, token_value, "token_value failed"); - - let expected_kind = input.kind[index]; - let kind = input.kind_of(index); - assert_eq!(expected_kind, kind, "kind_of failed"); - - let expected_position = input.position[index].clone(); - let position = input.position_of(index).unwrap(); - assert_eq!(expected_position, position, "position_of failed"); + insta::assert_yaml_snapshot!(snapshot_name, input); } #[test] @@ -148,40 +96,7 @@ mod tests { /*a + b == 10*/ a + 10 "#; - - let expected_input = Input { - kind: vec![ - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::BlockComment, - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::Identifier, - TokenKind::WhiteSpace, - TokenKind::Add, - TokenKind::WhiteSpace, - TokenKind::Number, - TokenKind::EndLine, - TokenKind::WhiteSpace, - ], - source: &source, - position: vec![ - { 0..1 }, - { 1..9 }, - { 9..24 }, - { 24..25 }, - { 25..33 }, - { 33..34 }, - { 34..35 }, - { 35..36 }, - { 36..37 }, - { 37..39 }, - { 39..40 }, - { 40..44 }, - ], - }; - - test(source, expected_input); + test(source, "test_comment_block"); } #[test] @@ -194,34 +109,7 @@ mod tests { /* "#; - - let expected_input = Input { - kind: vec![ - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::Pragma, - TokenKind::WhiteSpace, - TokenKind::Version, - TokenKind::Semicolon, - TokenKind::EndLine, - TokenKind::WhiteSpace, - TokenKind::Error, - ], - source: &source, - position: vec![ - 0..1, - 1..9, - 9..15, - 15..16, - 16..21, - 21..22, - 22..23, - 23..31, - 31..94, - ], - }; - - test(source, expected_input); + test(source, "test_comment_error"); } #[test] @@ -232,46 +120,7 @@ mod tests { pragma circom 2.0.0; "#; - - let expected_input = Input { - kind: vec![ - EndLine, - WhiteSpace, - BlockComment, - EndLine, - EndLine, - WhiteSpace, - Pragma, - WhiteSpace, - Circom, - WhiteSpace, - Version, - Semicolon, - EndLine, - EndLine, - WhiteSpace, - ], - source: &source, - position: vec![ - 0..1, - 1..9, - 9..38, - 38..39, - 39..40, - 40..44, - 44..50, - 50..51, - 51..57, - 57..58, - 58..63, - 63..64, - 64..65, - 65..66, - 66..70, - ], - }; - - test(source, expected_input); + test(source, "test_pragma"); } #[test] @@ -286,104 +135,6 @@ mod tests { } return r; }"#; - - let expected_input = Input { - kind: vec![ - EndLine, WhiteSpace, FunctionKw, WhiteSpace, Identifier, LParen, Identifier, - RParen, WhiteSpace, LCurly, EndLine, WhiteSpace, VarKw, WhiteSpace, Identifier, - WhiteSpace, Assign, WhiteSpace, Number, Semicolon, EndLine, WhiteSpace, VarKw, - WhiteSpace, Identifier, WhiteSpace, Assign, WhiteSpace, Number, Semicolon, EndLine, - WhiteSpace, WhileKw, WhiteSpace, LParen, Identifier, Sub, Number, LessThan, - Identifier, RParen, WhiteSpace, LCurly, EndLine, WhiteSpace, Identifier, Add, Add, - Semicolon, EndLine, WhiteSpace, Identifier, WhiteSpace, Mul, Assign, WhiteSpace, - Number, Semicolon, EndLine, WhiteSpace, RCurly, EndLine, WhiteSpace, ReturnKw, - WhiteSpace, Identifier, Semicolon, EndLine, WhiteSpace, RCurly, - ], - source: &source, - position: vec![ - 0..1, - 1..5, - 5..13, - 13..14, - 14..19, - 19..20, - 20..21, - 21..22, - 22..23, - 23..24, - 24..25, - 25..33, - 33..36, - 36..37, - 37..38, - 38..39, - 39..40, - 40..41, - 41..42, - 42..43, - 43..44, - 44..52, - 52..55, - 55..56, - 56..57, - 57..58, - 58..59, - 59..60, - 60..61, - 61..62, - 62..63, - 63..71, - 71..76, - 76..77, - 77..78, - 78..79, - 79..80, - 80..81, - 81..82, - 82..83, - 83..84, - 84..85, - 85..86, - 86..87, - 87..99, - 99..100, - 100..101, - 101..102, - 102..103, - 103..104, - 104..116, - 116..117, - 117..118, - 118..119, - 119..120, - 120..121, - 121..122, - 122..123, - 123..124, - 124..132, - 132..133, - 133..134, - 134..142, - 142..148, - 148..149, - 149..150, - 150..151, - 151..152, - 152..156, - 156..157, - ], - }; - - test(source, expected_input); + test(source, "test_function"); } - - // #[test] - // fn test_gen() { - // let source = r#" - // "#; - - // let input = Input::new(&source); - // println!("{:?}", input.kind); - // println!("{:?}", input.position); - // } } diff --git a/crates/parser/src/snapshots/parser__input__tests__test_comment_block.snap b/crates/parser/src/snapshots/parser__input__tests__test_comment_block.snap new file mode 100644 index 0000000..f6bdb27 --- /dev/null +++ b/crates/parser/src/snapshots/parser__input__tests__test_comment_block.snap @@ -0,0 +1,43 @@ +--- +source: crates/parser/src/input.rs +expression: input +--- +kind: + - EndLine + - WhiteSpace + - BlockComment + - EndLine + - WhiteSpace + - Identifier + - WhiteSpace + - Add + - WhiteSpace + - Number + - EndLine + - WhiteSpace +source: "\n /*a + b == 10*/\n a + 10\n " +position: + - start: 0 + end: 1 + - start: 1 + end: 9 + - start: 9 + end: 24 + - start: 24 + end: 25 + - start: 25 + end: 33 + - start: 33 + end: 34 + - start: 34 + end: 35 + - start: 35 + end: 36 + - start: 36 + end: 37 + - start: 37 + end: 39 + - start: 39 + end: 40 + - start: 40 + end: 44 diff --git a/crates/parser/src/snapshots/parser__input__tests__test_comment_error.snap b/crates/parser/src/snapshots/parser__input__tests__test_comment_error.snap new file mode 100644 index 0000000..f8307a5 --- /dev/null +++ b/crates/parser/src/snapshots/parser__input__tests__test_comment_error.snap @@ -0,0 +1,34 @@ +--- +source: crates/parser/src/input.rs +expression: input +--- +kind: + - EndLine + - WhiteSpace + - Pragma + - WhiteSpace + - Version + - Semicolon + - EndLine + - WhiteSpace + - Error +source: "\n pragma 2.1.1;\n /*a + b == 10*\n a + 10\n template\n\n /*\n " +position: + - start: 0 + end: 1 + - start: 1 + end: 9 + - start: 9 + end: 15 + - start: 15 + end: 16 + - start: 16 + end: 21 + - start: 21 + end: 22 + - start: 22 + end: 23 + - start: 23 + end: 31 + - start: 31 + end: 94 diff --git a/crates/parser/src/snapshots/parser__input__tests__test_function.snap b/crates/parser/src/snapshots/parser__input__tests__test_function.snap new file mode 100644 index 0000000..dd235e2 --- /dev/null +++ b/crates/parser/src/snapshots/parser__input__tests__test_function.snap @@ -0,0 +1,217 @@ +--- +source: crates/parser/src/input.rs +expression: input +--- +kind: + - EndLine + - WhiteSpace + - FunctionKw + - WhiteSpace + - Identifier + - LParen + - Identifier + - RParen + - WhiteSpace + - LCurly + - EndLine + - WhiteSpace + - VarKw + - WhiteSpace + - Identifier + - WhiteSpace + - Assign + - WhiteSpace + - Number + - Semicolon + - EndLine + - WhiteSpace + - VarKw + - WhiteSpace + - Identifier + - WhiteSpace + - Assign + - WhiteSpace + - Number + - Semicolon + - EndLine + - WhiteSpace + - WhileKw + - WhiteSpace + - LParen + - Identifier + - Sub + - Number + - LessThan + - Identifier + - RParen + - WhiteSpace + - LCurly + - EndLine + - WhiteSpace + - Identifier + - Add + - Add + - Semicolon + - EndLine + - WhiteSpace + - Identifier + - WhiteSpace + - Mul + - Assign + - WhiteSpace + - Number + - Semicolon + - EndLine + - WhiteSpace + - RCurly + - EndLine + - WhiteSpace + - ReturnKw + - WhiteSpace + - Identifier + - Semicolon + - EndLine + - WhiteSpace + - RCurly +source: "\n function nbits(a) {\n var n = 1;\n var r = 0;\n while (n-1 Date: Wed, 25 Dec 2024 12:17:10 +0700 Subject: [PATCH 23/31] change param type in find signal into str --- crates/parser/src/parser.rs | 4 ++-- .../src/abstract_syntax_tree/template.rs | 13 ++++++------- crates/syntax/src/syntax.rs | 19 +++++++++++-------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index a4f0efe..2fbe667 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -181,8 +181,8 @@ impl<'a> Parser<'a> { if self.at(kind) { self.advance(); } else { - // error report - // println!("expect {:?} but got {:?}", kind, current); + // TODO + // advance_with_error: ("expect {:?} but got {:?}", kind, current); } } diff --git a/crates/syntax/src/abstract_syntax_tree/template.rs b/crates/syntax/src/abstract_syntax_tree/template.rs index ab2f354..7253b12 100644 --- a/crates/syntax/src/abstract_syntax_tree/template.rs +++ b/crates/syntax/src/abstract_syntax_tree/template.rs @@ -1,5 +1,4 @@ use parser::token_kind::TokenKind::*; -use rowan::SyntaxText; use crate::syntax_node::CircomLanguage; use crate::syntax_node::SyntaxNode; @@ -45,11 +44,11 @@ impl AstTemplateDef { None } - pub fn find_input_signal(&self, name: &SyntaxText) -> Option { + pub fn find_input_signal(&self, name: &str) -> Option { if let Some(statements) = self.statements() { for input_signal in statements.find_children::() { if let Some(signal_name) = input_signal.name() { - if signal_name.equal(name) { + if signal_name.syntax().text() == name { return Some(input_signal); } } @@ -58,11 +57,11 @@ impl AstTemplateDef { None } - pub fn find_output_signal(&self, name: &SyntaxText) -> Option { + pub fn find_output_signal(&self, name: &str) -> Option { if let Some(statements) = self.statements() { for input_signal in statements.find_children::() { if let Some(signal_name) = input_signal.name() { - if signal_name.equal(name) { + if signal_name.syntax().text() == name { return Some(input_signal); } } @@ -71,11 +70,11 @@ impl AstTemplateDef { None } - pub fn find_internal_signal(&self, name: &SyntaxText) -> Option { + pub fn find_internal_signal(&self, name: &str) -> Option { if let Some(statements) = self.statements() { for signal in statements.find_children::() { if let Some(signal_name) = signal.name() { - if signal_name.equal(name) { + if signal_name.syntax().text() == name { return Some(signal); } } diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 0c1d1b8..a2fe669 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -230,16 +230,15 @@ mod grammar_tests { use parser::{grammar::entry::Scope, input::Input, parser::Parser}; use rowan::{ast::AstNode, SyntaxNode}; - use crate::{ - abstract_syntax_tree::{AstBlock, AstPragma, AstTemplateDef}, + abstract_syntax_tree::{AstBlock, AstOutputSignalDecl, AstPragma, AstTemplateDef}, syntax::SyntaxTreeBuilder, syntax_node::CircomLanguage, }; - fn syntax_node_from_source(source: &str) -> SyntaxNode { + fn syntax_node_from_source(source: &str, scope: Scope) -> SyntaxNode { let input = Input::new(&source); - let output = Parser::parsing_with_scope(&input, Scope::Pragma); + let output = Parser::parsing_with_scope(&input, scope); // output is a tree whose node is index of token, no content of token // convert output into green node @@ -259,7 +258,7 @@ mod grammar_tests { let version = r#"2.0.1"#; let source = format!(r#"pragma circom {};"#, version); - let syntax = syntax_node_from_source(&source); + let syntax = syntax_node_from_source(&source, Scope::Pragma); // cast syntax node into ast node to retrieve more information let pragma = AstPragma::cast(syntax).expect("Can not cast syntax node into ast pragma"); @@ -290,7 +289,7 @@ mod grammar_tests { }"#; - let syntax = syntax_node_from_source(&SOURCE); + let syntax = syntax_node_from_source(&SOURCE, Scope::Template); // cast syntax node into ast node to retrieve more information let template = @@ -329,8 +328,12 @@ mod grammar_tests { insta::assert_yaml_snapshot!(last_param, @"QQ"); // statements - let statements = template.statements().unwrap().statement_list(); + let statements = template.statements().unwrap(); + let output_signal = statements.find_children::(); + println!("{:?}", output_signal); + let statements: Vec = statements + .statement_list() .into_iter() .map(|statement| statement.syntax().text().to_string()) .collect(); @@ -372,7 +375,7 @@ mod grammar_tests { out <== comp[N-2].out; }"#; - let syntax = syntax_node_from_source(&source); + let syntax = syntax_node_from_source(&source, Scope::Block); // cast syntax node into ast node to retrieve more information let block = AstBlock::cast(syntax).expect("Can not cast syntax node into ast block"); From cafff028e7e9528b4d07ce64f9f5d8f2455d8b8e Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 27 Dec 2024 21:03:15 +0700 Subject: [PATCH 24/31] add snapshot guild --- SNAPSHOT_TEST.md | 11 +++++++++++ crates/syntax/src/syntax.rs | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 SNAPSHOT_TEST.md diff --git a/SNAPSHOT_TEST.md b/SNAPSHOT_TEST.md new file mode 100644 index 0000000..304166f --- /dev/null +++ b/SNAPSHOT_TEST.md @@ -0,0 +1,11 @@ +# Snapshot Test + +* Run all tests: + ``` + cargo test + ``` + +* Review snapshot changes + ``` + cargo insta review + ``` \ No newline at end of file diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index a2fe669..8e194e0 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -304,7 +304,7 @@ mod grammar_tests { .syntax() .text() .to_string(); - insta::assert_yaml_snapshot!(name, @"MultiplierN"); + insta::assert_yaml_snapshot!(name, @r###"" MultiplierN""###); // parameter list let first_param = template From 03da99a9dcd3c24258d3929ef329ca5365b4f4d6 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 27 Dec 2024 23:23:54 +0700 Subject: [PATCH 25/31] add some combine arithmetic operators --- crates/parser/src/token_kind.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs index 9eba89c..62b28bb 100644 --- a/crates/parser/src/token_kind.rs +++ b/crates/parser/src/token_kind.rs @@ -67,6 +67,26 @@ pub enum TokenKind { Semicolon, #[token(",")] Comma, + // TODO: review + #[token("++")] + UnitInc, + #[token("--")] + UnitDec, + #[token("+=")] + AddAssign, + #[token("-=")] + SubAssign, + #[token("/=")] + DivAssign, + #[token(r"\=")] + IntDivAssign, + #[token("%=")] + ModAssign, + #[token("*=")] + MulAssign, + #[token("**=")] + PowerAssign, + // end review #[token("=")] Assign, #[token("===")] @@ -227,12 +247,18 @@ impl TokenKind { Self::Add | Self::Sub => Some((92, 93)), Self::Mul | Self::Div | Self::IntDiv | Self::Mod => Some((94, 95)), Self::Power => Some((96, 97)), + // TODO: review + Self::AddAssign | Self::SubAssign => Some((98,99)), + Self::MulAssign | Self::DivAssign | Self::IntDivAssign | Self::ModAssign => Some((100,101)), + Self::PowerAssign => Some((102,103)), _ => None, } } pub fn prefix(self) -> Option { match self { + // TODO: review UnitDec, UnitInc + Self::UnitDec | Self::UnitInc => Some(101), Self::Sub => Some(100), Self::Not => Some(99), Self::BitNot => Some(98), @@ -242,6 +268,8 @@ impl TokenKind { pub fn postfix(self) -> Option { match self { + // TODO: review UnitDec, UnitInc + Self::UnitDec | Self::UnitInc => Some(202), Self::Dot => Some(200), Self::LBracket => Some(201), _ => None, From 7792a96c5d0a556a9b57ee638afe42267a0d92fe Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 27 Dec 2024 23:25:34 +0700 Subject: [PATCH 26/31] error report in expect and expect_any --- crates/parser/src/event.rs | 3 ++- crates/parser/src/output.rs | 8 ++++++++ crates/parser/src/parser.rs | 19 +++++++++++++------ crates/syntax/src/syntax.rs | 27 ++++++++++++++++++++------- 4 files changed, 43 insertions(+), 14 deletions(-) diff --git a/crates/parser/src/event.rs b/crates/parser/src/event.rs index 16996b9..bd9c2cc 100644 --- a/crates/parser/src/event.rs +++ b/crates/parser/src/event.rs @@ -1,8 +1,9 @@ use crate::token_kind::TokenKind; -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone)] pub enum Event { Open { kind: TokenKind }, Close, TokenPosition(usize), + ErrorReport(String), } diff --git a/crates/parser/src/output.rs b/crates/parser/src/output.rs index 271c2c3..b594a3e 100644 --- a/crates/parser/src/output.rs +++ b/crates/parser/src/output.rs @@ -3,6 +3,7 @@ use crate::{event::Event, token_kind::TokenKind}; #[derive(Debug)] pub enum Child { Token(usize), // position of token, + Error(String), Tree(Tree), } @@ -58,6 +59,13 @@ impl From> for Output { .children .push(Child::Token(*token)); } + Event::ErrorReport(error) => { + stack + .last_mut() + .unwrap() + .children + .push(Child::Error(error.clone())); + } } } } diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index 2fbe667..b121669 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -88,6 +88,15 @@ impl<'a> Parser<'a> { } self.close(m, TokenKind::Error); } + + pub fn error_report(&mut self, error: String) { + let m = self.open(); + + let token = Event::ErrorReport(error); + self.events.push(token); + + self.close(m, TokenKind::Error); + } } impl<'a> Parser<'a> { @@ -170,19 +179,17 @@ impl<'a> Parser<'a> { if kinds.contains(&kind) { self.advance(); } else { - // error report - // println!("expect {:?} but got {:?}", kinds, kind); + let error = format!("expect {:?} but got {:?}", kinds, kind); + self.error_report(error); } } pub fn expect(&mut self, kind: TokenKind) { - let _current = self.current(); - if self.at(kind) { self.advance(); } else { - // TODO - // advance_with_error: ("expect {:?} but got {:?}", kind, current); + let error = format!("expect {:?} but got {:?}", kind, self.current()); + self.error_report(error); } } diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 8e194e0..1d65867 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -1,6 +1,7 @@ use parser::input::Input; use parser::output::{Child, Output}; use parser::parser::Parser; +use parser::token_kind::TokenKind; use rowan::{GreenNode, GreenNodeBuilder}; use crate::syntax_node::SyntaxNode; @@ -30,6 +31,14 @@ impl<'a> SyntaxTreeBuilder<'a> { self.builder.finish_node(); } Child::Tree(child_tree) => self.build_rec(child_tree), + Child::Error(error) => { + let token_kind = TokenKind::Error; + let token_value = error.as_str(); + + self.builder.start_node(token_kind.into()); + self.builder.token(token_kind.into(), token_value); + self.builder.finish_node(); + } } } @@ -113,6 +122,10 @@ mod tests { .function_list() .iter() .map(|function| { + println!( + "function body:\n{:?}", + function.body().unwrap().syntax().text().to_string() + ); function .function_name() .unwrap() @@ -177,11 +190,11 @@ mod tests { let pragma = pragma_string_from_ast(&ast); insta::assert_yaml_snapshot!(pragma, @"pragma circom 2.0.0;"); - let template_names = template_names_from_ast(&ast); - insta::assert_yaml_snapshot!("syntax_test_2_templates", template_names); - let function_names = function_names_from_ast(&ast); insta::assert_yaml_snapshot!("syntax_test_2_functions", function_names); + + let template_names = template_names_from_ast(&ast); + insta::assert_yaml_snapshot!("syntax_test_2_templates", template_names); } #[test] @@ -228,13 +241,13 @@ mod tests { #[cfg(test)] mod grammar_tests { - use parser::{grammar::entry::Scope, input::Input, parser::Parser}; - use rowan::{ast::AstNode, SyntaxNode}; use crate::{ abstract_syntax_tree::{AstBlock, AstOutputSignalDecl, AstPragma, AstTemplateDef}, syntax::SyntaxTreeBuilder, syntax_node::CircomLanguage, }; + use parser::{grammar::entry::Scope, input::Input, parser::Parser}; + use rowan::{ast::AstNode, SyntaxNode}; fn syntax_node_from_source(source: &str, scope: Scope) -> SyntaxNode { let input = Input::new(&source); @@ -304,7 +317,7 @@ mod grammar_tests { .syntax() .text() .to_string(); - insta::assert_yaml_snapshot!(name, @r###"" MultiplierN""###); + insta::assert_yaml_snapshot!(name, @"MultiplierN"); // parameter list let first_param = template @@ -331,7 +344,7 @@ mod grammar_tests { let statements = template.statements().unwrap(); let output_signal = statements.find_children::(); println!("{:?}", output_signal); - + let statements: Vec = statements .statement_list() .into_iter() From 6ac9ac990ef9317eafc6cee33e44fa3c72513808 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Fri, 27 Dec 2024 23:30:44 +0700 Subject: [PATCH 27/31] update snapshot for combine operators --- .../snapshots/parser__input__tests__test_function.snap | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/crates/parser/src/snapshots/parser__input__tests__test_function.snap b/crates/parser/src/snapshots/parser__input__tests__test_function.snap index dd235e2..39cbb10 100644 --- a/crates/parser/src/snapshots/parser__input__tests__test_function.snap +++ b/crates/parser/src/snapshots/parser__input__tests__test_function.snap @@ -49,15 +49,13 @@ kind: - EndLine - WhiteSpace - Identifier - - Add - - Add + - UnitInc - Semicolon - EndLine - WhiteSpace - Identifier - WhiteSpace - - Mul - - Assign + - MulAssign - WhiteSpace - Number - Semicolon @@ -168,8 +166,6 @@ position: - start: 99 end: 100 - start: 100 - end: 101 - - start: 101 end: 102 - start: 102 end: 103 @@ -182,8 +178,6 @@ position: - start: 117 end: 118 - start: 118 - end: 119 - - start: 119 end: 120 - start: 120 end: 121 From ec73df88356ef33167bc043ad7f2c91fc0ee7962 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sat, 28 Dec 2024 14:31:30 +0700 Subject: [PATCH 28/31] update snapshot test --- .../syntax__syntax__tests__syntax_test_2_functions.snap | 2 +- .../syntax__syntax__tests__syntax_test_2_templates.snap | 2 +- .../syntax__syntax__tests__syntax_test_5_templates.snap | 2 +- .../syntax__syntax__tests__syntax_test_6_templates.snap | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap index 49ea709..0913804 100644 --- a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_functions.snap @@ -2,4 +2,4 @@ source: crates/syntax/src/syntax.rs expression: function_names --- -- "nbits" +- nbits diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap index 03a1b69..db61c87 100644 --- a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_2_templates.snap @@ -2,4 +2,4 @@ source: crates/syntax/src/syntax.rs expression: template_names --- -- "BinSum" +- BinSum diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap index 53cde17..8050901 100644 --- a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_5_templates.snap @@ -2,4 +2,4 @@ source: crates/syntax/src/syntax.rs expression: template_names --- -- "Multiplier2" +- Multiplier2 diff --git a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap index 53cde17..8050901 100644 --- a/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap +++ b/crates/syntax/src/snapshots/syntax__syntax__tests__syntax_test_6_templates.snap @@ -2,4 +2,4 @@ source: crates/syntax/src/syntax.rs expression: template_names --- -- "Multiplier2" +- Multiplier2 From 5bdda2fad69a2c84500f1b7da838ebc0fd917018 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sat, 28 Dec 2024 14:35:14 +0700 Subject: [PATCH 29/31] fix wrap trivial tokens before open a tree --- crates/parser/src/parser.rs | 37 ++++++++++++++++++++++++------------- crates/syntax/src/syntax.rs | 4 ---- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/crates/parser/src/parser.rs b/crates/parser/src/parser.rs index b121669..34a2f64 100644 --- a/crates/parser/src/parser.rs +++ b/crates/parser/src/parser.rs @@ -28,7 +28,29 @@ pub enum ParserError { } impl<'a> Parser<'a> { + pub fn wrap_trivial_tokens(&mut self) -> TokenKind { + loop { + let kind = self.input.kind_of(self.pos); + + if kind.is_trivial() == false { + return kind; + } + + self.events.push(Event::Open { kind }); + + self.fuel.set(256); + self.events.push(Event::TokenPosition(self.pos)); + self.skip(); + + self.events.push(Event::Close); + } + } + pub fn open(&mut self) -> Marker { + if self.events.len() > 0 { + self.wrap_trivial_tokens(); + } + let marker = Marker::Open(self.events.len()); self.events.push(Event::Open { kind: TokenKind::Error, @@ -119,19 +141,7 @@ impl<'a> Parser<'a> { } pub fn current(&mut self) -> TokenKind { - let mut kind: TokenKind; - loop { - kind = self.input.kind_of(self.pos); - if !kind.is_trivial() { - break; - } - - let m = self.open(); - self.advance(); - self.close(m, kind); - } - - kind + self.wrap_trivial_tokens() } pub fn next(&mut self) -> TokenKind { @@ -171,6 +181,7 @@ impl<'a> Parser<'a> { self.advance(); return true; } + false } diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 1d65867..054f39d 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -122,10 +122,6 @@ mod tests { .function_list() .iter() .map(|function| { - println!( - "function body:\n{:?}", - function.body().unwrap().syntax().text().to_string() - ); function .function_name() .unwrap() From 8e4853de21093dc348708f3ff50dc36d1e725214 Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sat, 28 Dec 2024 15:13:14 +0700 Subject: [PATCH 30/31] update template happy test --- crates/syntax/src/syntax.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/crates/syntax/src/syntax.rs b/crates/syntax/src/syntax.rs index 054f39d..a2f9f39 100644 --- a/crates/syntax/src/syntax.rs +++ b/crates/syntax/src/syntax.rs @@ -348,17 +348,21 @@ mod grammar_tests { .collect(); insta::assert_yaml_snapshot!("template_happy_test_statements", statements); - // // input signal - // println!("find_input_signal: {:?}", template.find_input_signal()); + // input signal + let input_signal = template.find_input_signal("in").unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(input_signal, @"signal input in[N];"); - // // output signal - // println!("find_output_signal: {:?}", template.find_output_signal()); + // output signal + let output_signal = template.find_output_signal("out").unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(output_signal, @"signal output out;"); - // // internal signal - // println!("find_internal_signal: {:?}", template.find_internal_signal()); + // internal signal + let internal_signal = template.find_internal_signal("in").is_none(); + insta::assert_yaml_snapshot!(internal_signal, @"true"); - // // component - // println!("find_component: {:?}", template.find_component()); + // component + let component = template.find_component("comp").unwrap().syntax().text().to_string(); + insta::assert_yaml_snapshot!(component, @"component comp[N-1];"); } #[test] From bb95297c47d898881305384c17c76362e678367d Mon Sep 17 00:00:00 2001 From: NTTVy03 Date: Sat, 28 Dec 2024 17:43:19 +0700 Subject: [PATCH 31/31] add operators into token kinds --- crates/lsp/src/handler/goto_definition.rs | 2 +- crates/parser/src/token_kind.rs | 180 +++++++++++++--------- 2 files changed, 105 insertions(+), 77 deletions(-) diff --git a/crates/lsp/src/handler/goto_definition.rs b/crates/lsp/src/handler/goto_definition.rs index 4cba681..10ef0ed 100644 --- a/crates/lsp/src/handler/goto_definition.rs +++ b/crates/lsp/src/handler/goto_definition.rs @@ -228,7 +228,7 @@ template Y() { let file = FileDB::create(&source, Url::from_file_path(Path::new("/tmp")).unwrap()); let syntax_node = SyntaxTreeBuilder::syntax_tree(&source); - + if let Some(program_ast) = AstCircomProgram::cast(syntax_node) { for template in program_ast.template_list() { println!("{template:?}"); diff --git a/crates/parser/src/token_kind.rs b/crates/parser/src/token_kind.rs index 62b28bb..1265324 100644 --- a/crates/parser/src/token_kind.rs +++ b/crates/parser/src/token_kind.rs @@ -5,24 +5,29 @@ use serde::Serialize; #[allow(non_camel_case_types)] #[repr(u16)] pub enum TokenKind { + // Error #[error] Error = 0, + // Comments #[regex(r"//[^\n]*")] CommentLine, #[token("/*")] CommentBlockOpen, #[token("*/")] CommentBlockClose, + // Trivial #[regex("[ \t]+")] WhiteSpace, #[regex("[\n]")] EndLine, + // Circom #[token("pragma")] Pragma, #[token("circom")] Circom, #[regex("2.[0-9].[0-9]")] Version, + // Literals #[regex("[0-9]+")] Number, #[regex("[$_]*[a-zA-Z][a-zA-Z0-9_$]*")] @@ -31,26 +36,7 @@ pub enum TokenKind { CircomString, #[token("template")] TemplateKw, - #[token("function")] - FunctionKw, - #[token("component")] - ComponentKw, - #[token("main")] - MainKw, - #[token("public")] - PublicKw, - #[token("signal")] - SignalKw, - #[token("var")] - VarKw, - #[token("include")] - IncludeKw, - #[token("input")] - InputKw, - #[token("output")] - OutputKw, - #[token("log")] - LogKw, + // Brackets #[token("(")] LParen, #[token(")")] @@ -63,30 +49,94 @@ pub enum TokenKind { LBracket, #[token("]")] RBracket, + // Punctuation #[token(";")] Semicolon, #[token(",")] Comma, - // TODO: review - #[token("++")] - UnitInc, - #[token("--")] - UnitDec, + #[token(".")] + Dot, + // Boolean operators + #[token("&&")] + BoolAnd, + #[token("||")] + BoolOr, + #[token("!")] + Not, + // Relational operators + #[token("==")] + Equal, + #[token("!=")] + NotEqual, + #[token("<")] + LessThan, + #[token(">")] + GreaterThan, + #[token("<=")] + LessThanAndEqual, + #[token(">=")] + GreaterThanAndEqual, + // Arithmetic operators + #[token("+")] + Add, + #[token("-")] + Sub, + #[token("*")] + Mul, + #[token("**")] + Power, + #[token("/")] + Div, + #[token("\\")] + IntDiv, + #[token("%")] + Mod, + // Combined arithmetic assignment #[token("+=")] AddAssign, #[token("-=")] SubAssign, + #[token("*=")] + MulAssign, + #[token("**=")] + PowerAssign, #[token("/=")] DivAssign, #[token(r"\=")] IntDivAssign, #[token("%=")] ModAssign, - #[token("*=")] - MulAssign, - #[token("**=")] - PowerAssign, - // end review + #[token("++")] + UnitInc, + #[token("--")] + UnitDec, + // Bitwise operators + #[token("&")] + BitAnd, + #[token("|")] + BitOr, + #[token("~")] + BitNot, + #[token("^")] + BitXor, + #[token(">>")] + ShiftR, + #[token("<<")] + ShiftL, + // Combined bitwise assignments + #[token("&=")] + BitAndAssign, + #[token("|=")] + BitOrAssign, + #[token("~=")] + BitNotAssign, + #[token("^=")] + BitXorAssign, + #[token(">>=")] + ShiftRAssign, + #[token("<<=")] + ShiftLAssign, + // Assign #[token("=")] Assign, #[token("===")] @@ -99,56 +149,33 @@ pub enum TokenKind { RAssignSignal, #[token("<==")] RAssignConstraintSignal, - #[token("+")] - Add, - #[token("-")] - Sub, - #[token("/")] - Div, - #[token("*")] - Mul, - #[token("!")] - Not, - #[token("~")] - BitNot, - #[token("**")] - Power, - #[token("\\")] - IntDiv, - #[token("%")] - Mod, - #[token("<<")] - ShiftL, - #[token(">>")] - ShiftR, - #[token("&")] - BitAnd, - #[token("|")] - BitOr, - #[token("^")] - BitXor, - #[token("==")] - Equal, - #[token("!=")] - NotEqual, - #[token("<")] - LessThan, - #[token(">")] - GreaterThan, - #[token("<=")] - LessThanAndEqual, - #[token(">=")] - GreaterThanAndEqual, - #[token("&&")] - BoolAnd, - #[token("||")] - BoolOr, + // Conditional expressions #[token("?")] MarkQuestion, #[token(":")] Colon, - #[token(".")] - Dot, + // Keywords + #[token("function")] + FunctionKw, + #[token("component")] + ComponentKw, + #[token("main")] + MainKw, + #[token("public")] + PublicKw, + #[token("signal")] + SignalKw, + #[token("var")] + VarKw, + #[token("include")] + IncludeKw, + #[token("input")] + InputKw, + #[token("output")] + OutputKw, + #[token("log")] + LogKw, + // Statement keywords #[token("if")] IfKw, #[token("else")] @@ -161,6 +188,7 @@ pub enum TokenKind { ReturnKw, #[token("assert")] AssertKw, + // Complex token kind ForLoop, AssignStatement, CircomProgram,