diff --git a/lexer/src/tests.rs b/lexer/src/tests.rs index 5ddabb0..6134556 100644 --- a/lexer/src/tests.rs +++ b/lexer/src/tests.rs @@ -355,3 +355,286 @@ fn lexer_test_unicode_escape_eight_digits() { &[Token::String(b"2".into())] ); } + +#[test] +fn lexer_test_operators() { + let arena = Bump::new(); + let str = b"~ !~ | |& >> ** **= ^= += -= *= %= == != <= >= < > && || !"; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::Matching, + Token::NotMatching, + Token::Pipe, + Token::DoublePipe, + Token::AppendPipe, + Token::Circumflex, + Token::CaretAssign, + Token::CaretAssign, + Token::PlusAssign, + Token::MinusAssign, + Token::StarAssign, + Token::PercentAssign, + Token::EqualTo, + Token::NotEqualTo, + Token::LesserOrEqualThan, + Token::GreaterOrEqualThan, + Token::LesserThan, + Token::GreaterThan, + Token::BooleanAnd, + Token::BooleanOr, + Token::Negation, + ] + ); +} + +#[test] +fn lexer_test_slash_assign() { + let arena = Bump::new(); + assert_eq!( + &lex(b"a/=1", &arena, false, false), + &[ + Token::Identifier(Identifier { namespace: None, literal: "a" }), + Token::SlashAssign, + Token::SmallInt(1), + ] + ); +} + +#[test] +fn lexer_test_control_flow_keywords() { + let arena = Bump::new(); + let str = b"switch case default getline printf next nextfile exit return continue break"; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::Switch, + Token::Case, + Token::Default, + Token::Getline, + Token::Printf, + Token::Next, + Token::NextFile, + Token::Exit, + Token::Return, + Token::Continue, + Token::Break, + ] + ); +} + +#[test] +fn lexer_test_builtin_variables() { + let arena = Bump::new(); + let str = + b"NR NF FS RS OFS ORS FILENAME ARGC ARGV SUBSEP FNR ARGIND OFMT RSTART RLENGTH ENVIRON"; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::NrVariable, + Token::NfVariable, + Token::FsVariable, + Token::RsVariable, + Token::OfsVariable, + Token::OrsVariable, + Token::FilenameVariable, + Token::ArgcVariable, + Token::ArgvVariable, + Token::SubsepVariable, + Token::FnrVariable, + Token::ArgindVariable, + Token::OfmtVariable, + Token::RstartVariable, + Token::RlengthVariable, + Token::EnvironVariable, + ] + ); +} + +#[test] +fn lexer_test_typed_regex() { + let arena = Bump::new(); + assert_eq!( + &lex(b"@/pat/", &arena, false, false), + &[Token::TypedRegex(b"pat".into())] + ); +} + +#[test] +fn lexer_test_comments() { + let arena = Bump::new(); + let str = b"print 1 # comment\nprint 2"; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::Print, + Token::SmallInt(1), + Token::Newline, + Token::Print, + Token::SmallInt(2), + ] + ); +} + +#[test] +fn lexer_test_indirect_call() { + let arena = Bump::new(); + assert_eq!( + &lex(b"@foo @ns::bar", &arena, false, false), + &[ + Token::IndirectCall(Identifier { namespace: None, literal: "foo" }), + Token::IndirectCall(Identifier { namespace: Some("ns"), literal: "bar" }), + ] + ); +} + +#[test] +#[should_panic] +fn lexer_test_indirect_call_posix() { + let arena = Bump::new(); + lex(b"@foo", &arena, true, false); +} + +#[test] +fn lexer_test_concurrent_directive() { + let arena = Bump::new(); + assert_eq!( + &lex(b"@concurrent", &arena, false, false), + &[Token::ConcurrentDirective] + ); +} + +#[test] +fn lexer_test_load_and_namespace_directives() { + let arena = Bump::new(); + assert_eq!( + &lex(br#"@load "lib.so" @namespace "ns""#, &arena, false, false), + &[ + Token::LoadDirective, + Token::String(b"lib.so".into()), + Token::NamespaceDirective, + Token::String(b"ns".into()), + ] + ); +} + +#[test] +fn lexer_test_regex_literals() { + let arena = Bump::new(); + assert_eq!( + &lex(b"/abc/", &arena, false, false), + &[Token::Regex(b"abc".into())] + ); + assert_eq!( + &lex(b"/a\\/b/", &arena, false, false), + &[Token::Regex(b"a/b".into())] + ); + assert_eq!( + &lex(b"x~/dot+/", &arena, false, false), + &[ + Token::Identifier(Identifier { namespace: None, literal: "x" }), + Token::Matching, + Token::Regex(b"dot+".into()), + ] + ); +} + +#[test] +fn lexer_test_switch_snippet() { + let arena = Bump::new(); + let str = br#"switch (x) { case 1: print; default: break }"#; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::Switch, + Token::OpenParent, + Token::Identifier(Identifier { namespace: None, literal: "x" }), + Token::ClosedParent, + Token::OpenBrace, + Token::Case, + Token::SmallInt(1), + Token::Colon, + Token::Print, + Token::Semicolon, + Token::Default, + Token::Colon, + Token::Break, + Token::ClosedBrace, + ] + ); +} + +#[test] +fn lexer_test_getline_redirection() { + let arena = Bump::new(); + let str = br#"getline getline x < "f" "cmd" | getline "cmd" |& getline"#; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::Getline, + Token::Getline, + Token::Identifier(Identifier { namespace: None, literal: "x" }), + Token::LesserThan, + Token::String(b"f".into()), + Token::String(b"cmd".into()), + Token::Pipe, + Token::Getline, + Token::String(b"cmd".into()), + Token::DoublePipe, + Token::Getline, + ] + ); +} + +#[test] +fn lexer_test_print_redirection() { + let arena = Bump::new(); + let str = br#"print > "out" print >> "out" print | "cmd" print |& "cmd""#; + assert_eq!( + &lex(str, &arena, false, false), + &[ + Token::Print, + Token::GreaterThan, + Token::String(b"out".into()), + Token::Print, + Token::AppendPipe, + Token::String(b"out".into()), + Token::Print, + Token::Pipe, + Token::String(b"cmd".into()), + Token::Print, + Token::DoublePipe, + Token::String(b"cmd".into()), + ] + ); +} + +#[test] +fn lexer_test_func_keyword() { + let arena = Bump::new(); + assert_eq!( + &lex(b"func function", &arena, false, false), + &[Token::Function, Token::Function] + ); +} + +#[test] +fn lexer_test_func_keyword_posix() { + let arena = Bump::new(); + assert_eq!( + &lex(b"func", &arena, true, false), + &[Token::Identifier(Identifier { + namespace: None, + literal: "func" + })] + ); +} + +#[test] +fn lexer_test_backslash_escape_in_string() { + let arena = Bump::new(); + assert_eq!( + &lex(b"\"\\n\\t\\\\\"", &arena, false, false), + &[Token::String(b"\n\t\\".into())] + ); +} diff --git a/parser/src/tests.rs b/parser/src/tests.rs index f3abdde..5929ee2 100644 --- a/parser/src/tests.rs +++ b/parser/src/tests.rs @@ -509,3 +509,230 @@ fn test_parser_do_while() { ], }); } + +#[test] +fn test_parser_switch() { + let source = r#" + { switch (x) { case 1: print; case "a": print 2; default: print 3 } } + { switch (1 + 1) { case /pat/: print } } + "#; + test_parser!(source => { + rules: [ + ( + None, + Some(concat!( + "(body (switch awk::x", + " (case 1 (body (Print)))", + " (case \"a\" (body (Print 2)))", + " (default 2 (body (Print 3)))))" + )) + ), + ( + None, + Some("(body (switch (Add 1 1) (case /pat/ (body (Print)))))") + ), + ], + }); + + test_parser!(is_err!( + "{ switch (x) { default: print; default: print } }", + "{ switch (x) { case a: print } }" + )); +} + +#[test] +fn test_parser_getline() { + let source = r#" + { getline } + { getline x } + { getline < "f" } + { getline x < "f" } + { "cmd" | getline } + { "cmd" | getline x } + { "cmd" |& getline } + { "cmd" |& getline x } + "#; + test_parser!(source => { + rules: [ + (None, Some("(body (getline))")), + (None, Some("(body (getline awk::x))")), + (None, Some("(body (getline< \"f\"))")), + (None, Some("(body (getline< \"f\" awk::x))")), + (None, Some("(body (getline| \"cmd\"))")), + (None, Some("(body (getline| \"cmd\" awk::x))")), + (None, Some("(body (getline|& \"cmd\"))")), + (None, Some("(body (getline|& \"cmd\" awk::x))")), + ], + }); +} + +#[test] +fn test_parser_printf() { + let source = r#" + { printf "%d", 1, 2 } + { printf("%d", 1) } + "#; + test_parser!(source => { + rules: [ + (None, Some("(body (Printf \"%d\" 1 2))")), + (None, Some("(body (Printf \"%d\" 1))")), + ], + }); +} + +#[test] +fn test_parser_control_flow() { + let source = r" + { next } + { nextfile } + { continue } + { return } + { return 1 } + { exit } + { exit 1 } + "; + test_parser!(source => { + rules: [ + (None, Some("(body (next))")), + (None, Some("(body (nextfile))")), + (None, Some("(body (continue))")), + (None, Some("(body (return))")), + (None, Some("(body (return 1))")), + (None, Some("(body (exit))")), + (None, Some("(body (exit 1))")), + ], + }); +} + +#[test] +fn test_parser_redirection() { + let source = r#" + { print > "out" } + { print >> "out" } + { print | "cmd" } + { print |& "cmd" } + "#; + test_parser!(source => { + rules: [ + (None, Some("(body (Print (> \"out\")))")), + (None, Some("(body (Print (>> \"out\")))")), + (None, Some("(body (Print (| \"cmd\")))")), + (None, Some("(body (Print (|& \"cmd\")))")), + ], + }); +} + +#[test] +fn test_parser_regex_matching() { + let source = r" + { a ~ /b/ } + { a !~ /b/ } + { /x/ ~ /y/ } + "; + test_parser!(source => { + rules: [ + (None, Some("(body (Matches awk::a @/b/))")), + (None, Some("(body (MatchesNot awk::a @/b/))")), + (None, Some("(body (Matches /x/ @/y/))")), + ], + }); +} + +#[test] +fn test_parser_function_calls() { + let source = r" + { foo(1, 2) } + { @bar(3) } + { foo::baz(a, b + 1) } + "; + test_parser!(source => { + rules: [ + (None, Some("(body (awk::foo 1 2))")), + (None, Some("(body (@awk::bar 3))")), + (None, Some("(body (foo::baz awk::a (Add awk::b 1)))")), + ], + }); +} + +#[test] +fn test_parser_compound_assignments() { + let source = r" + { a -= 1 } + { a *= 2 } + { a /= 3 } + { a ^= 4 } + { a %= 5 } + { a /= /pat/ } + "; + test_parser!(source => { + rules: [ + (None, Some("(body (SubAssign awk::a 1))")), + (None, Some("(body (MulAssign awk::a 2))")), + (None, Some("(body (DivAssign awk::a 3))")), + (None, Some("(body (PowAssign awk::a 4))")), + (None, Some("(body (ModAssign awk::a 5))")), + (None, Some("(body (DivAssign awk::a /pat/))")), + ], + }); +} + +#[test] +fn test_parser_builtin_variables() { + let source = r" + { NR; NF; FS; RS; OFS; ORS; FILENAME; ARGC; ARGV; SUBSEP; FNR; OFMT; RSTART; RLENGTH; ENVIRON } + "; + test_parser!(source => { + rules: [( + None, + Some(concat!( + "(body NR NF FS RS OFS ORS FILENAME ARGC ARGV SUBSEP FNR OFMT ", + "RSTART RLENGTH ENVIRON)" + )) + )], + }); +} + +#[test] +fn test_parser_concurrent() { + let source = r" + @concurrent { print 1 } + @concurrent $1 { print 2 } + "; + test_parser!(source => { + concurrent: [ + (None, Some("(body (Print 1))")), + (Some("(Record 1)"), Some("(body (Print 2))")), + ], + }); +} + +#[test] +fn test_parser_namespace_directive() { + let source = r#" + @namespace "myns"; + function bar() { print } + "#; + test_parser!(source => { + functions: [("myns::bar", &[], "(body (Print))")], + }); +} + +#[test] +fn test_parser_unary_and_divide() { + let source = r" + { -a; +a; !a; a / b; a + b - c } + { int(a) } + "; + test_parser!(source => { + rules: [ + ( + None, + Some(concat!( + "(body (Negative awk::a) (ToInt awk::a) (Negation awk::a) ", + "(Divide awk::a awk::b) (Subtract (Add awk::a awk::b) awk::c))" + )) + ), + (None, Some("(body (awk::int awk::a))")), + ], + }); +}