From 094e4fdd175ebbd3268e020c30c68550e2b7ebfc Mon Sep 17 00:00:00 2001 From: Pelle Johnsen Date: Mon, 14 Sep 2020 07:16:12 +0200 Subject: [PATCH] Fix lexer operator issues (with tests) (#188) * Fix lexer operator issues (with tests) * [glsl-in] Don't convet to string in lexer tests * [glsl-in] cleanup lexer tests further - Consolidate use statements - Iterate lex directly, check for None at end --- src/front/glsl/lex.rs | 66 +++--- src/front/glsl/lex_tests.rs | 409 +++++++++++++++++++++++++++++------- src/front/glsl/parser.rs | 2 +- src/front/glsl/token.rs | 1 + 4 files changed, 368 insertions(+), 110 deletions(-) diff --git a/src/front/glsl/lex.rs b/src/front/glsl/lex.rs index 060b16c2e1..7e44a7901b 100644 --- a/src/front/glsl/lex.rs +++ b/src/front/glsl/lex.rs @@ -156,39 +156,41 @@ pub fn consume_token(mut input: &str) -> (Option, &str) { } } - '+' | '-' | '&' | '|' => { + '+' | '-' | '&' | '|' | '^' => { input = chars.as_str(); - match chars.next() { - Some('=') => { - meta.chars.end = start + 2; - match cur { - '+' => (Some(Token::AddAssign(meta)), chars.as_str()), - '-' => (Some(Token::SubAssign(meta)), chars.as_str()), - '&' => (Some(Token::AndAssign(meta)), chars.as_str()), - '|' => (Some(Token::OrAssign(meta)), chars.as_str()), - '^' => (Some(Token::XorAssign(meta)), chars.as_str()), - _ => (None, input), - } - } - Some(cur) => { - meta.chars.end = start + 2; - match cur { - '+' => (Some(Token::IncOp(meta)), chars.as_str()), - '-' => (Some(Token::DecOp(meta)), chars.as_str()), - '&' => (Some(Token::AndOp(meta)), chars.as_str()), - '|' => (Some(Token::OrOp(meta)), chars.as_str()), - '^' => (Some(Token::XorOp(meta)), chars.as_str()), - _ => (None, input), - } - } - _ => match cur { - '+' => (Some(Token::Plus(meta)), input), - '-' => (Some(Token::Dash(meta)), input), - '&' => (Some(Token::Ampersand(meta)), input), - '|' => (Some(Token::VerticalBar(meta)), input), - '^' => (Some(Token::Caret(meta)), input), + let next = chars.next(); + if next == Some(cur) { + meta.chars.end = start + 2; + match cur { + '+' => (Some(Token::IncOp(meta)), chars.as_str()), + '-' => (Some(Token::DecOp(meta)), chars.as_str()), + '&' => (Some(Token::AndOp(meta)), chars.as_str()), + '|' => (Some(Token::OrOp(meta)), chars.as_str()), + '^' => (Some(Token::XorOp(meta)), chars.as_str()), _ => (None, input), - }, + } + } else { + match next { + Some('=') => { + meta.chars.end = start + 2; + match cur { + '+' => (Some(Token::AddAssign(meta)), chars.as_str()), + '-' => (Some(Token::SubAssign(meta)), chars.as_str()), + '&' => (Some(Token::AndAssign(meta)), chars.as_str()), + '|' => (Some(Token::OrAssign(meta)), chars.as_str()), + '^' => (Some(Token::XorAssign(meta)), chars.as_str()), + _ => (None, input), + } + } + _ => match cur { + '+' => (Some(Token::Plus(meta)), input), + '-' => (Some(Token::Dash(meta)), input), + '&' => (Some(Token::Ampersand(meta)), input), + '|' => (Some(Token::VerticalBar(meta)), input), + '^' => (Some(Token::Caret(meta)), input), + _ => (None, input), + }, + } } } @@ -224,7 +226,7 @@ pub fn consume_token(mut input: &str) -> (Option, &str) { meta.chars.end = start + 2; (Some(Token::CommentEnd((meta, ()))), chars.as_str()) } - _ => (Some(Token::MulAssign(meta)), input), + _ => (Some(Token::Star(meta)), input), } } '/' => { diff --git a/src/front/glsl/lex_tests.rs b/src/front/glsl/lex_tests.rs index 5d260cae50..7710b4eb9e 100644 --- a/src/front/glsl/lex_tests.rs +++ b/src/front/glsl/lex_tests.rs @@ -1,160 +1,415 @@ -use super::lex::Lexer; -use super::parser::Token; +use super::{lex::Lexer, parser::Token::*, token::TokenMetadata}; #[test] fn glsl_lex_simple() { let source = "void main() {\n}"; - let lex = Lexer::new(source); - let tokens: Vec = lex.collect(); - assert_eq!(tokens.len(), 6); + let mut lex = Lexer::new(source); - let mut iter = tokens.iter(); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Void(TokenMetadata { line: 0, chars: 0..4 })" + lex.next().unwrap(), + Void(TokenMetadata { + line: 0, + chars: 0..4 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 5..9 }, \"main\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 5..9 + }, + "main".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "LeftParen(TokenMetadata { line: 0, chars: 9..10 })" + lex.next().unwrap(), + LeftParen(TokenMetadata { + line: 0, + chars: 9..10 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "RightParen(TokenMetadata { line: 0, chars: 10..11 })" + lex.next().unwrap(), + RightParen(TokenMetadata { + line: 0, + chars: 10..11 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "LeftBrace(TokenMetadata { line: 0, chars: 12..13 })" + lex.next().unwrap(), + LeftBrace(TokenMetadata { + line: 0, + chars: 12..13 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "RightBrace(TokenMetadata { line: 1, chars: 0..1 })" + lex.next().unwrap(), + RightBrace(TokenMetadata { + line: 1, + chars: 0..1 + }) ); + assert_eq!(lex.next(), None); } #[test] fn glsl_lex_line_comment() { let source = "void main // myfunction\n//()\n{}"; - let lex = Lexer::new(source); - let tokens: Vec = lex.collect(); - assert_eq!(tokens.len(), 4); - - let mut iter = tokens.iter(); + let mut lex = Lexer::new(source); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Void(TokenMetadata { line: 0, chars: 0..4 })" + lex.next().unwrap(), + Void(TokenMetadata { + line: 0, + chars: 0..4 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 5..9 }, \"main\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 5..9 + }, + "main".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "LeftBrace(TokenMetadata { line: 2, chars: 0..1 })" + lex.next().unwrap(), + LeftBrace(TokenMetadata { + line: 2, + chars: 0..1 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "RightBrace(TokenMetadata { line: 2, chars: 1..2 })" + lex.next().unwrap(), + RightBrace(TokenMetadata { + line: 2, + chars: 1..2 + }) ); + assert_eq!(lex.next(), None); } #[test] fn glsl_lex_multi_line_comment() { let source = "void main /* comment [] {}\n/**\n{}*/{}"; - let lex = Lexer::new(source); - let tokens: Vec = lex.collect(); - assert_eq!(tokens.len(), 4); + let mut lex = Lexer::new(source); - let mut iter = tokens.iter(); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Void(TokenMetadata { line: 0, chars: 0..4 })" + lex.next().unwrap(), + Void(TokenMetadata { + line: 0, + chars: 0..4 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 5..9 }, \"main\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 5..9 + }, + "main".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "LeftBrace(TokenMetadata { line: 2, chars: 4..5 })" + lex.next().unwrap(), + LeftBrace(TokenMetadata { + line: 2, + chars: 4..5 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "RightBrace(TokenMetadata { line: 2, chars: 5..6 })" + lex.next().unwrap(), + RightBrace(TokenMetadata { + line: 2, + chars: 5..6 + }) ); + assert_eq!(lex.next(), None); } #[test] fn glsl_lex_identifier() { let source = "id123_OK 92No æNoø No¾ No好"; - let lex = Lexer::new(source); - let tokens: Vec = lex.collect(); - assert_eq!(tokens.len(), 10); + let mut lex = Lexer::new(source); - let mut iter = tokens.iter(); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 0..8 }, \"id123_OK\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 0..8 + }, + "id123_OK".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "IntConstant((TokenMetadata { line: 0, chars: 9..11 }, 92))" + lex.next().unwrap(), + IntConstant(( + TokenMetadata { + line: 0, + chars: 9..11 + }, + 92 + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 11..13 }, \"No\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 11..13 + }, + "No".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Unknown((TokenMetadata { line: 0, chars: 14..15 }, \'æ\'))" + lex.next().unwrap(), + Unknown(( + TokenMetadata { + line: 0, + chars: 14..15 + }, + 'æ' + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 15..17 }, \"No\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 15..17 + }, + "No".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Unknown((TokenMetadata { line: 0, chars: 17..18 }, \'ø\'))" + lex.next().unwrap(), + Unknown(( + TokenMetadata { + line: 0, + chars: 17..18 + }, + 'ø' + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 19..21 }, \"No\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 19..21 + }, + "No".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Unknown((TokenMetadata { line: 0, chars: 21..22 }, \'¾\'))" + lex.next().unwrap(), + Unknown(( + TokenMetadata { + line: 0, + chars: 21..22 + }, + '¾' + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 23..25 }, \"No\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 23..25 + }, + "No".into() + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Unknown((TokenMetadata { line: 0, chars: 25..26 }, \'好\'))" + lex.next().unwrap(), + Unknown(( + TokenMetadata { + line: 0, + chars: 25..26 + }, + '好' + )) ); + assert_eq!(lex.next(), None); } #[test] fn glsl_lex_version() { let source = "#version 890 core"; - let lex = Lexer::new(source); - let tokens: Vec = lex.collect(); - assert_eq!(tokens.len(), 3); + let mut lex = Lexer::new(source); - let mut iter = tokens.iter(); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Version(TokenMetadata { line: 0, chars: 0..8 })" + lex.next().unwrap(), + Version(TokenMetadata { + line: 0, + chars: 0..8 + }) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "IntConstant((TokenMetadata { line: 0, chars: 9..12 }, 890))" + lex.next().unwrap(), + IntConstant(( + TokenMetadata { + line: 0, + chars: 9..12 + }, + 890 + )) ); assert_eq!( - format!("{:?}", iter.next().unwrap()), - "Identifier((TokenMetadata { line: 0, chars: 13..17 }, \"core\"))" + lex.next().unwrap(), + Identifier(( + TokenMetadata { + line: 0, + chars: 13..17 + }, + "core".into() + )) ); + assert_eq!(lex.next(), None); +} + +#[test] +fn glsl_lex_operators() { + let source = "+ - * | & % / += -= *= |= &= %= /= ++ -- || && ^^"; + let mut lex = Lexer::new(source); + + assert_eq!( + lex.next().unwrap(), + Plus(TokenMetadata { + line: 0, + chars: 0..1 + }) + ); + assert_eq!( + lex.next().unwrap(), + Dash(TokenMetadata { + line: 0, + chars: 2..3 + }) + ); + assert_eq!( + lex.next().unwrap(), + Star(TokenMetadata { + line: 0, + chars: 4..5 + }) + ); + assert_eq!( + lex.next().unwrap(), + VerticalBar(TokenMetadata { + line: 0, + chars: 6..7 + }) + ); + assert_eq!( + lex.next().unwrap(), + Ampersand(TokenMetadata { + line: 0, + chars: 8..9 + }) + ); + assert_eq!( + lex.next().unwrap(), + Percent(TokenMetadata { + line: 0, + chars: 10..11 + }) + ); + assert_eq!( + lex.next().unwrap(), + Slash(TokenMetadata { + line: 0, + chars: 12..13 + }) + ); + assert_eq!( + lex.next().unwrap(), + AddAssign(TokenMetadata { + line: 0, + chars: 14..16 + }) + ); + assert_eq!( + lex.next().unwrap(), + SubAssign(TokenMetadata { + line: 0, + chars: 17..19 + }) + ); + assert_eq!( + lex.next().unwrap(), + MulAssign(TokenMetadata { + line: 0, + chars: 20..22 + }) + ); + assert_eq!( + lex.next().unwrap(), + OrAssign(TokenMetadata { + line: 0, + chars: 23..25 + }) + ); + assert_eq!( + lex.next().unwrap(), + AndAssign(TokenMetadata { + line: 0, + chars: 26..28 + }) + ); + assert_eq!( + lex.next().unwrap(), + ModAssign(TokenMetadata { + line: 0, + chars: 29..31 + }) + ); + assert_eq!( + lex.next().unwrap(), + DivAssign(TokenMetadata { + line: 0, + chars: 32..34 + }) + ); + assert_eq!( + lex.next().unwrap(), + IncOp(TokenMetadata { + line: 0, + chars: 35..37 + }) + ); + assert_eq!( + lex.next().unwrap(), + DecOp(TokenMetadata { + line: 0, + chars: 38..40 + }) + ); + assert_eq!( + lex.next().unwrap(), + OrOp(TokenMetadata { + line: 0, + chars: 41..43 + }) + ); + assert_eq!( + lex.next().unwrap(), + AndOp(TokenMetadata { + line: 0, + chars: 44..46 + }) + ); + assert_eq!( + lex.next().unwrap(), + XorOp(TokenMetadata { + line: 0, + chars: 47..49 + }) + ); + assert_eq!(lex.next(), None); } diff --git a/src/front/glsl/parser.rs b/src/front/glsl/parser.rs index 57bb843c7b..c9e1743ba1 100644 --- a/src/front/glsl/parser.rs +++ b/src/front/glsl/parser.rs @@ -10,7 +10,7 @@ pomelo! { Statement, StorageAccess, StorageClass, Type, TypeInner, Interpolation}; } - %token #[derive(Debug)] pub enum Token {}; + %token #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub enum Token {}; %parser pub struct Parser<'a> {}; %extra_argument &'a mut Program; %extra_token TokenMetadata; diff --git a/src/front/glsl/token.rs b/src/front/glsl/token.rs index 31daf24cb1..7b849d361f 100644 --- a/src/front/glsl/token.rs +++ b/src/front/glsl/token.rs @@ -1,6 +1,7 @@ use std::ops::Range; #[derive(Debug, Clone)] +#[cfg_attr(test, derive(PartialEq))] pub struct TokenMetadata { pub line: usize, pub chars: Range,