From 39ccec5e7d41dd4f6a76e07bab2b1eb224b4496f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Capucho?= Date: Sun, 9 May 2021 15:48:11 +0100 Subject: [PATCH] [glsl-in] Update pp-rs for location info --- Cargo.toml | 2 +- src/front/glsl/lex.rs | 55 ++++++---------------------- src/front/glsl/parser.rs | 67 ++++++++++++++++++++++------------ src/front/glsl/parser_tests.rs | 8 ++-- src/front/glsl/token.rs | 9 ++++- 5 files changed, 67 insertions(+), 74 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1809e624e4..b67f450468 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ thiserror = "1.0.21" serde = { version = "1.0", features = ["derive"], optional = true } petgraph = { version ="0.5", optional = true } rose_tree = { version ="0.2", optional = true } -pp-rs = { version = "0.1", optional = true } +pp-rs = { git = "https://github.com/Kangz/glslpp-rs.git", optional = true, rev = "c02476b" } #env_logger = "0.8" # uncomment temporarily for developing with the binary target [features] diff --git a/src/front/glsl/lex.rs b/src/front/glsl/lex.rs index a07cc4af28..95cb240944 100644 --- a/src/front/glsl/lex.rs +++ b/src/front/glsl/lex.rs @@ -30,19 +30,13 @@ impl<'a> Lexer<'a> { impl<'a> Iterator for Lexer<'a> { type Item = Token; fn next(&mut self) -> Option { - let mut meta = TokenMetadata { - line: 0, - chars: 0..0, - }; + let mut meta = TokenMetadata { chars: 0..0 }; let pp_token = match self.tokens.pop_front() { Some(t) => t, None => match self.pp.next()? { Ok(t) => t, Err((err, loc)) => { - meta.line = loc.line as usize; - meta.chars.start = loc.pos as usize; - //TODO: proper location end - meta.chars.end = loc.pos as usize + 1; + meta.chars = loc.into(); return Some(Token { value: TokenValue::Unknown(err), meta, @@ -51,10 +45,7 @@ impl<'a> Iterator for Lexer<'a> { }, }; - meta.line = pp_token.location.line as usize; - meta.chars.start = pp_token.location.pos as usize; - //TODO: proper location end - meta.chars.end = pp_token.location.pos as usize + 1; + meta.chars = pp_token.location.into(); let value = match pp_token.value { PPTokenValue::Extension(extension) => { for t in extension.tokens { @@ -195,10 +186,7 @@ mod tests { lex.next().unwrap(), Token { value: TokenValue::Version, - meta: TokenMetadata { - line: 1, - chars: 1..2 //TODO - } + meta: TokenMetadata { chars: 1..8 } } ); assert_eq!( @@ -209,70 +197,49 @@ mod tests { value: 450, width: 32 }), - meta: TokenMetadata { - line: 1, - chars: 9..10 //TODO - }, + meta: TokenMetadata { chars: 9..12 }, } ); assert_eq!( lex.next().unwrap(), Token { value: TokenValue::Void, - meta: TokenMetadata { - line: 2, - chars: 0..1 //TODO - } + meta: TokenMetadata { chars: 13..17 } } ); assert_eq!( lex.next().unwrap(), Token { value: TokenValue::Identifier("main".into()), - meta: TokenMetadata { - line: 2, - chars: 5..6 //TODO - } + meta: TokenMetadata { chars: 18..22 } } ); assert_eq!( lex.next().unwrap(), Token { value: TokenValue::LeftParen, - meta: TokenMetadata { - line: 2, - chars: 10..11 //TODO - } + meta: TokenMetadata { chars: 23..24 } } ); assert_eq!( lex.next().unwrap(), Token { value: TokenValue::RightParen, - meta: TokenMetadata { - line: 2, - chars: 11..12 //TODO - } + meta: TokenMetadata { chars: 24..25 } } ); assert_eq!( lex.next().unwrap(), Token { value: TokenValue::LeftBrace, - meta: TokenMetadata { - line: 2, - chars: 13..14 //TODO - } + meta: TokenMetadata { chars: 26..27 } } ); assert_eq!( lex.next().unwrap(), Token { value: TokenValue::RightBrace, - meta: TokenMetadata { - line: 2, - chars: 14..15 //TODO - } + meta: TokenMetadata { chars: 27..28 } } ); assert_eq!(lex.next(), None); diff --git a/src/front/glsl/parser.rs b/src/front/glsl/parser.rs index f243ded155..040dbd32e9 100644 --- a/src/front/glsl/parser.rs +++ b/src/front/glsl/parser.rs @@ -436,7 +436,7 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { } fn parse_primary(&mut self, ctx: &mut Context) -> Result { - let token = self.bump()?; + let mut token = self.bump()?; let (width, value) = match token.value { TokenValue::IntConstant(int) => ( @@ -454,7 +454,9 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { TokenValue::BoolConstant(value) => (1, ScalarValue::Bool(value)), TokenValue::LeftParen => { let expr = self.parse_expression(ctx)?; - self.expect(TokenValue::RightParen)?; + let meta = self.expect(TokenValue::RightParen)?.meta; + + token.meta = token.meta.union(&meta); return Ok(expr); } @@ -476,11 +478,18 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { fn parse_postfix(&mut self, ctx: &mut Context) -> Result { let mut expr = match self.expect_peek()?.value { TokenValue::Identifier(_) => { - let (name, meta) = self.expect_ident()?; + let (name, mut meta) = self.expect_ident()?; if self.bump_if(TokenValue::LeftParen).is_some() { let mut args = Vec::new(); - while TokenValue::RightParen != self.bump()?.value { + loop { + let token = self.bump()?; + + if let TokenValue::RightParen = token.value { + meta = meta.union(&token.meta); + break; + } + args.push(self.parse_expression(ctx)?) } @@ -504,7 +513,7 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { } } TokenValue::TypeName(_) => { - let Token { value, meta } = self.bump()?; + let Token { value, mut meta } = self.bump()?; let handle = if let TokenValue::TypeName(ty) = value { self.program.module.types.append(ty) @@ -515,7 +524,14 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { self.expect(TokenValue::LeftParen)?; let mut args = Vec::new(); - while TokenValue::RightParen != self.bump()?.value { + loop { + let token = self.bump()?; + + if let TokenValue::RightParen = token.value { + meta = meta.union(&token.meta); + break; + } + args.push(self.parse_expression(ctx)?) } @@ -537,27 +553,25 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { match value { TokenValue::LeftBracket => { let index = Box::new(self.parse_expression(ctx)?); - self.expect(TokenValue::RightBracket)?; + let end_meta = self.expect(TokenValue::RightBracket)?.meta; - // TODO: metadata unification expr = Expr { kind: ExprKind::Access { base: Box::new(expr), index, }, - meta, + meta: meta.union(&end_meta), } } TokenValue::Dot => { - let field = self.expect_ident()?.0; + let (field, end_meta) = self.expect_ident()?; - // TODO: metadata unification expr = Expr { kind: ExprKind::Select { base: Box::new(expr), field, }, - meta, + meta: meta.union(&end_meta), } } _ => unreachable!(), @@ -574,6 +588,7 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { let Token { value, meta } = self.bump()?; let expr = self.parse_unary(ctx)?; + let end_meta = expr.meta.clone(); let kind = match value { TokenValue::Dash => ExprKind::Unary { @@ -587,8 +602,10 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { _ => return Ok(expr), }; - // TODO: metadata unification - Ok(Expr { kind, meta }) + Ok(Expr { + kind, + meta: meta.union(&end_meta), + }) } _ => self.parse_postfix(ctx), } @@ -603,15 +620,17 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { let mut expr = passtrough .ok_or(ErrorKind::EndOfFile /* Dummy error */) .or_else(|_| self.parse_unary(ctx))?; + let start_meta = expr.meta.clone(); while let Some((l_bp, r_bp)) = binding_power(&self.expect_peek()?.value) { if l_bp < min_bp { break; } - let Token { value, meta } = self.bump()?; + let Token { value, .. } = self.bump()?; let right = Box::new(self.parse_binary(ctx, None, r_bp)?); + let end_meta = right.meta.clone(); expr = Expr { kind: ExprKind::Binary { @@ -640,7 +659,7 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { }, right, }, - meta, + meta: start_meta.union(&end_meta), } } @@ -649,13 +668,13 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { fn parse_conditional(&mut self, ctx: &mut Context, passtrough: Option) -> Result { let mut condition = self.parse_binary(ctx, passtrough, 0)?; + let start_meta = condition.meta.clone(); - if let TokenValue::Question = self.expect_peek()?.value { - let meta = self.bump()?.meta; - + if self.bump_if(TokenValue::Question).is_some() { let accept = Box::new(self.parse_expression(ctx)?); self.expect(TokenValue::Colon)?; let reject = Box::new(self.parse_assignment(ctx)?); + let end_meta = reject.meta.clone(); condition = Expr { kind: ExprKind::Conditional { @@ -663,7 +682,7 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { accept, reject, }, - meta, + meta: start_meta.union(&end_meta), }; } @@ -672,18 +691,18 @@ impl<'source, 'program, 'options> Parser<'source, 'program, 'options> { fn parse_assignment(&mut self, ctx: &mut Context) -> Result { let pointer = self.parse_unary(ctx)?; + let start_meta = pointer.meta.clone(); - if let TokenValue::Assign = self.expect_peek()?.value { - let meta = self.bump()?.meta; - + if self.bump_if(TokenValue::Assign).is_some() { let value = Box::new(self.parse_assignment(ctx)?); + let end_meta = value.meta.clone(); Ok(Expr { kind: ExprKind::Assign { tgt: Box::new(pointer), value, }, - meta, + meta: start_meta.union(&end_meta), }) } else { self.parse_conditional(ctx, Some(pointer)) diff --git a/src/front/glsl/parser_tests.rs b/src/front/glsl/parser_tests.rs index 72a9f891d0..fc73362093 100644 --- a/src/front/glsl/parser_tests.rs +++ b/src/front/glsl/parser_tests.rs @@ -29,7 +29,7 @@ fn version() { .err() .unwrap() ), - "InvalidVersion(TokenMetadata { line: 1, chars: 9..10 }, 99000)" //TODO: location + "InvalidVersion(TokenMetadata { chars: 9..14 }, 99000)" ); assert_eq!( @@ -37,7 +37,7 @@ fn version() { "{:?}", parse_program("#version 449", &entry_points).err().unwrap() ), - "InvalidVersion(TokenMetadata { line: 1, chars: 9..10 }, 449)" //TODO: location + "InvalidVersion(TokenMetadata { chars: 9..12 }, 449)" ); assert_eq!( @@ -47,7 +47,7 @@ fn version() { .err() .unwrap() ), - "InvalidProfile(TokenMetadata { line: 1, chars: 13..14 }, \"smart\")" //TODO: location + "InvalidProfile(TokenMetadata { chars: 13..18 }, \"smart\")" ); assert_eq!( @@ -57,7 +57,7 @@ fn version() { .err() .unwrap() ), - "InvalidToken(Token { value: Unknown(UnexpectedHash), meta: TokenMetadata { line: 2, chars: 11..12 } })" + "InvalidToken(Token { value: Unknown(UnexpectedHash), meta: TokenMetadata { chars: 24..25 } })" ); // valid versions diff --git a/src/front/glsl/token.rs b/src/front/glsl/token.rs index e17ba7d5cc..3483f9d272 100644 --- a/src/front/glsl/token.rs +++ b/src/front/glsl/token.rs @@ -6,10 +6,17 @@ use std::ops::Range; #[derive(Debug, Clone)] #[cfg_attr(test, derive(PartialEq))] pub struct TokenMetadata { - pub line: usize, pub chars: Range, } +impl TokenMetadata { + pub fn union(&self, other: &Self) -> Self { + TokenMetadata { + chars: (self.chars.start.min(other.chars.start))..(self.chars.end.max(self.chars.end)), + } + } +} + #[derive(Debug)] #[cfg_attr(test, derive(PartialEq))] pub struct Token {