From 2ffbeba8c69f44eff646be27c070f19ca843a83f Mon Sep 17 00:00:00 2001
From: Pelle Johnsen <pelle.johnsen@gmail.com>
Date: Sat, 4 Jul 2020 06:59:45 +0000
Subject: [PATCH] Fix illegal chars in identifier for glsl-new

---
 src/front/glsl_new/lex.rs       |  4 +--
 src/front/glsl_new/lex_tests.rs | 50 +++++++++++++++++++++++++++++++++
 src/front/glsl_new/parser.rs    |  1 +
 3 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/src/front/glsl_new/lex.rs b/src/front/glsl_new/lex.rs
index 6583d5e3f6..0b6d675025 100644
--- a/src/front/glsl_new/lex.rs
+++ b/src/front/glsl_new/lex.rs
@@ -107,7 +107,7 @@ pub fn consume_token(mut input: &str) -> (Option<Token>, &str) {
             }
         }
         'a'..='z' | 'A'..='Z' | '_' => {
-            let (word, rest, pos) = consume_any(input, |c| c.is_alphanumeric() || c == '_');
+            let (word, rest, pos) = consume_any(input, |c| c.is_ascii_alphanumeric() || c == '_');
             meta.chars.end = start + pos;
             match word {
                 "void" => (Some(Token::Void(meta)), rest),
@@ -227,7 +227,7 @@ pub fn consume_token(mut input: &str) -> (Option<Token>, &str) {
         }
         '~' => (Some(Token::Tilde(meta)), chars.as_str()),
         '?' => (Some(Token::Question(meta)), chars.as_str()),
-        _ => (None, chars.as_str()),
+        ch => (Some(Token::Unknown((meta, ch))), chars.as_str()),
     }
 }
 
diff --git a/src/front/glsl_new/lex_tests.rs b/src/front/glsl_new/lex_tests.rs
index 02c426ffa1..377f17f5fa 100644
--- a/src/front/glsl_new/lex_tests.rs
+++ b/src/front/glsl_new/lex_tests.rs
@@ -60,3 +60,53 @@ fn glsl_lex_line_comment() {
         "RightBrace(TokenMetadata { line: 2, chars: 1..2 })"
     );
 }
+
+#[test]
+fn glsl_lex_identifier() {
+    let source = "id123_OK 92No æNoø No¾ No好";
+    let lex = Lexer::new(source);
+    let tokens: Vec<Token> = lex.collect();
+    assert_eq!(tokens.len(), 10);
+
+    let mut iter = tokens.iter();
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 0..8 }, \"id123_OK\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "IntConstant((TokenMetadata { line: 0, chars: 9..11 }, 92))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 11..13 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 14..15 }, \'æ\'))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 15..17 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 17..18 }, \'ø\'))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 19..21 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 21..22 }, \'¾\'))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Identifier((TokenMetadata { line: 0, chars: 23..25 }, \"No\"))"
+    );
+    assert_eq!(
+        format!("{:?}", iter.next().unwrap()),
+        "Unknown((TokenMetadata { line: 0, chars: 25..26 }, \'好\'))"
+    );
+}
diff --git a/src/front/glsl_new/parser.rs b/src/front/glsl_new/parser.rs
index 3e80bcd7dd..10ac3af5c8 100644
--- a/src/front/glsl_new/parser.rs
+++ b/src/front/glsl_new/parser.rs
@@ -24,6 +24,7 @@ pomelo! {
         ErrorKind::ParserStackOverflow
     }
 
+    %type Unknown char;
     %type Identifier String;
     %type IntConstant i64;
     %type UintConstant u64;