Merge pull request #500 from bdke/dev

b3b00 · Nov 6, 2024 · ca49894 · ca49894
2 parents 3e5eed2 + 47639d4
commit ca49894
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 7 deletions.
diff --git a/src/sly/lexer/LexerBuilder.cs b/src/sly/lexer/LexerBuilder.cs
@@ -267,8 +267,7 @@ private static BuildResult<ILexer<IN>> BuildRegexLexer<IN>(
                     {
                         foreach (var lexeme in lexemes.lexemes)
                         {
-                            var channel = lexeme.Channel.HasValue ? lexeme.Channel.Value : 0;
-                            lexer.AddDefinition(new TokenDefinition<IN>(tokenID, lexeme.Pattern, channel,
+                            lexer.AddDefinition(new TokenDefinition<IN>(tokenID, lexeme.Pattern, lexeme.Channel,
                                 lexeme.IsSkippable,
                                 lexeme.IsLineEnding));
                         }

diff --git a/src/sly/lexer/attributes/LexemeAttribute.cs b/src/sly/lexer/attributes/LexemeAttribute.cs
@@ -21,10 +21,8 @@ public LexemeAttribute(string pattern, bool isSkippable = false, bool isLineEndi
             IsLineEnding = isLineEnding;
         }
 
-
-        public LexemeAttribute(GenericToken generic,  params string[] parameters)
+        public LexemeAttribute(GenericToken generic, params string[] parameters)
         {
-            Channel = 0;
             GenericToken = generic;
             GenericTokenParameters = parameters;
         }
@@ -62,8 +60,9 @@ public LexemeAttribute(GenericToken generic, IdentifierType idType, string start
         public bool IsSkippable { get; set; }
 
         public bool IsLineEnding { get; set; }
-
-        public int? Channel { get; set; }
+
+        private int _channel = int.MinValue;
+        public int Channel { get => _channel != int.MinValue ? _channel : 0; set => _channel = value; }
 
 
         public bool HasGenericTokenParameters => GenericTokenParameters != null && GenericTokenParameters.Length > 0;

diff --git a/tests/ParserTests/Issue499/Issue499Test.cs b/tests/ParserTests/Issue499/Issue499Test.cs
@@ -0,0 +1,24 @@
+using NFluent;
+using sly.lexer;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Xunit;
+
+namespace ParserTests.Issue499;
+public class Issue499Test
+{
+    [Fact]
+    public void LexemeChannelNullable_Test()
+    {
+        var lexerBuildResult = LexerBuilder.BuildLexer<Issue499Token>();
+        Check.That(lexerBuildResult).IsOk();
+        var lexer = lexerBuildResult.Result;
+        var lexResult = lexer.Tokenize("test\"");
+        Check.That(lexResult).IsOkLexing();
+        var tokenChannels = lexResult.Tokens;
+        Check.That(tokenChannels.GetChannel(1).Count).IsGreaterOrEqualThan(1);
+    }
+}
diff --git a/tests/ParserTests/Issue499/Issue499Token.cs b/tests/ParserTests/Issue499/Issue499Token.cs
@@ -0,0 +1,12 @@
+using sly.lexer;
+
+namespace ParserTests.Issue499;
+internal enum Issue499Token
+{
+    EOF = 0,
+
+    [UpTo("\"", Channel = 1)]
+    Test,
+    [Sugar("\"", Channel = 0)]
+    Quote
+}