fix(lexer): fix true,false literals

fink-lang · Dec 2, 2020 · 745a5c0 · 745a5c0
1 parent 973f720
commit 745a5c0
Show file tree

Hide file tree

Showing 3 changed files with 49 additions and 4 deletions.
diff --git a/src/lexer/tokens.fnk b/src/lexer/tokens.fnk
@@ -23,11 +23,12 @@ get_loc = fn start, text:
 
 fink_lex = rx'
   ^(
-    (?<keyword>(
+    (?<block>(
       fn|match|fold|unfold|else|map|filter|while|until
-      |await|import|try|throw|rec|list|true|false|pipe
-      |in|or|and|not
+      |await|import|try|throw|rec|list|pipe
     )(?=\s|:))
+    |(?<kwop>(in|or|and|not)(?=\s))
+    |(?<value>(true|false)\b)
     |(?<ignorable>\s+)
     |(?<comment_sl>#.*?(?=\n))
     |(?<comment_ml>---[\s\S]*?---)
@@ -97,6 +98,8 @@ jsx_elem_lex = rx'
     |(?<comment_sl>#.*?(?=\n))
     |(?<comment_ml>---[\s\S]*?---)
 
+    |(?<value>(true|false)\b)
+
     |(?<str_start>\'|")
 
     |(?<jsx_elem_close>/>)

diff --git a/src/lexer/tokens.test.fnk b/src/lexer/tokens.test.fnk
@@ -36,6 +36,10 @@ describe 'tokenizer', fn:
         "shrub"
         "ni \${ham} spam"
         foo.bar ...ni
+        [true, false]
+        true, false
+        a not b
+        foo in bar
       '
       to_match_snapshot
 

diff --git a/src/lexer/tokens.test.fnk.snap b/src/lexer/tokens.test.fnk.snap
@@ -255,7 +255,45 @@ ident (97-99) (8:11-8:13)
   \\"ni\\"
 ignorable (99-100) (8:13-9:0)
   \\"\\\\n\\"
-end (100-100) (9:0-9:0)
+[ (100-101) (9:0-9:1)
+true (101-105) (9:1-9:5)
+, (105-106) (9:5-9:6)
+ignorable (106-107) (9:6-9:7)
+  \\" \\"
+false (107-112) (9:7-9:12)
+] (112-113) (9:12-9:13)
+ignorable (113-114) (9:13-10:0)
+  \\"\\\\n\\"
+true (114-118) (10:0-10:4)
+, (118-119) (10:4-10:5)
+ignorable (119-120) (10:5-10:6)
+  \\" \\"
+false (120-125) (10:6-10:11)
+ignorable (125-126) (10:11-11:0)
+  \\"\\\\n\\"
+ident (126-127) (11:0-11:1)
+  \\"a\\"
+ignorable (127-128) (11:1-11:2)
+  \\" \\"
+not (128-131) (11:2-11:5)
+ignorable (131-132) (11:5-11:6)
+  \\" \\"
+ident (132-133) (11:6-11:7)
+  \\"b\\"
+ignorable (133-134) (11:7-12:0)
+  \\"\\\\n\\"
+ident (134-137) (12:0-12:3)
+  \\"foo\\"
+ignorable (137-138) (12:3-12:4)
+  \\" \\"
+in (138-140) (12:4-12:6)
+ignorable (140-141) (12:6-12:7)
+  \\" \\"
+ident (141-144) (12:7-12:10)
+  \\"bar\\"
+ignorable (144-145) (12:10-13:0)
+  \\"\\\\n\\"
+end (145-145) (13:0-13:0)
   \\"\\""
 `;