From 4bd5773ddee170527f65e70ac3086423e3bce7dc Mon Sep 17 00:00:00 2001
From: Elijah Voigt <elijah.caine.mv@gmail.com>
Date: Mon, 12 Aug 2024 18:11:26 -0700
Subject: [PATCH] Tokenizer test

---
 src/bin/tokenize.rs | 49 +++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 src/bin/tokenize.rs

diff --git a/src/bin/tokenize.rs b/src/bin/tokenize.rs
new file mode 100644
index 0000000..9c39605
--- /dev/null
+++ b/src/bin/tokenize.rs
@@ -0,0 +1,49 @@
+use nom::bytes::complete::tag;
+use nom::bytes::complete::take_until1;
+use nom::sequence::tuple;
+
+fn main() {
+    todo!()
+}
+
+#[derive(Debug, PartialEq)]
+enum Token {
+    Tag(String),
+    Str(String),
+    Num(f32),
+}
+
+fn tokenize(line: &str) -> Vec<Token> {
+    line.split(" ")
+        .map(|piece| {
+            if let Ok(n) = piece.parse::<f32>() {
+                Token::Num(n)
+            } else if let Ok((_, (_, s, _))) = tuple((
+                tag::<&str, &str, ()>("\""),
+                take_until1("\""),
+                tag::<&str, &str, ()>("\""),
+            ))(piece)
+            {
+                Token::Str(s.into())
+            } else {
+                Token::Tag(piece.into())
+            }
+        })
+        .collect()
+}
+
+#[test]
+fn test_tokenize() {
+    let line = "foo \"bar\" 1.23 baz \"asdf\" etc";
+    assert_eq!(
+        tokenize(line),
+        vec![
+            Token::Tag("foo".into()),
+            Token::Str("bar".into()),
+            Token::Num(1.23),
+            Token::Tag("baz".into()),
+            Token::Str("asdf".into()),
+            Token::Tag("etc".into())
+        ]
+    );
+}