|
|
|
|
@ -21,6 +21,8 @@ pub(crate) enum Token {
|
|
|
|
|
Tag(String),
|
|
|
|
|
Str(String),
|
|
|
|
|
Num(f32),
|
|
|
|
|
Comment(String),
|
|
|
|
|
Etc,
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
///
|
|
|
|
|
@ -30,32 +32,46 @@ pub(crate) fn tokenize(line: &str) -> Vec<Token> {
|
|
|
|
|
let mut l = line;
|
|
|
|
|
let mut tokens = Vec::new();
|
|
|
|
|
|
|
|
|
|
while l.len() > 0 {
|
|
|
|
|
debug!("Line: {:?}", l);
|
|
|
|
|
if let Ok((rem, (_, s, _))) = tuple((
|
|
|
|
|
char::<&str, ()>('"'),
|
|
|
|
|
take_until("\""),
|
|
|
|
|
char::<&str, ()>('"'),
|
|
|
|
|
))(l)
|
|
|
|
|
{
|
|
|
|
|
debug!("Parsed string {:?}", s);
|
|
|
|
|
tokens.push(Token::Str(s.into()));
|
|
|
|
|
l = rem;
|
|
|
|
|
} else if let Ok((rem, num)) = float::<&str, ()>(l) {
|
|
|
|
|
debug!("Parsed float {:?}", num);
|
|
|
|
|
tokens.push(Token::Num(num.into()));
|
|
|
|
|
l = rem;
|
|
|
|
|
} else if let Ok((rem, (_, tag, _))) = tuple((space0, alphanumeric1::<&str, ()>, space0))(l)
|
|
|
|
|
{
|
|
|
|
|
debug!("Parsed tag {:?}", tag);
|
|
|
|
|
tokens.push(Token::Tag(tag.into()));
|
|
|
|
|
l = rem;
|
|
|
|
|
} else {
|
|
|
|
|
debug!("Breaking loop");
|
|
|
|
|
break;
|
|
|
|
|
// Check for comment
|
|
|
|
|
if let Ok((_, (_start, content))) = tuple((char::<&str, ()>('#'), not_line_ending))(line) {
|
|
|
|
|
tokens.push(Token::Comment(content.strip_prefix(" ").unwrap().into()));
|
|
|
|
|
} else {
|
|
|
|
|
// Check for all other token types in a loop
|
|
|
|
|
while l.len() > 0 {
|
|
|
|
|
if let Ok((rem, (_, s, _))) = tuple((
|
|
|
|
|
char::<&str, ()>('"'),
|
|
|
|
|
take_until("\""),
|
|
|
|
|
char::<&str, ()>('"'),
|
|
|
|
|
))(l)
|
|
|
|
|
{
|
|
|
|
|
debug!("Parsed string {:?}", s);
|
|
|
|
|
tokens.push(Token::Str(s.into()));
|
|
|
|
|
l = rem;
|
|
|
|
|
} else if let Ok((rem, num)) = float::<&str, ()>(l) {
|
|
|
|
|
debug!("Parsed float {:?}", num);
|
|
|
|
|
tokens.push(Token::Num(num.into()));
|
|
|
|
|
l = rem;
|
|
|
|
|
} else if let Ok((rem, (_, etc, _))) =
|
|
|
|
|
tuple((space0, tag::<&str, &str, ()>("..."), space0))(l)
|
|
|
|
|
{
|
|
|
|
|
debug!("Parsed etc. {:?}", etc);
|
|
|
|
|
tokens.push(Token::Etc);
|
|
|
|
|
l = rem;
|
|
|
|
|
} else if let Ok((rem, (_, tag, _))) =
|
|
|
|
|
tuple((space0, alphanumeric1::<&str, ()>, space0))(l)
|
|
|
|
|
{
|
|
|
|
|
debug!("Parsed tag {:?}", tag);
|
|
|
|
|
tokens.push(Token::Tag(tag.into()));
|
|
|
|
|
l = rem;
|
|
|
|
|
} else {
|
|
|
|
|
debug!("Breaking loop");
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
debug!("Parsed tokens: {:?}", tokens);
|
|
|
|
|
|
|
|
|
|
tokens
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|