Mark
/
daisy
mirror of https://github.com/rm-dr/daisy
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rewrote parser

pull/2/head
Mark 2023-03-27 21:22:29 -07:00
parent 0bfeea09ed
commit 1996693008
Signed by: Mark
GPG Key ID: AD62BB059C2AAEE4
8 changed files with 401 additions and 380 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/evaluate.rs
View File

@ -27,13 +27,13 @@ pub fn evaluate(
if inner.is_none() || *t >= inner.as_ref().unwrap().len() {
coords.pop();
if coords.len() == 0 { break 'outer; }
let p = get_at_coords(&mut g, &coords);
let e = p.eval();
*p = e;
if coords.len() == 0 { break 'outer; }
let l = coords.pop().unwrap();
coords.push(l + 1);
continue 'outer;
@ -43,13 +43,7 @@ pub fn evaluate(
}
match h {
Token::Multiply(_) |
Token::Divide(_) |
Token::Add(_) |
Token::Factorial(_) |
Token::Negative(_) |
Token::Power(_) |
Token::Modulo(_)
Token::Operator(_,_,_)
=> {
coords.push(0);
continue 'outer;
@ -61,17 +55,8 @@ pub fn evaluate(
coords.push(l + 1);
continue 'outer;
}
Token::PreNumber(_,_) |
Token::PreWord(_,_) |
Token::PreOperator(_,_) |
Token::PreGroup(_,_) |
Token::PreGroupStart(_) |
Token::PreGroupEnd(_)
=> panic!()
};
}
return Ok(g);
}

17
src/main.rs
View File

@ -140,12 +140,10 @@ fn main() -> Result<(), std::io::Error> {
#[cfg(debug_assertions)]
RawTerminal::suspend_raw_mode(&stdout)?;
let g = evaluate::evaluate(g).unwrap();
let n = g.eval();
#[cfg(debug_assertions)]
RawTerminal::activate_raw_mode(&stdout)?;
if let Token::Number(_, v) = n {
if let Token::Number(_, v) = g {
write!(
stdout, "\r\n {}{}={} {v}{}\r\n\n",
style::Bold,
@ -204,9 +202,6 @@ fn main() -> Result<(), std::io::Error> {
#[cfg(test)]
mod tests {
// Many of these have been borrowed from insect.
@ -293,6 +288,14 @@ mod tests {
bad_expr("3+2)");
}
#[test]
fn implicit_multiply() {
good_expr(15f64, "5(3)");
good_expr(15f64, "(5)3");
good_expr(15f64, "(5)(3)");
bad_expr("5 2");
}
#[test]
fn operators() {
good_expr(125f64, "5^3");
@ -329,8 +332,6 @@ mod tests {
good_expr(15f64, "( 5 ) ( 3 )");
good_expr(15f64, "( ( 5 ) * ( 3 ) )");
good_expr(15f64, "( 5 * 3 )");
good_expr(15f64, "5(3)");
good_expr(15f64, "(5)3");
//good_expr(15f64, "5(+3)");
//good_expr(15f64, "+5*3");

35
src/parser/find_subs.rs
View File

@ -1,22 +1,21 @@
use std::collections::VecDeque;
use crate::tokens::Token;
use crate::tokens::Operator;
use crate::parser::PreToken;
use crate::tokens::LineLocation;
pub fn p_find_subs(
mut g: VecDeque<Token>,
pub(in crate::parser) fn find_subs(
mut g: VecDeque<PreToken>,
) -> (
Vec<(LineLocation, String)>,
VecDeque<Token>
VecDeque<PreToken>
) {
// Array of replacements
let mut r: Vec<(LineLocation, String)> = Vec::with_capacity(8);
// New token array, with updated locations
let mut n: VecDeque<Token> = VecDeque::with_capacity(g.len());
let mut n: VecDeque<PreToken> = VecDeque::with_capacity(g.len());
let mut offset: usize = 0;
@ -25,15 +24,20 @@ pub fn p_find_subs(
let mut t = g.pop_back().unwrap();
let target: Option<&str> = match &mut t {
Token::PreOperator(_, o) => {
match o {
Operator::Multiply => {Some("×")},
Operator::Divide => {Some("÷")},
PreToken::PreOperator(_, s) => {
let target = match &s[..] {
"*" => {Some("×")},
"/" => {Some("÷")},
_ => {None}
}
};
// Update token contents too.
// This makes sure that errors also contain the updated text.
if target.is_some() { *s = String::from(target.unwrap()); }
target
},
Token::PreWord(_, s) => {
PreToken::PreWord(_, s) => {
let target = match &s[..] {
// Greek letters
"alpha" => {Some("α")},
@ -65,12 +69,7 @@ pub fn p_find_subs(
_ => {None}
};
// Update preword contents too.
// This makes sure future prints of this token
// contain substituted text too.
if target.is_some() {*
s = String::from(target.unwrap());
}
if target.is_some() { *s = String::from(target.unwrap()); }
target
},

135
src/parser/groupify.rs
View File

@ -1,46 +1,40 @@
use std::collections::VecDeque;
use crate::tokens::Token;
use crate::tokens::Operator;
use crate::tokens::LineLocation;
use crate::tokens::Operator;
use crate::parser::PreToken;
use crate::parser::ParserError;
/// Looks backwards at the elements of g.
/// - Inserts ImplicitMultiply
/// - Removes multiple PreNegatives
/// - Applies PreNegative to Numbers
/// - Parses factorials
/// - Checks syntax
#[inline(always)]
// Inserts implicit operators
fn lookback(
g: &mut VecDeque<Token>
g: &mut VecDeque<PreToken>
) -> Result<(), (LineLocation, ParserError)> {
if g.len() >= 2 {
let b: Token = g.pop_back().unwrap();
let a: Token = g.pop_back().unwrap();
let b: PreToken = g.pop_back().unwrap();
let a: PreToken = g.pop_back().unwrap();
match (&a, &b) {
// Insert ImplicitMultiply
(Token::PreGroup(_,_), Token::PreGroup(l ,_)) |
(Token::PreGroup(_,_), Token::Number(l,_)) |
(Token::Number(_,_), Token::PreGroup(l,_)) |
(Token::Constant(_,_,_), Token::Number(l,_)) |
(Token::Number(_,_), Token::Constant(l,_,_)) |
(Token::Constant(_,_,_), Token::PreGroup(l,_)) |
(Token::PreGroup(_,_), Token::Constant(l,_,_)) |
(Token::Constant(_,_,_), Token::Constant(l,_,_))
(PreToken::PreGroup(_,_), PreToken::PreGroup(l ,_))
| (PreToken::PreGroup(_,_), PreToken::PreNumber(l,_))
| (PreToken::PreNumber(_,_), PreToken::PreGroup(l,_))
| (PreToken::PreGroup(_,_), PreToken::PreWord(l,_))
| (PreToken::PreWord(_,_), PreToken::PreGroup(l,_))
| (PreToken::PreNumber(_,_), PreToken::PreWord(l,_))
| (PreToken::PreWord(_,_), PreToken::PreNumber(l,_))
| (PreToken::PreWord(_,_), PreToken::PreWord(l,_))
=> {
g.push_back(a);
g.push_back(Token::PreOperator(
g.push_back(PreToken::PreOperator(
LineLocation{pos: l.pos-1, len: 0},
Operator::ImplicitMultiply
String::from("i*")
));
g.push_back(b);
},
// The following are syntax errors
(Token::Number(la, _), Token::Number(lb,_))
(PreToken::PreNumber(la,_), PreToken::PreNumber(lb,_))
=> {
return Err((
LineLocation{pos: la.pos, len: lb.pos - la.pos + lb.len},
@ -49,22 +43,32 @@ fn lookback(
}
// The following are fine
(Token::PreOperator(_,_), _) |
(_, Token::PreOperator(_,_))
(PreToken::PreOperator(_,_), _) |
(_, PreToken::PreOperator(_,_))
=> { g.push_back(a); g.push_back(b); },
// If we get this far, we found a Token
// that shouldn't be here.
_ => panic!()
// This shouldn't ever happen.
(PreToken::PreGroupStart(_), _)
| (_, PreToken::PreGroupStart(_))
| (PreToken::PreGroupEnd(_), _)
| (_, PreToken::PreGroupEnd(_))
| (PreToken::Container(_), _)
| (_, PreToken::Container(_))
=> panic!()
}
};
return Ok(());
}
pub fn p_groupify(mut g: VecDeque<Token>) -> Result<Token, (LineLocation, ParserError)> {
pub(in crate::parser) fn groupify(
mut g: VecDeque<PreToken>
) -> Result<
PreToken,
(LineLocation, ParserError)
> {
// Vector of grouping levels
let mut levels: Vec<(LineLocation, VecDeque<Token>)> = Vec::with_capacity(8);
let mut levels: Vec<(LineLocation, VecDeque<PreToken>)> = Vec::with_capacity(8);
levels.push((LineLocation{pos: 0, len: 0}, VecDeque::with_capacity(8)));
// Makes sure parenthesis are matched
@ -75,65 +79,45 @@ pub fn p_groupify(mut g: VecDeque<Token>) -> Result<Token, (LineLocation, Parser
let (l_now, v_now) = levels.last_mut().unwrap();
match t {
Token::PreOperator(_, _) => {
v_now.push_back(t);
lookback(v_now)?;
},
Token::PreNumber(l, s) => {
let n = match s.parse() {
Ok(n) => n,
Err(_) => return Err((l, ParserError::BadNumber))
};
v_now.push_back(Token::Number(l, n));
lookback(v_now)?;
},
Token::PreWord(l, s) => {
// This method must support both plain text and
// unicode versions of each word.
v_now.push_back(match &s[..] {
"mod" => { Token::PreOperator(l, Operator::ModuloLong) },
// Mathematical constants
"π"|"pi" => { Token::Constant(l, 3.141592653, String::from("pi")) },
"e" => { Token::Constant(l, 2.71828, String::from("e")) },
"phi"|"φ" => { Token::Constant(l, 1.61803, String::from("phi")) },
_ => { return Err((l, ParserError::Undefined(s))); }
});
lookback(v_now)?;
},
Token::PreGroupStart(l) => {
PreToken::PreGroupStart(l) => {
levels.push((l, VecDeque::with_capacity(8)));
i_level += 1;
},
Token::PreGroupEnd(l) => {
PreToken::PreGroupEnd(l) => {
let l = LineLocation {
pos: l_now.pos,
len: l.len + l.pos - l_now.pos
};
if i_level == 0 {
return Err((l, ParserError::ExtraCloseParen))
}
i_level -= 1;
if i_level == 0 { return Err((l, ParserError::ExtraCloseParen)) }
if v_now.len() == 0 { return Err((l, ParserError::EmptyGroup)) }
// Catch empty groups
if v_now.len() == 0 {
return Err((l, ParserError::EmptyGroup))
}
i_level -= 1;
let (_, v) = levels.pop().unwrap();
let (_, v_now) = levels.last_mut().unwrap();
v_now.push_back(Token::PreGroup(l, v));
v_now.push_back(PreToken::PreGroup(l, v));
lookback(v_now)?;
},
_ => panic!()
PreToken::PreWord(ref l, ref s) => {
let o = Operator::from_string(&s[..]);
if o.is_some() {
v_now.push_back(PreToken::PreOperator(*l, s.clone()));
} else {
v_now.push_back(t);
}
lookback(v_now)?;
}
_ => {
v_now.push_back(t);
lookback(v_now)?;
}
}
}
/*
@ -149,16 +133,13 @@ pub fn p_groupify(mut g: VecDeque<Token>) -> Result<Token, (LineLocation, Parser
let (l, v) = levels.pop().unwrap();
let (_, v_now) = levels.last_mut().unwrap();
// Catch empty groups
if v.len() == 0 {
return Err((l, ParserError::EmptyGroup))
}
if v.len() == 0 { return Err((l, ParserError::EmptyGroup)) }
v_now.push_back(Token::PreGroup(l, v));
v_now.push_back(PreToken::PreGroup(l, v));
lookback(v_now)?;
}
let (_, v) = levels.pop().unwrap();
return Ok(Token::PreGroup(LineLocation{pos:0, len:0}, v));
return Ok(PreToken::PreGroup(LineLocation{pos:0, len:0}, v));
}

110
src/parser/mod.rs
View File

@ -1,15 +1,97 @@
use std::collections::VecDeque;
mod tokenize;
mod treeify;
mod groupify;
mod find_subs;
use crate::parser::tokenize::p_tokenize;
use crate::parser::groupify::p_groupify;
use crate::parser::treeify::p_treeify;
use crate::parser::find_subs::p_find_subs;
use crate::parser::tokenize::tokenize;
use crate::parser::groupify::groupify;
use crate::parser::treeify::treeify;
use crate::parser::find_subs::find_subs;
use crate::tokens::LineLocation;
use crate::tokens::Token;
use crate::tokens::Operator;
#[derive(Debug)]
enum PreToken {
PreNumber(LineLocation, String),
PreWord(LineLocation, String),
PreOperator(LineLocation, String),
PreGroupStart(LineLocation),
PreGroupEnd(LineLocation),
PreGroup(LineLocation, VecDeque<PreToken>),
Container(Token)
}
impl PreToken {
#[inline(always)]
pub fn get_line_location(&self) -> &LineLocation {
match self {
PreToken::PreNumber(l, _)
| PreToken::PreWord(l, _)
| PreToken::PreOperator(l, _)
| PreToken::PreGroupStart(l)
| PreToken::PreGroupEnd(l)
| PreToken::PreGroup(l, _)
=> l,
_ => panic!()
}
}
#[inline(always)]
pub fn get_mut_line_location(&mut self) -> &mut LineLocation {
match self {
PreToken::PreNumber(l, _)
| PreToken::PreWord(l, _)
| PreToken::PreOperator(l, _)
| PreToken::PreGroupStart(l)
| PreToken::PreGroupEnd(l)
| PreToken::PreGroup(l, _)
=> l,
_ => panic!()
}
}
#[inline(always)]
pub fn to_token(self) -> Result<Token, (LineLocation, ParserError)>{
match self {
PreToken::PreNumber(l, s) => {
let n = match s.parse() {
Ok(n) => n,
Err(_) => return Err((l, ParserError::BadNumber))
};
return Ok(Token::Number(l, n));
},
PreToken::PreWord(l, s) => {
return Ok(match &s[..] {
// Mathematical constants
"π"|"pi" => { Token::Constant(l, 3.141592653, String::from("pi")) },
"e" => { Token::Constant(l, 2.71828, String::from("e")) },
"phi"|"φ" => { Token::Constant(l, 1.61803, String::from("phi")) },
_ => { return Err((l, ParserError::Undefined(s))); }
});
}
PreToken::Container(v) => { return Ok(v); }
PreToken::PreOperator(_,_)
| PreToken::PreGroupStart(_)
| PreToken::PreGroupEnd(_)
| PreToken::PreGroup(_, _)
=> panic!()
};
}
}
use crate::tokens;
/// Types of parser errors.
/// If we cannot parse a string, one of these is returned.
@ -53,16 +135,16 @@ impl ParserError {
pub fn parse(
s: &String
) -> Result<
tokens::Token,
(tokens::LineLocation, ParserError)
Token,
(LineLocation, ParserError)
> {
let tokens = p_tokenize(s);
let (_, tokens) = p_find_subs(tokens);
let mut g = p_groupify(tokens)?;
g = p_treeify(g)?;
let tokens = tokenize(s);
let (_, tokens) = find_subs(tokens);
let g = groupify(tokens)?;
let t = treeify(g)?;
return Ok(g);
return Ok(t);
}
@ -70,8 +152,8 @@ pub fn substitute(s: &String) -> String{
if s == "" { return s.clone() }
let mut new_s = s.clone();
let tokens = p_tokenize(s);
let (subs, _) = p_find_subs(tokens);
let tokens = tokenize(s);
let (subs, _) = find_subs(tokens);
for r in subs.iter() {
new_s.replace_range(

72
src/parser/tokenize.rs
View File

@ -1,19 +1,18 @@
use std::collections::VecDeque;
use crate::tokens::Token;
use crate::tokens::Operator;
use crate::parser::PreToken;
use crate::tokens::LineLocation;
/// Updates the length of a Token's LineLocation.
/// Run whenever a token is finished.
#[inline(always)]
fn update_line_location(mut t: Token, stop_i: usize) -> Token {
fn update_line_location(mut t: PreToken, stop_i: usize) -> PreToken {
match t {
Token::PreGroupStart(ref mut l) |
Token::PreGroupEnd(ref mut l) |
Token::PreOperator(ref mut l, _) |
Token::PreNumber(ref mut l, _) |
Token::PreWord(ref mut l, _)
PreToken::PreGroupStart(ref mut l) |
PreToken::PreGroupEnd(ref mut l) |
PreToken::PreOperator(ref mut l, _) |
PreToken::PreNumber(ref mut l, _) |
PreToken::PreWord(ref mut l, _)
=> {
*l = LineLocation{
pos: l.pos,
@ -27,9 +26,9 @@ fn update_line_location(mut t: Token, stop_i: usize) -> Token {
}
/// Turns a string into Tokens. First stage of parsing.
pub fn p_tokenize(input: &String) -> VecDeque<Token> {
let mut t: Option<Token> = None; // The current token we're reading
let mut g: VecDeque<Token> = VecDeque::with_capacity(32);
pub(in crate::parser) fn tokenize(input: &String) -> VecDeque<PreToken> {
let mut t: Option<PreToken> = None; // The current token we're reading
let mut g: VecDeque<PreToken> = VecDeque::with_capacity(32);
for (i, c) in input.chars().enumerate() {
@ -41,32 +40,32 @@ pub fn p_tokenize(input: &String) -> VecDeque<Token> {
match g.back().as_ref() {
// If previous token was any of the following,
// this is the "minus" operator
Some(Token::PreNumber(_, _)) |
Some(Token::PreGroupEnd(_)) |
Some(Token::PreWord(_, _)) => {
t = Some(Token::PreOperator(
Some(PreToken::PreNumber(_, _)) |
Some(PreToken::PreGroupEnd(_)) |
Some(PreToken::PreWord(_, _)) => {
t = Some(PreToken::PreOperator(
LineLocation{pos: i, len: 1},
Operator::Subtract
String::from("-")
));
},
// Otherwise, this is a negative sign.
_ => {
t = Some(Token::PreOperator(
t = Some(PreToken::PreOperator(
LineLocation{pos: i, len: 1},
Operator::Negative
String::from("neg")
));
}
};
},
// Number.
// Number
// Commas act just like dots.
',' | '.' | '0'..='9' => {
match &mut t {
// If we're already building a number,
// append.
Some(Token::PreNumber(_, val)) => {
Some(PreToken::PreNumber(_, val)) => {
val.push(if c == ',' {'.'} else {c});
},
@ -74,39 +73,32 @@ pub fn p_tokenize(input: &String) -> VecDeque<Token> {
// previous token and start one.
_ => {
if t.is_some() { g.push_back(update_line_location(t.unwrap(), i)); }
t = Some(Token::PreNumber(LineLocation{pos: i, len: 0}, String::from(c)));
t = Some(PreToken::PreNumber(LineLocation{pos: i, len: 0}, String::from(c)));
}
};
},
// Operator
// Always one character
'*'|'×'|'/'|'÷'|
'+'|'^'|'!'|'%'
=> {
if t.is_some() { g.push_back(update_line_location(t.unwrap(), i)); }
t = Some(Token::PreOperator(
LineLocation{pos: i, len: 0},
match c {
'^' => Operator::Power,
'%' => Operator::Modulo,
'*'|'×' => Operator::Multiply,
'/'|'÷' => Operator::Divide,
'+' => Operator::Add,
'!' => Operator::Factorial,
_ => panic!()
match &mut t {
Some(PreToken::PreOperator(_, val)) => { val.push(c); },
_ => {
if t.is_some() { g.push_back(update_line_location(t.unwrap(), i)); }
t = Some(PreToken::PreOperator(LineLocation{pos: i, len: 0}, String::from(c)));
}
));
}
};
},
// Group
'(' => {
if t.is_some() { g.push_back(update_line_location(t.unwrap(), i)); }
t = Some(Token::PreGroupStart(LineLocation{pos: i, len: 0}));
t = Some(PreToken::PreGroupStart(LineLocation{pos: i, len: 0}));
},
')' => {
if t.is_some() { g.push_back(update_line_location(t.unwrap(), i)); }
t = Some(Token::PreGroupEnd(LineLocation{pos: i, len: 0}));
t = Some(PreToken::PreGroupEnd(LineLocation{pos: i, len: 0}));
},
// Space. Basic seperator.
@ -120,13 +112,11 @@ pub fn p_tokenize(input: &String) -> VecDeque<Token> {
// Word
_ => {
match &mut t {
Some(Token::PreWord(_, val)) => {
val.push(c);
},
Some(PreToken::PreWord(_, val)) => { val.push(c); },
_ => {
if t.is_some() { g.push_back(update_line_location(t.unwrap(), i)); }
t = Some(Token::PreWord(LineLocation{pos: i, len: 0}, String::from(c)));
t = Some(PreToken::PreWord(LineLocation{pos: i, len: 0}, String::from(c)));
}
};
}

182
src/parser/treeify.rs
View File

@ -1,35 +1,40 @@
use std::collections::VecDeque;
use crate::parser::PreToken;
use crate::parser::ParserError;
use crate::tokens::Token;
use crate::tokens::Operator;
use crate::tokens::LineLocation;
use crate::parser::ParserError;
fn treeify_binary(
i: usize,
g_inner: &mut VecDeque<Token>,
g_inner: &mut VecDeque<PreToken>,
left_associative: bool
) -> Result<(), (LineLocation, ParserError)> {
let this: &Token = &g_inner[i];
let this: &PreToken = &g_inner[i];
if i == 0 {
// This binary operator is at the end of an expression.
let l = match this {
Token::PreOperator(l, _) => l,
PreToken::PreOperator(l, _) => l,
_ => panic!()
};
return Err((*l, ParserError::Syntax));
}
let next: &Token;
let next: &PreToken;
if left_associative {
next = {
if i < g_inner.len()-1 {
&g_inner[i+1]
} else {
let l = match this {
Token::PreOperator(l, _) => l,
PreToken::PreOperator(l, _) => l,
_ => panic!()
};
return Err((*l, ParserError::Syntax));
@ -41,7 +46,7 @@ fn treeify_binary(
&g_inner[i-1]
} else {
let l = match this {
Token::PreOperator(l, _) => l,
PreToken::PreOperator(l, _) => l,
_ => panic!()
};
return Err((*l, ParserError::Syntax));
@ -51,7 +56,11 @@ fn treeify_binary(
if let Token::PreOperator(l, o) = next {
if let PreToken::PreOperator(l, s) = next {
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
let o = o.unwrap();
if {
(!o.is_binary()) &&
!(o.is_left_associative() && left_associative)
@ -66,25 +75,27 @@ fn treeify_binary(
));
}
} else {
// Precedence of this operator
let this_val: isize = match this {
Token::PreOperator(_, q) => *q as isize,
_ => panic!()
let this_val = {
let PreToken::PreOperator(l, s) = this else {panic!()};
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
o.unwrap() as isize
};
// Precedence of the operators contesting our arguments
let left_val = if i > 1 {
match &g_inner[i-2] {
Token::PreOperator(_, q) => Some(*q as isize),
_ => panic!()
}
let PreToken::PreOperator(l, s) = &g_inner[i-2] else {panic!()};
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
Some(o.unwrap() as isize)
} else { None };
let right_val = if i < g_inner.len()-2 {
match &g_inner[i+2] {
Token::PreOperator(_, q) => Some(*q as isize),
_ => panic!()
}
let PreToken::PreOperator(l, s) = &g_inner[i+2] else {panic!()};
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
Some(o.unwrap() as isize)
} else { None };
if {
@ -92,22 +103,25 @@ fn treeify_binary(
(right_val.is_none() || this_val >= right_val.unwrap())
} {
// This operator has higher precedence, it takes both arguments
let mut left = g_inner.remove(i-1).unwrap();
let this = g_inner.remove(i-1).unwrap();
let mut right = g_inner.remove(i-1).unwrap();
if let Token::PreGroup(_, _) = right { right = p_treeify(right)?; }
if let Token::PreGroup(_, _) = left { left = p_treeify(left)?; }
let left_pre = g_inner.remove(i-1).unwrap();
let this_pre = g_inner.remove(i-1).unwrap();
let right_pre = g_inner.remove(i-1).unwrap();
let left: Token; let right: Token;
if let PreToken::PreGroup(_, _) = right_pre { right = treeify(right_pre)?; } else {right = right_pre.to_token()?;}
if let PreToken::PreGroup(_, _) = left_pre { left = treeify(left_pre)?; } else {left = left_pre.to_token()?;}
let k = match this {
Token::PreOperator(_, k) => k,
_ => panic!()
let (l, o) = {
let PreToken::PreOperator(l, s) = this_pre else {panic!()};
let o = Operator::from_string(&s);
if o.is_none() { panic!() }
(l, o.unwrap())
};
let mut new_token_args: VecDeque<Token> = VecDeque::with_capacity(3);
let mut new_token_args: VecDeque<Token> = VecDeque::with_capacity(2);
new_token_args.push_back(left);
new_token_args.push_back(right);
g_inner.insert(i-1, k.into_token(new_token_args));
g_inner.insert(i-1, PreToken::Container(o.into_token(l, new_token_args)));
return Ok(());
} else {
@ -116,22 +130,21 @@ fn treeify_binary(
};
}
fn treeify_unary(
i: usize,
g_inner: &mut VecDeque<Token>,
g_inner: &mut VecDeque<PreToken>,
left_associative: bool
) -> Result<(), (LineLocation, ParserError)> {
let this: &Token = &g_inner[i];
let next: &Token;
let this: &PreToken = &g_inner[i];
let next: &PreToken;
if left_associative {
next = {
if i > 0 {
&g_inner[i-1]
} else {
let l = match this {
Token::PreOperator(l, _) => l,
PreToken::PreOperator(l, _) => l,
_ => panic!()
};
return Err((*l, ParserError::Syntax));
@ -143,7 +156,7 @@ fn treeify_unary(
&g_inner[i+1]
} else {
let l = match this {
Token::PreOperator(l, _) => l,
PreToken::PreOperator(l, _) => l,
_ => panic!()
};
return Err((*l, ParserError::Syntax));
@ -153,17 +166,19 @@ fn treeify_unary(
// We need to check the element after unary operators too.
// Bad syntax like `3!3` won't be caught otherwise.
let prev: Option<&Token>;
let prev: Option<&PreToken>;
if left_associative {
prev = if i < g_inner.len()-1 { Some(&g_inner[i+1]) } else {None};
} else {
prev = if i > 0 { Some(&g_inner[i-1]) } else {None};
}
if prev.is_some() {
if let Token::PreOperator(l, o) = prev.unwrap() {
if let PreToken::PreOperator(l, s) = prev.unwrap() {
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
let o = o.unwrap();
if o.is_left_associative() && left_associative {
return Err((*l, ParserError::Syntax));
}
@ -177,59 +192,64 @@ fn treeify_unary(
if let Token::PreOperator(l, _) = next {
if let PreToken::PreOperator(l, _) = next {
let tl = *this.get_line_location();
return Err((
LineLocation{pos: tl.pos, len: l.pos - tl.pos + l.len},
ParserError::Syntax
));
} else {
// Precedence of this operator
let this_val: isize = match this {
Token::PreOperator(_, q) => *q as isize,
_ => panic!()
let this_val = {
let PreToken::PreOperator(l, s) = this else {panic!()};
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
o.unwrap() as isize
};
// Precedence of the operator contesting its argument
let next_val = if left_associative {
if i > 1 {
match &g_inner[i-2] {
Token::PreOperator(_, q) => Some(*q as isize),
_ => panic!()
}
let PreToken::PreOperator(l, s) = &g_inner[i-2] else {panic!()};
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
Some(o.unwrap() as isize)
} else { None }
} else {
if i < g_inner.len()-2 {
match &g_inner[i+2] {
Token::PreOperator(_, q) => Some(*q as isize),
_ => panic!()
}
let PreToken::PreOperator(l, s) = &g_inner[i+2] else {panic!()};
let o = Operator::from_string(s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
Some(o.unwrap() as isize)
} else { None }
};
if next_val.is_none() || this_val > next_val.unwrap() {
let this = g_inner.remove(i).unwrap();
let mut next;
let this_pre = g_inner.remove(i).unwrap();
let next_pre: PreToken; let next: Token;
if left_associative {
next = g_inner.remove(i-1).unwrap();
next_pre = g_inner.remove(i-1).unwrap();
} else {
next = g_inner.remove(i).unwrap();
next_pre = g_inner.remove(i).unwrap();
}
if let Token::PreGroup(_, _) = next { next = p_treeify(next)?; }
if let PreToken::PreGroup(_, _) = next_pre { next = treeify(next_pre)?; } else { next = next_pre.to_token()? }
let k = match this {
Token::PreOperator(_, k) => k,
_ => panic!()
let (l, o) = {
let PreToken::PreOperator(l, s) = this_pre else {panic!()};
let o = Operator::from_string(&s);
if o.is_none() { panic!() }
(l, o.unwrap())
};
let mut new_token_args: VecDeque<Token> = VecDeque::with_capacity(3);
new_token_args.push_back(next);
if left_associative {
g_inner.insert(i-1, k.into_token(new_token_args));
g_inner.insert(i-1, PreToken::Container(o.into_token(l, new_token_args)));
} else {
g_inner.insert(i, k.into_token(new_token_args));
g_inner.insert(i, PreToken::Container(o.into_token(l, new_token_args)));
}
return Ok(());
@ -241,12 +261,13 @@ fn treeify_unary(
};
}
pub fn p_treeify(
mut g: Token,
pub(in crate::parser) fn treeify(
mut g: PreToken,
) -> Result<Token, (LineLocation, ParserError)> {
let g_inner: &mut VecDeque<Token> = match g {
Token::PreGroup(_, ref mut x) => x,
let g_inner: &mut VecDeque<PreToken> = match g {
PreToken::PreGroup(_, ref mut x) => x,
_ => panic!()
};
@ -265,7 +286,11 @@ pub fn p_treeify(
let i = j as usize;
let this_op = match &g_inner[i] {
Token::PreOperator(_, o) => o,
PreToken::PreOperator(l, s) => {
let o = Operator::from_string(&s);
if o.is_none() { return Err((*l, ParserError::Syntax)); }
o.unwrap()
},
_ => {
if left_associative { j += 1 } else { j -= 1 };
continue;
@ -293,18 +318,17 @@ pub fn p_treeify(
}
}
g = g_inner.pop_front().unwrap();
// Catch edge cases
match g {
Token::PreOperator(l, _) => {
return Err((l, ParserError::Syntax));
let g = g_inner.pop_front().unwrap();
return match g {
// Catch edge cases
PreToken::PreOperator(l, _) => {
Err((l, ParserError::Syntax))
},
Token::PreGroup(_,_) => {
g = p_treeify(g)?;
}
_ => {}
PreToken::PreGroup(_,_) => {
treeify(g)
},
_ => { Ok(g.to_token()?) }
};
return Ok(g);
}
}

211
src/tokens.rs
View File

@ -16,49 +16,22 @@ pub struct LineLocation {
/// will never show up in a fully-parsed expression tree.
#[derive(Debug)]
pub enum Token {
/// Used only while tokenizing.
/// Will be replaced with a Number once we finish.
PreNumber(LineLocation, String),
/// Used only while tokenizing.
/// Will be replaced with one of the Tokens below once we finish.
PreWord(LineLocation, String),
/// Used only until operators are parsed.
/// Each of these will become one of the operators below.
PreOperator(LineLocation, Operator),
PreGroupStart(LineLocation),
PreGroupEnd(LineLocation),
/// Used only until operators are parsed.
/// PreGroups aren't needed once we have a tree.
PreGroup(LineLocation, VecDeque<Token>),
Number(LineLocation, f64),
Constant(LineLocation, f64, String),
Multiply(VecDeque<Token>),
Divide(VecDeque<Token>),
Add(VecDeque<Token>),
Factorial(VecDeque<Token>),
Negative(VecDeque<Token>),
Power(VecDeque<Token>),
Modulo(VecDeque<Token>),
Operator(
LineLocation,
Operator,
VecDeque<Token>
),
}
impl Token {
#[inline(always)]
pub fn get_args(&mut self) -> Option<&mut VecDeque<Token>> {
match self {
Token::Multiply(ref mut v)
| Token::Divide(ref mut v)
| Token::Add(ref mut v)
| Token::Factorial(ref mut v)
| Token::Negative(ref mut v)
| Token::Power(ref mut v)
| Token::Modulo(ref mut v)
=> Some(v),
Token::Operator(_, _, ref mut a)
=> Some(a),
_ => None
}
}
@ -66,38 +39,10 @@ impl Token {
#[inline(always)]
pub fn get_line_location(&self) -> &LineLocation {
match self {
Token::PreNumber(l, _) |
Token::PreWord(l, _) |
Token::PreOperator(l, _) |
Token::PreGroupStart(l) |
Token::PreGroupEnd(l) |
Token::PreGroup(l, _)
Token::Number(l, _)
| Token::Operator(l, _, _)
| Token::Constant(l, _, _)
=> l,
// These have a line location, but we shouldn't ever need to get it.
Token::Number(_l, _) |
Token::Constant(_l, _, _)
=> panic!(),
_ => panic!()
}
}
#[inline(always)]
pub fn get_mut_line_location(&mut self) -> &mut LineLocation {
match self {
Token::PreNumber(l, _) |
Token::PreWord(l, _) |
Token::PreOperator(l, _) |
Token::PreGroupStart(l) |
Token::PreGroupEnd(l) |
Token::PreGroup(l, _)
=> l,
// These have a line location, but we shouldn't ever need to get it.
Token::Number(_l, _) |
Token::Constant(_l, _, _)
=> panic!(),
_ => panic!()
}
}
@ -115,8 +60,15 @@ impl Token {
}
pub fn eval(&self) -> Token {
match self {
Token::Negative(ref v) => {
let (o, v) = match self {
Token::Operator(_, o, v) => (o, v),
Token::Number(l, v) => { return Token::Number(*l, *v); },
_ => panic!()
};
match o {
Operator::Negative => {
if v.len() != 1 {panic!()};
let v = v[0].as_number();
@ -125,7 +77,7 @@ impl Token {
} else { panic!(); }
},
Token::Add(ref v) => {
Operator::Add => {
let mut sum: f64 = 0f64;
let mut new_pos: usize = 0;
let mut new_len: usize = 0;
@ -146,7 +98,8 @@ impl Token {
)
},
Token::Multiply(ref v) => {
Operator::ImplicitMultiply |
Operator::Multiply => {
let mut prod: f64 = 1f64;
let mut new_pos: usize = 0;
let mut new_len: usize = 0;
@ -167,7 +120,7 @@ impl Token {
)
},
Token::Divide(ref v) => {
Operator::Divide => {
if v.len() != 2 {panic!()};
let a = v[0].as_number();
let b = v[1].as_number();
@ -182,7 +135,8 @@ impl Token {
} else { panic!(); }
},
Token::Modulo(ref v) => {
Operator::ModuloLong |
Operator::Modulo => {
if v.len() != 2 {panic!()};
let a = v[0].as_number();
let b = v[1].as_number();
@ -197,7 +151,7 @@ impl Token {
} else { panic!(); }
},
Token::Power(ref v) => {
Operator::Power => {
if v.len() != 2 {panic!()};
let a = v[0].as_number();
let b = v[1].as_number();
@ -212,15 +166,13 @@ impl Token {
} else { panic!(); }
},
Token::Factorial(ref _v) => { todo!() },
_ => self.as_number()
Operator::Factorial => { todo!() },
Operator::Subtract => { panic!() }
}
}
}
/// Operator types, in order of increasing priority.
/// The Null operator MUST be equal to zero.
#[derive(Debug)]
#[derive(Copy, Clone)]
pub enum Operator {
@ -240,6 +192,62 @@ pub enum Operator {
}
impl Operator {
#[inline(always)]
pub fn into_token(self, l: LineLocation, mut args: VecDeque<Token>) -> Token {
match self {
Operator::Subtract => {
if args.len() != 2 { panic!() }
let a = args.pop_front().unwrap();
let b = args.pop_front().unwrap();
let b = Token::Operator(l, Operator::Negative, VecDeque::from(vec!(b)));
Token::Operator(
l, Operator::Add,
VecDeque::from(vec!(a,b))
)
},
Operator::Factorial
| Operator::Negative
=> {
if args.len() != 1 { panic!() }
Token::Operator(l, self, args)
},
Operator::Modulo
| Operator::ModuloLong
| Operator::Divide
| Operator::Power
=> {
if args.len() != 2 { panic!() }
Token::Operator(l, self, args)
},
Operator::Add
| Operator::Multiply
| Operator::ImplicitMultiply
=> Token::Operator(l, self, args)
}
}
#[inline(always)]
pub fn from_string(s: &str) -> Option<Operator> {
match s {
"+" => {Some( Operator::Add )},
"-" => {Some( Operator::Subtract )},
"neg" => {Some( Operator::Negative )},
"*"|"×" => {Some( Operator::Multiply )},
"/"|"÷" => {Some( Operator::Divide )},
"i*" => {Some( Operator::ImplicitMultiply )},
"%" => {Some( Operator::Modulo )},
"mod" => {Some( Operator::ModuloLong )},
"^" => {Some( Operator::Power )},
"!" => {Some( Operator::Factorial )},
_ => None
}
}
#[inline(always)]
pub fn is_binary(&self) -> bool {
match self {
@ -258,53 +266,4 @@ impl Operator {
_ => true
}
}
#[inline(always)]
pub fn into_token(&self, mut args: VecDeque<Token>) -> Token {
match self {
Operator::Add => Token::Add(args),
Operator::Multiply
| Operator::ImplicitMultiply
=> Token::Multiply(args),
Operator::Subtract => {
if args.len() != 2 { panic!() }
let a = args.pop_front().unwrap();
let b = args.pop_front().unwrap();
Token::Add(
VecDeque::from(vec!(
a,
Token::Negative(VecDeque::from(vec!(b)))
)))
},
Operator::Divide => {
if args.len() != 2 { panic!() }
Token::Divide(args)
},
Operator::ModuloLong |
Operator::Modulo => {
if args.len() != 2 { panic!() }
Token::Modulo(args)
},
Operator::Power => {
if args.len() != 2 { panic!() }
Token::Power(args)
},
Operator::Negative => {
if args.len() != 1 { panic!() }
Token::Negative(args)
},
Operator::Factorial => {
if args.len() != 1 { panic!() }
Token::Factorial(args)
}
}
}
}
}