Mark
/
daisy
mirror of https://github.com/rm-dr/daisy
1
1
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Added lookback parsing

pull/2/head
Mark 2023-03-22 15:06:58 -07:00
parent af85e8a6d5
commit 9f53847ed3
Signed by: Mark
GPG Key ID: AD62BB059C2AAEE4
3 changed files with 84 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/parser.rs
View File

@ -30,14 +30,16 @@ pub enum Token {
/// PreGroups aren't needed once we have a tree. /// PreGroups aren't needed once we have a tree.
PreGroup(LineLocation, VecDeque<Token>), PreGroup(LineLocation, VecDeque<Token>),
PreNegative(LineLocation),
PreFactorial(LineLocation),
Number(LineLocation, f64), Number(LineLocation, f64),
Multiply(VecDeque<Token>), Multiply(VecDeque<Token>),
Divide(VecDeque<Token>), Divide(VecDeque<Token>),
Add(VecDeque<Token>), Add(VecDeque<Token>),
Subtract(VecDeque<Token>), Subtract(VecDeque<Token>),
Factorial(VecDeque<Token>), //Factorial(VecDeque<Token>),
Negative(VecDeque<Token>), //Negative(VecDeque<Token>),
Power(VecDeque<Token>), Power(VecDeque<Token>),
Modulo(VecDeque<Token>), Modulo(VecDeque<Token>),
} }
@ -56,9 +58,7 @@ pub enum Operators {
Multiply, Multiply,
ImplicitMultiply, ImplicitMultiply,
Modulo, // Mod invoked with % Modulo, // Mod invoked with %
Power, Power
Negative,
Factorial,
} }
/// Specifies the location of a token in an input string. /// Specifies the location of a token in an input string.
@ -77,7 +77,6 @@ pub enum ParserError {
InvalidChar, InvalidChar,
MissingCloseParen, MissingCloseParen,
Syntax, Syntax,
InvalidImplicitMultiply,
BadNumber BadNumber
} }

127
src/parser/tokenize.rs
View File

@ -13,7 +13,10 @@ fn update_line_location(mut t: Token, stop_i: usize) -> Token {
Token::PreGroup(ref mut l, _) | Token::PreGroup(ref mut l, _) |
Token::PreOperator(ref mut l, _) | Token::PreOperator(ref mut l, _) |
Token::PreNumber(ref mut l, _) | Token::PreNumber(ref mut l, _) |
Token::PreWord(ref mut l, _) => { Token::PreWord(ref mut l, _) |
Token::PreNegative(ref mut l) |
Token::PreFactorial(ref mut l)
=> {
let LineLocation{pos, .. } = l; let LineLocation{pos, .. } = l;
*l = LineLocation{ *l = LineLocation{
pos: *pos, pos: *pos,
@ -27,30 +30,33 @@ fn update_line_location(mut t: Token, stop_i: usize) -> Token {
} }
/// Look at the last two elements of `g`: /// Looks backwards at the elements of g.
/// - if one is an operator, do nothing. /// - Inserts ImplicitMultiply
/// - if they are a valid implicit multiplication pair, add an ImplicitMultiply between them /// - Removes multiple PreNegatives
/// - if they aren't, throw an error. /// - Applies PreNegative to Numbers
/// - Parses factorials
/// - Checks syntax
#[inline(always)] #[inline(always)]
fn insert_implicit( fn lookback(
g: &mut VecDeque<Token> g: &mut VecDeque<Token>
) -> Result<(), (LineLocation, ParserError)> { ) -> Result<(), (LineLocation, ParserError)> {
if g.len() >= 2 { if g.len() >= 2 {
let b: Token = g.pop_back().unwrap(); let b: Token = g.pop_back().unwrap();
let a: &Token = g.back().unwrap(); let a: Token = g.pop_back().unwrap();
match (a, &b) { match (&a, &b) {
// Not implicit multiplication, ignore ( // Delete consecutive negatives
(Token::PreOperator(_,_), _) | Token::PreNegative(_),
(_, Token::PreOperator(_,_)) Token::PreNegative(_)
=> { g.push_back(b); }, ) => {},
// Valid implicit multiplications // Insert ImplicitMultiply
(Token::PreGroup(_,_), Token::PreGroup(ref l,_)) | (Token::PreGroup(_,_), Token::PreGroup(l ,_)) |
(Token::PreGroup(_,_), Token::Number(ref l,_)) | (Token::PreGroup(_,_), Token::Number(l,_)) |
(Token::Number(_,_), Token::PreGroup(ref l,_)) (Token::Number(_,_), Token::PreGroup(l,_))
=> { => {
g.push_back(a);
let LineLocation { pos: i, .. } = l; let LineLocation { pos: i, .. } = l;
g.push_back(Token::PreOperator( g.push_back(Token::PreOperator(
LineLocation{pos: i-1, len: 0}, LineLocation{pos: i-1, len: 0},
@ -59,16 +65,35 @@ fn insert_implicit(
g.push_back(b); g.push_back(b);
}, },
// Invalid implicit multiplications // The following are syntax errors
(Token::Number(_,_), Token::Number(l,_)) (Token::PreOperator(la,_), Token::PreOperator(lb,_)) |
(Token::Number(la, _), Token::Number(lb,_)) |
(Token::PreNegative(la), Token::PreOperator(lb,_)) |
(Token::PreOperator(la, _), Token::PreFactorial(lb)) |
(Token::PreNegative(la), Token::PreFactorial(lb))
=> { => {
let LineLocation { pos: i, .. } = l; let LineLocation { pos: posa, .. } = *la;
let LineLocation { pos: posb, len: lenb } = *lb;
return Err(( return Err((
LineLocation{pos: i-1, len: 2}, LineLocation{pos: posa, len: posb - posa + lenb},
ParserError::InvalidImplicitMultiply ParserError::Syntax
)); ));
}, }
// The following are fine
(Token::PreOperator(_,_), Token::PreNegative(_)) |
(Token::PreOperator(_,_), Token::Number(_,_)) |
(Token::Number(_,_), Token::PreOperator(_,_)) |
(Token::PreOperator(_,_), Token::PreGroup(_,_)) |
(Token::PreGroup(_,_), Token::PreOperator(_,_)) |
(Token::PreNegative(_), Token::PreGroup(_,_)) |
(Token::PreNegative(_), Token::Number(_,_)) |
(Token::PreGroup(_,_), Token::PreFactorial(_)) |
(Token::Number(_,_), Token::PreFactorial(_))
=> { g.push_back(a); g.push_back(b); },
// If we get this far, we found a Token
// that shouldn't be here.
_ => panic!() _ => panic!()
} }
}; };
@ -105,9 +130,13 @@ fn push_token(
} }
}, },
Token::PreOperator(_, _) => t, Token::PreOperator(_, _) => t,
Token::PreGroup(_, _) => t,
Token::PreNegative(_) => t,
Token::PreFactorial(_) => t,
_ => panic!() _ => panic!()
}); });
insert_implicit(g_now)?;
lookback(g_now)?;
} }
return Ok(()); return Ok(());
} }
@ -130,41 +159,29 @@ pub fn tokenize(input: &String) -> Result<Token, (LineLocation, ParserError)> {
match c { match c {
'!' => { '!' => {
if t.is_some() { g_now.push_back(update_line_location(t.unwrap(), i)); t = None; } push_token(g_now, i, t)?;
g_now.push_back( t = Some(Token::PreFactorial(LineLocation{pos: i, len: 1}));
Token::PreOperator(
LineLocation{pos: i, len: 1},
Operators::Factorial
)
);
}, },
// The minus sign can be both a Negative and an Operator. // The minus sign can be both a Negative and an Operator.
// Needs special treatment. // Needs special treatment.
'-' => { '-' => {
push_token(g_now, i, t)?; t = None; push_token(g_now, i, t)?;
match g_now.back() { match g_now.back() {
// If previous token was any of the following, // If previous token was any of the following,
// this is the "minus" operator // this is the "minus" operator
Some(Token::PreNumber(_, _)) | Some(Token::Number(_, _)) |
Some(Token::PreGroup(_, _)) | Some(Token::PreGroup(_, _)) |
Some(Token::PreWord(_, _)) => { Some(Token::PreWord(_, _)) => {
g_now.push_back( t = Some(Token::PreOperator(
Token::PreOperator(
LineLocation{pos: i, len: 1}, LineLocation{pos: i, len: 1},
Operators::Subtract Operators::Subtract
) ));
);
}, },
// Otherwise, this is a negative sign. // Otherwise, this is a negative sign.
_ => { _ => {
g_now.push_back( t = Some(Token::PreNegative(LineLocation{pos: i, len: 1}));
Token::PreOperator(
LineLocation{pos: i, len: 1},
Operators::Negative
)
);
} }
}; };
}, },
@ -192,16 +209,12 @@ pub fn tokenize(input: &String) -> Result<Token, (LineLocation, ParserError)> {
'A'..='Z' | 'A'..='Z' |
'a'..='z' => { 'a'..='z' => {
match &mut t { match &mut t {
// If we're already building a number,
// append.
Some(Token::PreWord(_, val)) => { Some(Token::PreWord(_, val)) => {
val.push(c); val.push(c);
}, },
// If we're not building a number, finalize
// previous token and start one.
_ => { _ => {
if t.is_some() { g_now.push_back(update_line_location(t.unwrap(), i)); } push_token(g_now, i, t)?;
t = Some(Token::PreWord(LineLocation{pos: i, len: 0}, String::from(c))); t = Some(Token::PreWord(LineLocation{pos: i, len: 0}, String::from(c)));
} }
}; };
@ -210,10 +223,8 @@ pub fn tokenize(input: &String) -> Result<Token, (LineLocation, ParserError)> {
// Operator // Operator
// Always one character // Always one character
'+' | '*' | '/' | '^' | '%' => { '+' | '*' | '/' | '^' | '%' => {
// Finalize previous token push_token(g_now, i, t)?;
push_token(g_now, i, t)?; t = None; t = Some(Token::PreOperator(
g_now.push_back(
Token::PreOperator(
LineLocation{pos: i, len: 1}, LineLocation{pos: i, len: 1},
match c { match c {
'^' => Operators::Power, '^' => Operators::Power,
@ -223,8 +234,7 @@ pub fn tokenize(input: &String) -> Result<Token, (LineLocation, ParserError)> {
'+' => Operators::Add, '+' => Operators::Add,
_ => panic!() _ => panic!()
} }
) ));
);
} }
// Group // Group
@ -233,15 +243,8 @@ pub fn tokenize(input: &String) -> Result<Token, (LineLocation, ParserError)> {
g.push(Token::PreGroup(LineLocation{pos: i, len: 0}, VecDeque::with_capacity(8))); g.push(Token::PreGroup(LineLocation{pos: i, len: 0}, VecDeque::with_capacity(8)));
}, },
')' => { ')' => {
push_token(g_now, i, t)?; t = None; push_token(g_now, i, t)?;
let new_group: Token = g.pop().unwrap(); t = Some(g.pop().unwrap());
let g_now: &mut VecDeque<Token> = match g.last_mut().unwrap() {
Token::PreGroup(_, ref mut x) => x,
_ => panic!()
};
g_now.push_back(update_line_location(new_group, i+1));
}, },
// Space. Basic seperator. // Space. Basic seperator.

9
src/parser/treeify.rs
View File

@ -14,7 +14,6 @@ pub fn treeify(
Token::PreGroup(_, ref mut x) => x, Token::PreGroup(_, ref mut x) => x,
_ => panic!() _ => panic!()
}; };
let mut new: VecDeque<Token> = VecDeque::with_capacity(8);
let mut i = 1; let mut i = 1;
while g_inner.len() > 1 { while g_inner.len() > 1 {
@ -53,8 +52,8 @@ pub fn treeify(
let this = g_inner.remove(i-1).unwrap(); let this = g_inner.remove(i-1).unwrap();
let right = g_inner.remove(i-1).unwrap(); let right = g_inner.remove(i-1).unwrap();
let (l, k) = match this { let k = match this {
Token::PreOperator(l, k) => (l, k), Token::PreOperator(_, k) => k,
_ => panic!() _ => panic!()
}; };
@ -71,10 +70,8 @@ pub fn treeify(
Operators::Multiply => Token::Multiply(new_token_args), Operators::Multiply => Token::Multiply(new_token_args),
Operators::ImplicitMultiply => Token::Multiply(new_token_args), Operators::ImplicitMultiply => Token::Multiply(new_token_args),
Operators::Modulo => Token::Modulo(new_token_args), Operators::Modulo => Token::Modulo(new_token_args),
Operators::Power => Token::Power(new_token_args),
Operators::ModuloLong => Token::Modulo(new_token_args), Operators::ModuloLong => Token::Modulo(new_token_args),
Operators::Negative => panic!(), Operators::Power => Token::Power(new_token_args),
Operators::Factorial => panic!(),
Operators::Null => panic!() Operators::Null => panic!()
} }
); );