From 7eef21b3b90898f4ef05fa4220fde608cf55c6ab Mon Sep 17 00:00:00 2001 From: Graham Northup Date: Sun, 24 Sep 2017 03:41:45 -0400 Subject: parser appears to work (oh my) --- src/lang/tokenizer.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) (limited to 'src/lang/tokenizer.rs') diff --git a/src/lang/tokenizer.rs b/src/lang/tokenizer.rs index d1b34e0..4f2858c 100644 --- a/src/lang/tokenizer.rs +++ b/src/lang/tokenizer.rs @@ -173,13 +173,20 @@ impl> Tokenizer { } } - fn next_token(&mut self) -> Result { + pub fn next_token(&mut self) -> Result { + let res = self._next_token(); + eprintln!("next_token: {:?}", res); + res + } + + fn _next_token(&mut self) -> Result { let mut c = self.next_char(); if c == None { return Ok(Token::EOF); } let mut cc = c.unwrap(); + /* Whitespace */ while cc.is_whitespace() { c = self.next_char(); if c == None { @@ -315,10 +322,16 @@ impl> Tokenizer { radix = 16; } else if ncc == self.lexemes.esc_oct { radix = 8; + } else if ncc == self.lexemes.radix_point { + floating = true; + buffer.push(cc); + buffer.push(ncc); } else { buffer.push(cc); buffer.push(ncc); } + } else { + buffer.push(cc); } loop { -- cgit v1.2.3-70-g09d2