{ : whitespace : token } Where Define (whitespace text i) Let m (STRING.length text) In Define (chomp_line i) Iterate i Cond { | [i = m] i | [(STRING.fetch text i) = `\n`] [i + 1] | True (Continue [i + 1]) } In Iterate i Cond { | [i = m] i | True Let c (STRING.fetch text i) In Cond { | [c = ` `] (Continue [i + 1]) | [c = `\n`] (Continue [i + 1]) | [c = `\\`] (Continue (chomp_line [i + 1])) | True i } } Define (token text i) Let m (STRING.length text) In Cond { | [i = m] {i 'eof} | True Let {i scan} Let c (STRING.fetch text i) In Cond { | [c = `"`] {[i + 1] scan_string} | [c = ```] {[i + 1] scan_character} | (ASCII.is_digit c) {i scan_number} | (ASCII.is_letter c) {i scan_word} | (is_op_symbol c) {i scan_operator} | True {i scan_basic_symbol} } In (scan text i m) } Where Define (scan_string text i m) Let begin i Let end Iterate {i is_escaped} From {i False} Cond { | [i = m] (OS.die "Unexpected EOF while scanning string literal.") | True Let c (STRING.fetch text i) In Cond { | is_escaped If (Or [c = `\\`] [c = `"`] [c = `n`] [c = `0`]) (Continue [i + 1] False) (OS.die "Invalid escape sequence in string literal.") | True Cond { | [c = `"`] i | [c = `\\`] (Continue [i + 1] True) | [c = `\n`] (OS.die "Incomplete string literal.") | True (Continue [i + 1] False) } } } In {[end + 1] 'str.(STRING.clip text [begin - 1] [end + 1])} Define (scan_character text i m) Iterate {i state} From {i 'start} Cond { | [i = m] (OS.die "Unexpected EOF in character literal.") | True Let c (STRING.fetch text i) In Match state { | 'start Cond { | [c = `\\`] (Continue [i + 1] 'escaped) | [c = `\n`] (OS.die "Incomplete character literal.") | True (Continue [i + 1] 'stop.c) } | 'escaped Let num Cond { | [c = `\\`] `\\` | [c = `n`] `\n` | [c = `0`] `\0` | True (OS.die "Invalid escape sequence in character literal.") } In (Continue [i + 1] 'stop.num) | 'stop.num If [c = ```] {[i + 1] 'num.num} (OS.die "Invalid character literal.") } } Define (scan_number text i m) Iterate {i num} From {i 0} Cond { | [i = m] {i 'num.num} | True Let c (STRING.fetch text i) In If (ASCII.is_digit c) (Continue [i + 1] [[num * 10] + [c - `0`]]) {i 'num.num} } Define (scan_word text i m) Let begin i Let {end has_upper has_lower} Iterate {i has_upper has_lower} From {i False False} Cond { | [i = m] {i has_upper has_lower} | True Let c (STRING.fetch text i) In Cond { | (ASCII.is_upper c) (Continue [i + 1] True has_lower) | (ASCII.is_lower c) (Continue [i + 1] has_upper True) | (ASCII.is_digit c) (Continue [i + 1] has_upper has_lower) | [c = `_`] (Continue [i + 1] has_upper has_lower) | True {i has_upper has_lower} } } In Let word (STRING.clip text begin end) In If (And has_upper has_lower) {end 'sym.word} {end 'id.word} Define (scan_operator text i m) Let begin i Let end Iterate i From [i + 1] Cond { | [i = m] i | (is_op_symbol (STRING.fetch text i)) (Continue [i + 1]) | True i } In {end 'op.(STRING.clip text begin end)} Define (scan_basic_symbol text i m) {[i + 1] 'sym.(STRING.clip text i [i + 1])} Where Define (is_op_symbol c) (Or [c = `+`] [c = `-`] [c = `*`] [c = `/`] [c = `%`] [c = `=`] [c = `<`] [c = `>`] [c = `!`] [c = `&`]) Where Let ASCII Package "ascii" Let OS Package "os" Let STRING Package "string"