{
: whitespace
: token
}

Where

Define (whitespace text i)
    Let m (STRING.length text)
    In
    Define (chomp_line i)
        Iterate i
            Cond
            | [i = m] i
            | [(STRING.fetch text i) = `\n`] [i + 1]
            | True (Continue [i + 1])
            ;
    In
    Iterate i
        Cond
        | [i = m] i
        | True
            Let c (STRING.fetch text i)
            In
            Cond
            | [c = ` `] (Continue [i + 1])
            | [c = `\n`] (Continue [i + 1])
            | [c = `\\`] (Continue (chomp_line [i + 1]))
            | True i
            ;
        ;

Define (token text i)
    Let m (STRING.length text)
    In
    Cond
    | [i = m] {i 'eof}
    | True
        Let {i scan}
            Let c (STRING.fetch text i)
            In
            Cond
            | [c = `"`] {[i + 1] scan_string}
            | [c = ```] {[i + 1] scan_character}
            | (ASCII.is_digit c) {i scan_number}
            | (ASCII.is_letter c) {i scan_word}
            | (is_op_symbol c) {i scan_operator}
            | True {i scan_basic_symbol}
            ;
        In
        (scan text i m)
    ;

Where

Define (scan_string text i m)
    Let begin i
    Let end
        Iterate {i is_escaped} From {i False}
            Cond
            | [i = m] (OS.die "Unexpected EOF while scanning string literal.")
            | True
                Let c (STRING.fetch text i)
                In
                Cond
                | is_escaped
                    Let ok
                        Cond
                        | [c = `\\`] True
                        | [c = `"`] True
                        | [c = `n`] True
                        | [c = `0`] True
                        | True False
                        ;
                    In
                    If ok
                        (Continue [i + 1] False)
                        (OS.die "Invalid escape sequence in string literal.")
                | True
                    Cond
                    | [c = `"`] i
                    | [c = `\\`] (Continue [i + 1] True)
                    | [c = `\n`] (OS.die "Incomplete string literal.")
                    | True (Continue [i + 1] False)
                    ;
                ;
            ;
    In
    {[end + 1] 'str.(STRING.clip text [begin - 1] [end + 1])}

Define (scan_character text i m)
    Iterate {i state} From {i 'start}
        Cond
        | [i = m] (OS.die "Unexpected EOF in character literal.")
        | True
            Let c (STRING.fetch text i)
            In
            Match state
            | 'start
                Cond
                | [c = `\\`] (Continue [i + 1] 'escaped)
                | [c = `\n`] (OS.die "Incomplete character literal.")
                | True (Continue [i + 1] 'stop.c)
                ;
            | 'escaped
                Let num
                    Cond
                    | [c = `\\`] `\\`
                    | [c = `n`] `\n`
                    | [c = `0`] `\0`
                    | True (OS.die "Invalid escape sequence in character literal.")
                    ;
                In
                (Continue [i + 1] 'stop.num)
            | 'stop.num
                If [c = ```]
                    {[i + 1] 'num.num}
                    (OS.die "Invalid character literal.")
            ;
        ;

Define (scan_number text i m)
    Iterate {i num} From {i 0}
        Cond
        | [i = m] {i 'num.num}
        | True
            Let c (STRING.fetch text i)
            In
            If (ASCII.is_digit c)
                (Continue [i + 1] [[num * 10] + [c - `0`]])
                {i 'num.num}
        ;

Define (scan_word text i m)
    Let begin i
    Let {end has_upper has_lower}
        Iterate {i has_upper has_lower} From {i False False}
            Cond
            | [i = m] {i has_upper has_lower}
            | True
                Let c (STRING.fetch text i)
                In
                Cond
                | (ASCII.is_upper c) (Continue [i + 1] True has_lower)
                | (ASCII.is_lower c) (Continue [i + 1] has_upper True)
                | (ASCII.is_digit c) (Continue [i + 1] has_upper has_lower)
                | [c = `_`] (Continue [i + 1] has_upper has_lower)
                | True {i has_upper has_lower}
                ;
            ;
    In
    Let word (STRING.clip text begin end)
    In
    If And has_upper has_lower
        {end 'sym.word}
        {end 'id.word}

Define (scan_operator text i m)
    Let begin i
    Let end
        Iterate i From [i + 1]
            Cond
            | [i = m] i
            | (is_op_symbol (STRING.fetch text i)) (Continue [i + 1])
            | True i
            ;
    In
    {end 'op.(STRING.clip text begin end)}

Define (scan_basic_symbol text i m)
    {[i + 1] 'sym.(STRING.clip text i [i + 1])}

Where

Define (is_op_symbol c)
    Cond
    | [c = `+`] True
    | [c = `-`] True
    | [c = `*`] True
    | [c = `/`] True
    | [c = `%`] True
    | [c = `=`] True
    | [c = `<`] True
    | [c = `>`] True
    | [c = `!`] True
    | [c = `&`] True
    | True False
    ;

Where

Let ASCII Package "ascii"
Let OS Package "os"
Let STRING Package "string"