{Record whitespace token}

Where

Let (whitespace text i)
    Let n (STRING.length text)
    In
    Let (chomp_line i)
        Iterate {i} From {i}
            If (i = n)
                i
                If ((STRING.fetch text i) = '\n')
                    (i + 1)
                    Continue {(i + 1)}
    In
    Iterate {i} From {i}
        If (i = n)
            i
            Switch (STRING.fetch text i)
            | ' ' Continue {(i + 1)}
            | '\n' Continue {(i + 1)}
            | '\\' Continue {(chomp_line i)}
            | _ i
            ;

Let (token text i)
    If (i = (STRING.length text))
        {i `eof}
        Switch (STRING.fetch text i)
        | '\'' (scan_char text i)
        | '"' (scan_string text i)
        | c
            Cond
            | (ASCII.is_letter c)
                (scan_word text i)
            | (ASCII.is_digit c)
                (scan_number text i)
            | True
                (scan_basic_symbol text i)
            ;
        ;

Where

Let (scan_char text i)
    Let i (i + 1)
    In
    Let {c i}
        Switch (peek text i)
        | '\\'
            Let i (i + 1)
            In
            Switch (peek text i)
            | '\\' {'\\' i}
            | 'n' {'\n' i}
            | '\'' {'\'' i}
            | _ (OS.die "Invalid character escape sequence.")
            ;
        | '\'' (OS.die "Empty character literal.")
        | '\n' (OS.die "Character literal contains newline.")
        | c {c i}
        ;
    In
    If !((peek text (i + 1)) = '\'')
        (OS.die "Expected quotation mark to end character literal.")
        {(i + 2) `num.c}

Let (scan_string text i)
    Let begin i
    Let i (i + 1)
    In
    Iterate {i} From {i}
        Switch (peek text i)
        | '"'
            Let end (i + 1)
            In
            Let token `str.(STRING.clip text begin end)
            In
            {end token}
        | '\\'
            Let j (i + 1)
            Let k (i + 2)
            In
            Switch (peek text j)
            | '\\' Continue {k}
            | 'n' Continue {k}
            | '"' Continue {k}
            | _ (OS.die "Invalid character escape sequence.")
            ;
        | '\n' (OS.die "String literal contains newline.")
        | _ Continue {(i + 1)}
        ;

Let (scan_number text i)
    Iterate {i num} From {i 0}
        Let c (peek text i)
        In
        If !(ASCII.is_digit c)
            {i `num.num}
            Continue {(i + 1) ((10 * num) + (c - '0'))}

Let (scan_word text i)
    Let begin i
    In
    Iterate {i seen_upper seen_lower} From {i False False}
        Switch (peek text i)
        | '_' Continue {(i + 1) seen_upper seen_lower}
        | '\'' Continue {(i + 1) seen_upper seen_lower}
        | c
            If (ASCII.is_digit c)
                Continue {(i + 1) seen_upper seen_lower}
                If (ASCII.is_letter c)
                    Continue {(i + 1)
                        Or seen_upper (ASCII.is_upper c)
                        Or seen_lower (ASCII.is_lower c)}
                    If And seen_upper seen_lower
                        {i `sym.(STRING.clip text begin i)}
                        {i `id.(STRING.clip text begin i)}
        ;

Let (scan_basic_symbol text i)
    {(i + 1) `sym.(STRING.clip text i (i + 1))}

Where

Let (peek text i)
    If (i >= (STRING.length text))
        (OS.die "Unexpected EOF.")
        (STRING.fetch text i)

Where

Let ASCII Package "ascii"
Let OS Package "os"
Let STRING Package "string"