lib/cc/lexer.viv

# The lexer: C source text in, a list of tokens out.
#
# A token is the smallest unit the parser reads: a number, a name, a
# keyword or a punctuator. The lexer walks the text once, left to right,
# skipping spaces and taking the longest token that starts at each point.
# The order of features follows chibicc (github.com/rui314/chibicc, Rui
# Ueyama, MIT).

enum TokKind {
    Num         # an integer literal
    Ident       # a name: a letter or _, then letters, digits and _
    Keyword     # a name the language reserves: return, int, if, ...
    Punct       # an operator or other punctuation: + - * / ( ) == < ...
    Eof         # the end of the input
}

#| One token: what kind it is, its text, its value when it is a number,
#| and where it starts (line and column, from 1), for error messages.
record Token {
    kind: TokKind
    text: Str
    val: Int = 0
    line: Int = 1
    col: Int = 1
}

#| The punctuators, longest first, so that a two-character one wins over
#| its first character.
PUNCTS = ["==", "!=", "<=", ">=", "&&", "||", "%", "!", "+", "-", "*", "/", "(", ")", "<", ">", ";", "=", ",", "{", "}"]

#| The punctuator that starts at src[i], or "" when none does.
def punct_at(src: Str, i: Int) -> Str {
    for p in PUNCTS {
        if startswith(text_between(src, i, i + len(p)), p) {
            return p
        }
    }
    return ""
}

KEYWORDS = ["return", "int", "if", "else", "for", "while"]

#| Can c begin a C identifier: a letter or `_`?
def is_ident_start(c: Str) -> Bool {
    return (c >= "a" and c <= "z") or (c >= "A" and c <= "Z") or c == "_"
}

#| Can c continue an identifier after its first character?
def is_ident_char(c: Str) -> Bool {
    return is_ident_start(c) or is_digit(c)
}

#| Is c a decimal digit (number literals, identifier tails)?
def is_digit(c: Str) -> Bool {
    return c >= "0" and c <= "9"
}

#| The text of s from index a up to (not including) b. (Viv's slice bounds
#| take numbers but not Ints, bug B342, hence the float().)
def text_between(s: Str, a: Int, b: Int) -> Str {
    return s[float(a):float(b)]
}

#| Stop with a message that points at a place in the source.
def error_at(line: Int, col: Int, msg: Str) {
    stop(f"{line}:{col}: {msg}")
}

#| The tokens of `src`, ending with an Eof token.
def tokenize(src: Str) -> [Token] {
    toks: [Token] = []
    i: Int = 0
    line: Int = 1
    line_start: Int = 0
    n: Int = len(src)
    while i < n {
        c = src[i]
        col = i - line_start + 1

        #: whitespace separates tokens and is dropped; a newline starts a new line
        if c == "\n" {
            line = line + 1
            line_start = i + 1
            i = i + 1
        }
        else if c == " " or c == "\t" or c == "\r" {
            i = i + 1
        }

        #: a line comment runs to the end of the line
        else if text_between(src, i, i + 2) == "//" {
            while i < n and src[i] != "\n" {
                i = i + 1
            }
        }

        #: a block comment runs to the next */, across lines
        else if text_between(src, i, i + 2) == "/*" {
            close = int(find(src, "*/", i + 2))
            if close < 0 {
                error_at(line, col, "unclosed block comment")
            }
            comment = text_between(src, i, close + 2)
            newlines = int(count(comment, "\n"))
            if newlines > 0 {
                line = line + newlines
                line_start = i + int(rfind(comment, "\n")) + 1
            }
            i = close + 2
        }

        #: a number: the longest run of digits
        else if is_digit(c) {
            start = i
            v: Int = 0
            while i < n and is_digit(src[i]) {
                v = v * 10 + int(src[i])
                i = i + 1
            }
            toks = append(toks, Token { kind: TokKind.Num, text: text_between(src, start, i), val: v, line: line, col: col })
        }
        #: a name, which is a keyword when it is one of KEYWORDS
        else if is_ident_start(c) {
            start = i
            while i < n and is_ident_char(src[i]) {
                i = i + 1
            }
            name = text_between(src, start, i)
            kind = TokKind.Ident
            if name in KEYWORDS {
                kind = TokKind.Keyword
            }
            toks = append(toks, Token { kind: kind, text: name, line: line, col: col })
        }

        #: a punctuator
        else if punct_at(src, i) != "" {
            p = punct_at(src, i)
            toks = append(toks, Token { kind: TokKind.Punct, text: p, line: line, col: col })
            i = i + len(p)
        }
        else {
            error_at(line, col, f"unexpected character `{c}`")
        }
    }
    toks = append(toks, Token { kind: TokKind.Eof, text: "", line: line, col: n - line_start + 1 })
    return toks
}

test "numbers and operators, spaces between or not" {
    toks = tokenize("12+34 - 5")
    assert_true(len(toks) == 6)
    assert_true(toks[1].text == "+")
    assert_true(toks[3].text == "-")
    assert_true(toks[4].col == 9)
}

test "names and keywords" {
    toks = tokenize("return x_1;")
    match toks[0].kind {
        Keyword => { assert_true(toks[0].text == "return") }
        _ => { stop("return is a keyword") }
    }
    match toks[1].kind {
        Ident => { assert_true(toks[1].text == "x_1") }
        _ => { stop("x_1 is a name") }
    }
}

test "comments are skipped, and lines still counted" {
    toks = tokenize("1 // one\n/* two\n lines */ 2")
    assert_true(len(toks) == 3)
    assert_true(toks[1].val == 2)
    assert_true(toks[1].line == 3)
}

test "a number and the end" {
    toks = tokenize(" 42 ")
    assert_true(len(toks) == 2)
    assert_true(toks[0].val == 42)
    assert_true(toks[0].col == 2)
}

# SPDX-License-Identifier: MIT
# SPDX-FileCopyrightText: 2019 Rui Ueyama
# SPDX-FileCopyrightText: 2026 The Viv authors