orinium_browser/engine/css/

tokenizer.rs

//! CSS Tokenizer
//!
//! This module implements a **CSS tokenizer**, responsible for converting
//! a raw CSS source string into a flat stream of tokens.
//!
//! ## Responsibilities
//!
//! - Consume raw characters
//! - Produce syntactic tokens defined by the CSS specification
//! - Preserve the original structure of the input as much as possible
//!
//! ## Non-responsibilities
//!
//! - Parsing selectors or declarations
//! - Interpreting values (lengths, colors, percentages, etc.)
//! - Building trees or nested structures
//!
//! ## Design notes
//!
//! - Tokens are produced in a **linear stream**
//! - Function tokens only represent the function name
//! - Matching of parentheses and function arguments is handled by the parser

/// CSS token produced by the tokenizer.
///
/// This represents *syntactic units* only.
/// No semantic interpretation (length, color, etc.) is performed here.
#[derive(Debug, Clone, PartialEq)]
pub enum Token {
    /// Identifier token (e.g. `div`, `color`, `--custom`)
    Ident(String),

    /// Function token (e.g. `calc`, `var`)
    Function(String),

    /// Unquoted `url(...)` token (e.g. `url(data:image/png;base64,...)`). The
    /// raw inner content is captured as a single lexeme, including characters
    /// that would otherwise be tokenized further (such as `:` or `%` in a
    /// base64 `data:` URL).
    Url(String),

    /// Plain number without unit (e.g. `0`, `1.5`)
    Number(f32),

    /// Quoted string token (e.g. `"hello"`, `'world'`)
    String(String),

    /// Dimension token (e.g. `10px`, `50%`, `2em`)
    ///
    /// Percentages are also represented as a dimension
    /// with `%` as the unit.
    Dimension(f32, String),

    /// Delimiter token (single-character symbols such as `:`, `;`, `>`, `+`)
    Delim(char),

    /// Hash with String (e.g. `#fff`)
    Hash(String),

    /// AtKeyword (e.g. `@media`)
    AtKeyword(String),

    /// One or more whitespace characters
    Whitespace,

    /// Comment
    Comment(String),

    /// End-of-input marker
    EOF,
}

/// CSS tokenizer.
///
/// This struct is responsible for converting a CSS source string
/// into a stream of `Token`s.
///
/// Responsibilities:
/// - Consume raw characters
/// - Produce syntactic tokens
///
/// Non-responsibilities:
/// - Parsing declarations or selectors
/// - Interpreting values (length, color, etc.)
/// - Building trees or higher-level structures
#[derive(Clone)]
pub struct Tokenizer<'a> {
    /// Iterator over the input characters
    chars: std::str::Chars<'a>,

    /// Current character under examination
    current: Option<char>,
}

impl<'a> Tokenizer<'a> {
    /// Create a new tokenizer from a CSS source string.
    pub fn new(input: &'a str) -> Self {
        let mut chars = input.chars();
        let current = chars.next();

        Self { chars, current }
    }

    /// Advance to the next character.
    ///
    /// This method should update `self.current`.
    fn bump(&mut self) {
        self.current = self.chars.next();
    }

    /// Peek the current character without consuming it.
    fn peek(&self) -> Option<char> {
        self.current
    }

    /// Peek the next character from the current one without consuming it.
    fn peek_next(&self) -> Option<char> {
        self.chars.clone().next()
    }

    /// Consume and return the next token from the input.
    ///
    /// This is the main entry point used by the parser.
    pub fn next_token(&mut self) -> Token {
        let token = match self.peek() {
            Some(c) if c.is_whitespace() => self.consume_whitespace(),
            Some(c) if is_number_start(c, self.peek_next()) => self.consume_number_like(),
            Some(c) if is_ident_start(c) => self.consume_ident_like(),
            Some(c) if is_string_delimiter(c) => self.consume_string_like(),
            Some('/') => {
                if self.peek_next() == Some('*') {
                    self.bump(); // consume '/'
                    self.bump(); // consume '*'
                    self.consume_comment()
                } else {
                    self.bump();
                    Token::Delim('/')
                }
            }
            Some('#') => {
                self.bump(); // consume '#'
                let mut value = String::new();
                while let Some(c) = self.peek() {
                    if is_ident_continue(c) {
                        value.push(c);
                        self.bump();
                    } else {
                        break;
                    }
                }
                Token::Hash(value)
            }
            Some('@') => {
                self.bump();
                let mut value = String::new();
                while let Some(c) = self.peek() {
                    if is_ident_continue(c) {
                        value.push(c);
                        self.bump();
                    } else {
                        break;
                    }
                }
                Token::AtKeyword(value)
            }
            Some(c) => {
                self.bump();
                Token::Delim(c)
            }
            None => Token::EOF,
        };

        log::debug!(target: "CssTokenizer", "Tokenized: {:?}", token);

        token
    }

    /// Consume consecutive whitespace characters.
    ///
    /// Produces a single `Token::Whitespace`.
    fn consume_whitespace(&mut self) -> Token {
        while matches!(self.current, Some(c) if c.is_whitespace()) {
            self.bump();
        }
        Token::Whitespace
    }

    /// Consume an identifier or function token.
    ///
    /// If an identifier is immediately followed by `(`,
    /// this method should produce a `Token::Function`.
    fn consume_ident_like(&mut self) -> Token {
        let mut ident = String::new();

        while let Some(c) = self.peek() {
            if c == '\\' {
                if let Some(escaped) = self.consume_escape() {
                    ident.push(escaped);
                }
            } else if is_ident_continue(c) {
                ident.push(c);
                self.bump();
            } else {
                break;
            }
        }
        if self.peek() == Some('(') {
            if ident.eq_ignore_ascii_case("url") {
                self.consume_url_after_name(ident)
            } else {
                Token::Function(ident)
            }
        } else {
            Token::Ident(ident)
        }
    }

    /// Consume the body of an unquoted `url(...)` token after the name has
    /// already been read.
    ///
    /// Per the CSS Syntax spec, `url(` followed by a string delimiter is a
    /// plain function token whose quoted string argument is handled by the
    /// parser. Otherwise we consume the raw content up to the closing `)`,
    /// resolving escapes, so that the URL is preserved byte-for-byte.
    fn consume_url_after_name(&mut self, ident: String) -> Token {
        self.bump(); // consume '('

        if let Some(c) = self.peek()
            && is_string_delimiter(c)
        {
            // Quoted URL (e.g. `url("foo.png")`) — behave as a function token
            // so the parser handles the string argument.
            return Token::Function(ident);
        }

        let mut value = String::new();

        loop {
            match self.peek() {
                None => break, // EOF without a closing paren
                Some(')') => {
                    self.bump();
                    break;
                }
                Some('\\') => {
                    if let Some(escaped) = self.consume_escape() {
                        value.push(escaped);
                    } else {
                        // A lone trailing backslash: bad URL, but keep the
                        // eventual closing paren in sync.
                        self.bump();
                        break;
                    }
                }
                Some(c) => {
                    value.push(c);
                    self.bump();
                }
            }
        }

        Token::Url(value.trim().to_string())
    }

    fn consume_string_like(&mut self) -> Token {
        let quote = self.peek().unwrap(); // '"' or '\''
        self.bump(); // consume opening quote

        let mut value = String::new();

        while let Some(c) = self.peek() {
            if c == quote {
                self.bump(); // consume closing quote
                break;
            }

            if c == '\\' {
                if let Some(escaped) = self.consume_escape() {
                    value.push(escaped);
                }
                continue;
            }

            value.push(c);
            self.bump();
        }

        Token::String(value)
    }

    /// Consume a number-like token.
    ///
    /// This may produce:
    /// - `Token::Number`
    /// - `Token::Dimension` (including `%`)
    fn consume_number_like(&mut self) -> Token {
        let mut buf = String::new();

        let mut has_dot = if self.peek() == Some('.') {
            buf.push('.');
            self.bump();
            true
        } else {
            false
        };

        if self.peek() == Some('-') {
            buf.push('-');
            self.bump();
        }

        while let Some(c) = self.peek() {
            if c.is_ascii_digit() {
                buf.push(c);
                self.bump();
            } else if c == '.' && !has_dot {
                has_dot = true;
                buf.push(c);
                self.bump();
            } else {
                break;
            }
        }

        let value: f32 = buf.parse().unwrap_or(0.0);

        // --- unit / percentage branching ---
        match self.peek() {
            Some('%') => {
                self.bump();
                Token::Dimension(value, "%".to_string())
            }
            Some(c) if is_ident_start(c) => {
                let mut unit = String::new();
                while let Some(c) = self.peek() {
                    if is_ident_continue(c) {
                        unit.push(c);
                        self.bump();
                    } else {
                        break;
                    }
                }
                Token::Dimension(value, unit)
            }
            _ => Token::Number(value),
        }
    }

    /// Consume a CSS comment.
    ///
    /// Assumes the opening `/*` has already been consumed.
    fn consume_comment(&mut self) -> Token {
        let mut value = String::new();

        while let Some(c) = self.peek() {
            if c == '*' && self.peek_next() == Some('/') {
                self.bump(); // consume '*'
                self.bump(); // consume '/'
                break;
            } else {
                value.push(c);
                self.bump();
            }
        }

        Token::Comment(value)
    }

    fn consume_escape(&mut self) -> Option<char> {
        self.bump(); // consume '\'

        // 1. Line continuation: backslash + newline => nothing
        match self.peek() {
            Some('\n') => {
                self.bump();
                return None;
            }
            Some('\r') => {
                self.bump();
                if self.peek() == Some('\n') {
                    self.bump(); // CRLF
                }
                return None;
            }
            _ => {}
        }

        // 2. Unicode escape
        let mut hex = String::new();
        for _ in 0..6 {
            match self.peek() {
                Some(c) if c.is_ascii_hexdigit() => {
                    hex.push(c);
                    self.bump();
                }
                _ => break,
            }
        }

        if !hex.is_empty() {
            if matches!(self.peek(), Some(c) if c.is_whitespace()) {
                self.bump(); // optional whitespace
            }

            let code = u32::from_str_radix(&hex, 16).ok()?;
            return std::char::from_u32(code).or(Some('\u{FFFD}'));
        }

        // 3. Simple escape
        if let Some(c) = self.peek() {
            self.bump();
            Some(c)
        } else {
            None
        }
    }
}

/// Returns true if the character can start an identifier.
///
/// This is a simplified CSS identifier start check.
/// It supports:
/// - ASCII letters (A–Z, a–z)
/// - underscore (`_`)
/// - hyphen (`-`)
/// - non-ASCII characters
fn is_ident_start(c: char) -> bool {
    c.is_ascii_alphabetic() || c == '\\' || c == '_' || c == '-' || !c.is_ascii()
}

/// Returns true if the character is a CSS string delimiter.
///
/// CSS strings are delimited by either double quotes (`"`)
/// or single quotes (`'`).
fn is_string_delimiter(c: char) -> bool {
    matches!(c, '"' | '\'')
}

/// Returns true if the character can continue an identifier.
///
/// - ASCII letters (A–Z, a–z)
/// - ASCII digits (0–9)
/// - Underscore (`_`)
/// - Hyphen (`-`)
/// - Non-ASCII characters
fn is_ident_continue(c: char) -> bool {
    c.is_ascii_alphanumeric() || c == '_' || c == '-' || !c.is_ascii()
}

/// Returns true if the character is a CSS number start.
///
/// - ASCII digits (0-9)
/// - A dot followed by a digit (e.g. `.5`)
/// - A hyphen followed by a digit or dot (e.g. `-1`, `-.5`)
fn is_number_start(current: char, next: Option<char>) -> bool {
    current.is_ascii_digit()
        || (current == '.' && matches!(next, Some(c) if c.is_ascii_digit()))
        || (current == '-' && matches!(next, Some(c) if c.is_ascii_digit() || c == '.'))
}