orinium_browser/engine/html/
tokenizer.rs

1//! HTML tokenizer. Converts raw HTML input into token stream.
2
3use super::util::{MAX_ENTITY_NAME_LEN, decode_entity};
4
5/// Represents a single HTML attribute
6#[derive(Debug, Clone, PartialEq)]
7pub struct Attribute {
8    pub name: String,
9    pub value: String,
10}
11
12/// HTML tokens emitted by the tokenizer
13#[derive(Debug, Clone, PartialEq)]
14pub enum Token {
15    Doctype {
16        name: Option<String>,
17        public_id: Option<String>,
18        system_id: Option<String>,
19        force_quirks: bool,
20    },
21    StartTag {
22        name: String,
23        attributes: Vec<Attribute>,
24        self_closing: bool,
25    },
26    EndTag {
27        name: String,
28    },
29    Comment(String),
30    Text(String),
31}
32
33/// Represents the internal state of the tokenizer
34#[derive(Debug, PartialEq, Clone, Copy)]
35pub enum TokenizerState {
36    Data,
37    ScriptData,
38    StyleData,
39    EscapeDecoding,
40    AttributeEscapeDecoding,
41    TagOpen,
42    EndTagOpen,
43    TagName,
44    BeforeAttributeName,
45    AttributeName,
46    AfterAttributeName,
47    BeforeAttributeValue,
48    AttributeValueDoubleQuoted,
49    AttributeValueSingleQuoted,
50    AttributeValueUnquoted,
51    SelfClosingStartTag,
52    CommentStartDash,
53    Comment,
54    CommentEndDash,
55    CommentEnd,
56    BogusComment,
57    Doctype,
58    DoctypeName,
59    BeforeDoctypePublicId,
60    DoctypePublicIdWithSingleQuote,
61    DoctypePublicIdWithDoubleQuote,
62    AfterDoctypePublicId,
63    DoctypeSystemId,
64    BogusDoctype,
65}
66
67impl TokenizerState {
68    /// Returns true if the current state is a doctype-related state
69    fn is_doctype(&self) -> bool {
70        matches!(
71            self,
72            TokenizerState::Doctype
73                | TokenizerState::DoctypeName
74                | TokenizerState::BeforeDoctypePublicId
75                | TokenizerState::DoctypePublicIdWithSingleQuote
76                | TokenizerState::DoctypePublicIdWithDoubleQuote
77                | TokenizerState::AfterDoctypePublicId
78                | TokenizerState::DoctypeSystemId
79                | TokenizerState::BogusDoctype
80        )
81    }
82
83    /// Returns true if the current state is a comment-related state
84    fn is_comment(&self) -> bool {
85        matches!(
86            self,
87            TokenizerState::Comment
88                | TokenizerState::CommentStartDash
89                | TokenizerState::CommentEndDash
90                | TokenizerState::CommentEnd
91                | TokenizerState::BogusComment
92        )
93    }
94}
95
96/// HTML tokenizer implementation
97pub struct Tokenizer<'a> {
98    input: &'a str,
99    pos: usize,
100    token: Option<Token>,
101    state: TokenizerState,
102    current_token: Option<Token>,
103    current_attribute: Option<Attribute>,
104    buffer: String,
105    escape_return_state: TokenizerState,
106}
107
108impl<'a> Tokenizer<'a> {
109    /// Creates a new tokenizer for the given input
110    pub fn new(input: &'a str) -> Self {
111        Self {
112            input,
113            pos: 0,
114            token: None,
115            state: TokenizerState::Data,
116            current_token: None,
117            current_attribute: None,
118            buffer: String::new(),
119            escape_return_state: TokenizerState::Data,
120        }
121    }
122
123    /// Returns the next character from input and advances the position
124    fn next_char(&mut self) -> Option<char> {
125        if self.pos >= self.input.len() {
126            None
127        } else {
128            let c = self.input[self.pos..].chars().next().unwrap();
129            self.pos += c.len_utf8();
130            Some(c)
131        }
132    }
133
134    /// Emits the current token and clears the buffer
135    fn commit_token(&mut self) {
136        self.token = self.current_token.take();
137        self.buffer.clear();
138    }
139
140    /// Pushes the current attribute to the start tag if exists
141    fn push_current_attribute(&mut self) {
142        if let (Some(attr), Some(Token::StartTag { attributes, .. })) =
143            (self.current_attribute.take(), &mut self.current_token)
144        {
145            attributes.push(attr);
146        }
147    }
148
149    fn handle_special_tag_state_transition(&mut self, token: &Token) {
150        if let Token::StartTag { name, .. } = token {
151            // Switch to ScriptData or StyleData state if we encounter <script> or <style> start tags
152            match name.to_lowercase().as_str() {
153                "script" => self.state = TokenizerState::ScriptData,
154                "style" => self.state = TokenizerState::StyleData,
155                _ => self.state = TokenizerState::Data,
156            }
157        } else {
158            // Do nothing. We only switch to ScriptData or StyleData on StartTag, and we return to Data on EndTag.
159        }
160    }
161
162    /// Debug log for emitted tokens
163    #[inline(always)]
164    fn debug_emit(&self, _token: &Token) {
165        #[cfg(debug_assertions)]
166        match _token {
167            Token::StartTag { name, .. } => {
168                log::debug!(target:"HtmlTokenizer::EmitToken::TagStart", "Emitting token: {name}, Pos: {}", self.pos)
169            }
170            Token::EndTag { name } => {
171                log::debug!(target:"HtmlTokenizer::EmitToken::TagEnd", "Emitting token: {name}, Pos: {}", self.pos)
172            }
173            Token::Comment(comment) => {
174                log::debug!(target:"HtmlTokenizer::EmitToken::Comment", "Emitting token: {}, Pos: {}", comment, self.pos)
175            }
176            Token::Text(text) => {
177                log::debug!(target:"HtmlTokenizer::EmitToken::Text", "Emitting token: `{text}`, Pos: {}", self.pos)
178            }
179            _ => {}
180        }
181    }
182
183    /// Returns the next token if available
184    pub fn next_token(&mut self) -> Option<Token> {
185        while let Some(c) = self.next_char() {
186            log::debug!(target:"HtmlTokenizer::Char", "State: {:?}, Char: '{}'", self.state, c);
187
188            match self.state {
189                TokenizerState::Data | TokenizerState::StyleData | TokenizerState::ScriptData => {
190                    self.state_data(c)
191                }
192                TokenizerState::EscapeDecoding => self.state_escape_decoding(c),
193                TokenizerState::AttributeEscapeDecoding => self.state_attribute_escape_decoding(c),
194                _ if self.state.is_doctype() => self.state_doctype(c),
195                TokenizerState::TagOpen => self.state_tag_open(c),
196                TokenizerState::TagName => self.state_tag_name(c),
197                TokenizerState::BeforeAttributeName => self.state_before_attribute_name(c),
198                TokenizerState::AttributeName => self.state_attribute_name(c),
199                TokenizerState::BeforeAttributeValue => self.state_before_attribute_value(c),
200                TokenizerState::AttributeValueDoubleQuoted
201                | TokenizerState::AttributeValueSingleQuoted => {
202                    self.state_attribute_value_quoted(c)
203                }
204                TokenizerState::AfterAttributeName => self.state_after_attribute_name(c),
205                TokenizerState::AttributeValueUnquoted => self.state_attribute_value_unquoted(c),
206                TokenizerState::SelfClosingStartTag => self.state_self_closing_start_tag(c),
207                TokenizerState::EndTagOpen => self.state_end_tag_open(c),
208                _ if self.state.is_comment() => self.state_comment(c),
209                _ => {
210                    log::error!(target:"HtmlTokenizer::State", "Unimplemented state: {:?}, returning to Data state", self.state);
211                    self.state = TokenizerState::Data;
212                }
213            }
214
215            if let Some(token) = self.token.take() {
216                self.debug_emit(&token);
217                self.handle_special_tag_state_transition(&token);
218                return Some(token);
219            }
220        }
221
222        // End of input: commit remaining current_token if exists
223        if self.current_token.is_some() {
224            self.commit_token();
225            return self.token.take();
226        }
227
228        // Emit BogusComment if input ended while in comment
229        if self.state.is_comment() {
230            self.state = TokenizerState::BogusComment;
231            self.commit_token();
232            return self.token.take();
233        }
234
235        None
236    }
237
238    // --- State handlers ---
239    fn state_data(&mut self, c: char) {
240        match c {
241            '<' => {
242                // In raw-text states (<script>, <style>) a `<` only opens a tag
243                // when it is `</` (the closing tag); any other `<` is literal
244                // text (e.g. the `<` inside a JS string like 'di<v').
245                let raw_text = matches!(
246                    self.state,
247                    TokenizerState::ScriptData | TokenizerState::StyleData
248                );
249                if raw_text && !self.input[self.pos..].starts_with('/') {
250                    self.buffer.push('<');
251                    match &mut self.current_token {
252                        Some(Token::Text(text)) => text.push('<'),
253                        _ => self.current_token = Some(Token::Text("<".to_string())),
254                    }
255                } else {
256                    self.commit_token();
257                    self.state = TokenizerState::TagOpen;
258                }
259            }
260            // Handle escape entities only in Data. Not in ScriptData, and StyleData states.
261            '&' if self.state == TokenizerState::Data => {
262                // Reset the buffer so it holds only the entity being decoded;
263                // it may still contain the preceding text characters, whose
264                // length would otherwise trip the MAX_ENTITY_NAME_LEN check
265                // mid-entity and mangle the entity.
266                self.buffer.clear();
267                self.buffer.push('&');
268                self.state = TokenizerState::EscapeDecoding;
269            }
270            _ => {
271                self.buffer.push(c);
272                match &mut self.current_token {
273                    Some(Token::Text(text)) => text.push(c),
274                    _ => self.current_token = Some(Token::Text(c.to_string())),
275                }
276            }
277        }
278    }
279
280    fn state_escape_decoding(&mut self, c: char) {
281        if c == ';' {
282            let mut iter = self.buffer.rsplitn(2, '&');
283            let entity = iter.next().unwrap_or("");
284
285            let decoded = decode_entity(entity).unwrap_or_else(|| format!("&{};", entity));
286
287            match &mut self.current_token {
288                Some(Token::Text(text)) => text.push_str(&decoded),
289                _ => self.current_token = Some(Token::Text(decoded)),
290            }
291
292            self.buffer.clear();
293            self.state = TokenizerState::Data;
294        } else if self.buffer.len() > MAX_ENTITY_NAME_LEN || self.input[self.pos..].starts_with('<')
295        {
296            let mut iter = self.buffer.rsplitn(2, '&');
297            let entity = iter.next().unwrap_or("");
298            let decoded = format!("&{}", entity);
299
300            match &mut self.current_token {
301                Some(Token::Text(text)) => text.push_str(&decoded),
302                _ => self.current_token = Some(Token::Text(decoded)),
303            }
304
305            self.buffer.clear();
306            self.state = TokenizerState::Data;
307        } else {
308            self.buffer.push(c);
309        }
310    }
311
312    fn state_tag_open(&mut self, c: char) {
313        match c {
314            '/' => self.state = TokenizerState::EndTagOpen,
315            '!' => {
316                if self.input[self.pos..].starts_with('-') {
317                    self.pos += 1;
318                    self.state = TokenizerState::CommentStartDash;
319                } else if self.input[self.pos..].to_lowercase().starts_with("doctype") {
320                    self.pos += 7;
321                    self.state = TokenizerState::Doctype;
322                    self.current_token = Some(Token::Doctype {
323                        name: None,
324                        public_id: None,
325                        system_id: None,
326                        force_quirks: false,
327                    });
328                } else {
329                    self.state = TokenizerState::BogusComment;
330                }
331            }
332            c if c.is_ascii_alphabetic() => {
333                self.state = TokenizerState::TagName;
334                self.buffer.push(c);
335                self.current_token = Some(Token::StartTag {
336                    name: c.to_string(),
337                    attributes: Vec::new(),
338                    self_closing: false,
339                });
340            }
341            _ => {
342                self.buffer.push('<');
343                self.buffer.push(c);
344                match &mut self.current_token {
345                    Some(Token::Text(text)) => {
346                        text.push('<');
347                        text.push(c);
348                    }
349                    _ => self.current_token = Some(Token::Text(format!("<{c}"))),
350                }
351                self.state = TokenizerState::Data;
352            }
353        }
354    }
355
356    fn state_tag_name(&mut self, c: char) {
357        match c {
358            c if c.is_whitespace() => self.state = TokenizerState::BeforeAttributeName,
359            '/' => self.state = TokenizerState::SelfClosingStartTag,
360            '>' => {
361                self.commit_token();
362                self.state = TokenizerState::Data;
363            }
364            c if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == ':' => {
365                self.buffer.push(c);
366                match &mut self.current_token {
367                    Some(Token::StartTag { name, .. }) => name.push(c),
368                    Some(Token::EndTag { name }) => name.push(c),
369                    _ => {}
370                }
371            }
372            _ => {
373                self.commit_token();
374                self.state = TokenizerState::Data;
375            }
376        }
377    }
378
379    fn state_before_attribute_name(&mut self, c: char) {
380        match c {
381            c if c.is_whitespace() => {}
382            '/' => self.state = TokenizerState::SelfClosingStartTag,
383            '>' => {
384                self.commit_token();
385                self.state = TokenizerState::Data;
386            }
387            c if c.is_ascii_alphanumeric() => {
388                self.state = TokenizerState::AttributeName;
389                self.buffer.push(c);
390                self.current_attribute = Some(Attribute {
391                    name: c.to_string(),
392                    value: String::new(),
393                });
394            }
395            _ => {}
396        }
397    }
398
399    fn state_attribute_name(&mut self, c: char) {
400        match c {
401            c if c.is_whitespace() => self.state = TokenizerState::AfterAttributeName,
402            '=' => self.state = TokenizerState::BeforeAttributeValue,
403            '/' => {
404                self.push_current_attribute();
405                self.state = TokenizerState::SelfClosingStartTag;
406            }
407            '>' => {
408                self.push_current_attribute();
409                self.commit_token();
410                self.state = TokenizerState::Data;
411            }
412            c if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == ':' => {
413                self.buffer.push(c);
414                if let Some(attr) = &mut self.current_attribute {
415                    attr.name.push(c);
416                }
417            }
418            _ => {}
419        }
420    }
421
422    fn state_before_attribute_value(&mut self, c: char) {
423        match c {
424            c if c.is_whitespace() => {}
425            '"' => self.state = TokenizerState::AttributeValueDoubleQuoted,
426            '\'' => self.state = TokenizerState::AttributeValueSingleQuoted,
427            '>' => {
428                self.push_current_attribute();
429                self.commit_token();
430                self.state = TokenizerState::Data;
431            }
432            _ => {
433                self.state = TokenizerState::AttributeValueUnquoted;
434                self.state_attribute_value_unquoted(c);
435            }
436        }
437    }
438
439    fn state_attribute_value_quoted(&mut self, c: char) {
440        match (&self.state, c) {
441            (&TokenizerState::AttributeValueDoubleQuoted, '"')
442            | (&TokenizerState::AttributeValueSingleQuoted, '\'') => {
443                self.push_current_attribute();
444                self.state = TokenizerState::AfterAttributeName;
445            }
446            (_, '&') => self.start_attribute_escape(),
447            _ => {
448                if let Some(attr) = &mut self.current_attribute {
449                    attr.value.push(c);
450                }
451            }
452        }
453    }
454
455    fn state_after_attribute_name(&mut self, c: char) {
456        match c {
457            c if c.is_whitespace() => {}
458            '/' => {
459                self.push_current_attribute();
460                self.state = TokenizerState::SelfClosingStartTag;
461            }
462            '=' => self.state = TokenizerState::BeforeAttributeValue,
463            '>' => {
464                self.push_current_attribute();
465                self.commit_token();
466                self.state = TokenizerState::Data;
467            }
468            c if c.is_ascii_alphanumeric() => {
469                self.push_current_attribute();
470                self.state = TokenizerState::AttributeName;
471                self.buffer.push(c);
472                self.current_attribute = Some(Attribute {
473                    name: c.to_string(),
474                    value: String::new(),
475                });
476            }
477            _ => {}
478        }
479    }
480
481    fn state_attribute_value_unquoted(&mut self, c: char) {
482        match c {
483            c if c.is_whitespace() => {
484                self.push_current_attribute();
485                self.state = TokenizerState::BeforeAttributeName;
486            }
487            '>' => {
488                self.push_current_attribute();
489                self.commit_token();
490                self.state = TokenizerState::Data;
491            }
492            '&' => self.start_attribute_escape(),
493            _ => {
494                if let Some(attr) = &mut self.current_attribute {
495                    attr.value.push(c);
496                }
497            }
498        }
499    }
500
501    /// Begins decoding a character reference inside an attribute value. The
502    /// entity is buffered and decoded on `;` (or flushed literally when the
503    /// following character cannot be part of an entity name), then control
504    /// returns to the attribute-value state that was active before the `&`.
505    fn start_attribute_escape(&mut self) {
506        self.buffer.clear();
507        self.buffer.push('&');
508        self.escape_return_state = self.state;
509        self.state = TokenizerState::AttributeEscapeDecoding;
510    }
511
512    fn state_attribute_escape_decoding(&mut self, c: char) {
513        if c == ';' {
514            let mut iter = self.buffer.rsplitn(2, '&');
515            let entity = iter.next().unwrap_or("");
516            let decoded = decode_entity(entity).unwrap_or_else(|| format!("&{};", entity));
517            if let Some(attr) = &mut self.current_attribute {
518                attr.value.push_str(&decoded);
519            }
520            self.buffer.clear();
521            self.state = std::mem::replace(&mut self.escape_return_state, TokenizerState::Data);
522        } else if !(c.is_ascii_alphanumeric() || matches!(c, '#' | 'x' | 'X'))
523            || self.buffer.len() > MAX_ENTITY_NAME_LEN
524            || self.input[self.pos..].starts_with('<')
525        {
526            // Not part of an entity name (or the entity ran too long): flush
527            // the literal `&…` text and reconsume this character so the
528            // active attribute-value state can handle it (closing quote, `>`,
529            // whitespace, …).
530            let mut iter = self.buffer.rsplitn(2, '&');
531            let entity = iter.next().unwrap_or("");
532            if let Some(attr) = &mut self.current_attribute {
533                attr.value.push_str(&format!("&{}", entity));
534            }
535            self.buffer.clear();
536            self.pos -= c.len_utf8();
537            self.state = std::mem::replace(&mut self.escape_return_state, TokenizerState::Data);
538        } else {
539            self.buffer.push(c);
540        }
541    }
542
543    fn state_self_closing_start_tag(&mut self, c: char) {
544        match c {
545            '>' => {
546                if let Some(Token::StartTag { self_closing, .. }) = &mut self.current_token {
547                    *self_closing = true;
548                }
549                self.commit_token();
550                self.state = TokenizerState::Data;
551            }
552            _ => self.state = TokenizerState::Data,
553        }
554    }
555
556    fn state_end_tag_open(&mut self, c: char) {
557        match c {
558            c if c.is_ascii_alphabetic() => {
559                self.state = TokenizerState::TagName;
560                self.buffer.push(c);
561                self.current_token = Some(Token::EndTag {
562                    name: c.to_string(),
563                });
564            }
565            _ => self.state = TokenizerState::Data,
566        }
567    }
568
569    fn state_comment(&mut self, c: char) {
570        match self.state {
571            TokenizerState::CommentStartDash => {
572                if c == '-' {
573                    self.state = TokenizerState::Comment;
574                    self.current_token = Some(Token::Comment(String::new()));
575                } else {
576                    self.state = TokenizerState::BogusComment;
577                }
578            }
579            TokenizerState::Comment => {
580                if c == '-' {
581                    self.state = TokenizerState::CommentEndDash;
582                } else if let Some(Token::Comment(comment)) = &mut self.current_token {
583                    comment.push(c);
584                }
585            }
586            TokenizerState::CommentEndDash => {
587                if c == '-' {
588                    self.state = TokenizerState::CommentEnd;
589                } else {
590                    self.state = TokenizerState::Comment;
591                    if let Some(Token::Comment(comment)) = &mut self.current_token {
592                        comment.push('-');
593                        comment.push(c);
594                    }
595                }
596            }
597            TokenizerState::CommentEnd => {
598                if c == '>' {
599                    self.commit_token();
600                    self.state = TokenizerState::Data;
601                } else {
602                    self.state = TokenizerState::Comment;
603                    if let Some(Token::Comment(comment)) = &mut self.current_token {
604                        comment.push_str("--");
605                        comment.push(c);
606                    }
607                }
608            }
609            _ => {}
610        }
611    }
612
613    /// Returns whether the remaining input at the current position starts with
614    /// `keyword`, compared ASCII case-insensitively without allocating.
615    fn peek_starts_with_ascii_ci(&self, keyword: &str) -> bool {
616        self.input[self.pos..]
617            .get(..keyword.len())
618            .is_some_and(|prefix| prefix.eq_ignore_ascii_case(keyword))
619    }
620
621    fn state_doctype(&mut self, c: char) {
622        match c {
623            c if c.is_whitespace() => match self.state {
624                TokenizerState::Doctype => self.state = TokenizerState::DoctypeName,
625                TokenizerState::DoctypeName
626                    if (self.peek_starts_with_ascii_ci("public")
627                        || self.peek_starts_with_ascii_ci("system")) =>
628                {
629                    self.pos += 6;
630                    self.state = TokenizerState::BeforeDoctypePublicId;
631                }
632                TokenizerState::AfterDoctypePublicId => {
633                    self.state = TokenizerState::DoctypeSystemId;
634                }
635                _ => {}
636            },
637            '>' => {
638                if let Some(Token::Doctype { force_quirks, .. }) = &mut self.current_token
639                    && self.state == TokenizerState::BogusDoctype
640                {
641                    *force_quirks = true;
642                }
643                self.commit_token();
644                self.state = TokenizerState::Data;
645            }
646            _ => {
647                self.buffer.push(c);
648                match self.state {
649                    TokenizerState::Doctype => self.state = TokenizerState::BogusDoctype,
650                    TokenizerState::DoctypeName => {
651                        if let Some(Token::Doctype { name, .. }) = &mut self.current_token {
652                            if name.is_none() {
653                                *name = Some(c.to_string());
654                            } else if let Some(n) = name {
655                                n.push(c);
656                            }
657                        }
658                    }
659                    TokenizerState::BeforeDoctypePublicId => {
660                        match c {
661                            '"' => self.state = TokenizerState::DoctypePublicIdWithDoubleQuote,
662                            '\'' => self.state = TokenizerState::DoctypePublicIdWithSingleQuote,
663                            _ if c.is_whitespace() => {}
664                            _ => self.state = TokenizerState::BogusDoctype,
665                        }
666                        if let Some(Token::Doctype { public_id, .. }) = &mut self.current_token {
667                            *public_id = Some(c.to_string());
668                        }
669                    }
670                    TokenizerState::DoctypePublicIdWithSingleQuote
671                    | TokenizerState::DoctypePublicIdWithDoubleQuote => {
672                        if let Some(Token::Doctype { public_id, .. }) = &mut self.current_token
673                            && let Some(pid) = public_id
674                        {
675                            pid.push(c);
676                        }
677                        if (self.state == TokenizerState::DoctypePublicIdWithSingleQuote
678                            && c == '\'')
679                            || (self.state == TokenizerState::DoctypePublicIdWithDoubleQuote
680                                && c == '"')
681                        {
682                            self.state = TokenizerState::AfterDoctypePublicId;
683                        }
684                    }
685                    TokenizerState::DoctypeSystemId => {
686                        if let Some(Token::Doctype { system_id, .. }) = &mut self.current_token {
687                            if system_id.is_none() {
688                                *system_id = Some(c.to_string());
689                            } else if let Some(sid) = system_id {
690                                sid.push(c);
691                            }
692                        }
693                    }
694                    _ => {}
695                }
696            }
697        }
698    }
699}
700
701#[cfg(test)]
702mod tests {
703    use super::*;
704
705    fn collect_tokens(input: &str) -> Vec<Token> {
706        let mut tokenizer = Tokenizer::new(input);
707        let mut tokens = Vec::new();
708        while let Some(token) = tokenizer.next_token() {
709            tokens.push(token);
710        }
711        tokens
712    }
713
714    #[test]
715    fn test_text_node() {
716        let input = "Hello, world!";
717        let tokens = collect_tokens(input);
718        assert_eq!(tokens, vec![Token::Text("Hello, world!".to_string())]);
719    }
720
721    #[test]
722    fn test_script_data_less_than_is_literal() {
723        // A lone `<` inside a <script> body (e.g. `'di<v'` in a JS string) must
724        // stay as text; only `</` opens a tag.
725        let input = r#"<script>test('di<v');</script>"#;
726        let tokens = collect_tokens(input);
727        assert_eq!(
728            tokens,
729            vec![
730                Token::StartTag {
731                    name: "script".to_string(),
732                    attributes: vec![],
733                    self_closing: false
734                },
735                Token::Text("test('di<v');".to_string()),
736                Token::EndTag {
737                    name: "script".to_string()
738                }
739            ]
740        );
741    }
742
743    #[test]
744    fn test_simple_tag() {
745        let input = "<div></div>";
746        let tokens = collect_tokens(input);
747        assert_eq!(
748            tokens,
749            vec![
750                Token::StartTag {
751                    name: "div".to_string(),
752                    attributes: vec![],
753                    self_closing: false
754                },
755                Token::EndTag {
756                    name: "div".to_string()
757                }
758            ]
759        );
760    }
761
762    #[test]
763    fn test_tag_with_attributes() {
764        let input = r#"<a href="https://example.com" target='_blank'>Link</a>"#;
765        let tokens = collect_tokens(input);
766        assert_eq!(
767            tokens,
768            vec![
769                Token::StartTag {
770                    name: "a".to_string(),
771                    attributes: vec![
772                        Attribute {
773                            name: "href".to_string(),
774                            value: "https://example.com".to_string()
775                        },
776                        Attribute {
777                            name: "target".to_string(),
778                            value: "_blank".to_string()
779                        },
780                    ],
781                    self_closing: false
782                },
783                Token::Text("Link".to_string()),
784                Token::EndTag {
785                    name: "a".to_string()
786                }
787            ]
788        );
789    }
790
791    #[test]
792    fn test_boolean_attributes() {
793        let input = r#"<script async defer src="script.js"></script>"#;
794        let tokens = collect_tokens(input);
795        assert_eq!(
796            tokens,
797            vec![
798                Token::StartTag {
799                    name: "script".to_string(),
800                    attributes: vec![
801                        Attribute {
802                            name: "async".to_string(),
803                            value: String::new(),
804                        },
805                        Attribute {
806                            name: "defer".to_string(),
807                            value: String::new(),
808                        },
809                        Attribute {
810                            name: "src".to_string(),
811                            value: "script.js".to_string(),
812                        },
813                    ],
814                    self_closing: false,
815                },
816                Token::EndTag {
817                    name: "script".to_string(),
818                },
819            ]
820        );
821    }
822
823    #[test]
824    fn test_self_closing_tag() {
825        let input = "<img src='image.png'/>";
826        let tokens = collect_tokens(input);
827        assert_eq!(
828            tokens,
829            vec![Token::StartTag {
830                name: "img".to_string(),
831                attributes: vec![Attribute {
832                    name: "src".to_string(),
833                    value: "image.png".to_string()
834                }],
835                self_closing: true
836            }]
837        );
838    }
839
840    #[test]
841    fn test_comment() {
842        let input = "<!-- This is a comment -->";
843        let tokens = collect_tokens(input);
844        assert_eq!(
845            tokens,
846            vec![Token::Comment(" This is a comment ".to_string())]
847        );
848    }
849
850    #[test]
851    fn test_doctype() {
852        let input = "<!DOCTYPE html>";
853        let tokens = collect_tokens(input);
854        assert_eq!(
855            tokens,
856            vec![Token::Doctype {
857                name: Some("html".to_string()),
858                public_id: None,
859                system_id: None,
860                force_quirks: false
861            }]
862        );
863    }
864
865    #[test]
866    fn doctype_public_keyword_is_detected_case_insensitively() {
867        // A miss would leave the keyword appended to the name instead.
868        let tokens = collect_tokens(r#"<!DOCTYPE html PuBlIc "x">"#);
869        match &tokens[..] {
870            [
871                Token::Doctype {
872                    name, public_id, ..
873                },
874            ] => {
875                assert_eq!(name.as_deref(), Some("html"));
876                assert!(public_id.as_deref().is_some_and(|id| id.contains('x')));
877            }
878            other => panic!("unexpected tokens: {other:?}"),
879        }
880    }
881
882    #[test]
883    fn peek_starts_with_ascii_ci_avoids_boundary_panics() {
884        let mut tokenizer = Tokenizer::new("html sYsTeM");
885        tokenizer.pos = 5;
886        assert!(tokenizer.peek_starts_with_ascii_ci("system"));
887        assert!(!tokenizer.peek_starts_with_ascii_ci("public"));
888
889        // A non-ASCII input at the boundary must not panic.
890        let tokenizer = Tokenizer::new("é");
891        assert!(!tokenizer.peek_starts_with_ascii_ci("public"));
892    }
893
894    #[test]
895    fn test_escape_entity() {
896        let input = "Hello &amp; goodbye";
897        let tokens = collect_tokens(input);
898        assert_eq!(tokens, vec![Token::Text("Hello & goodbye".to_string())]);
899    }
900
901    #[test]
902    fn attribute_values_decode_character_references() {
903        let input = r#"<img src="https://x.example/a.png?w=1&amp;h=2&amp;s=&quot;q&quot;" alt='it&#39;s &apos;x&apos;'>"#;
904        let tokens = collect_tokens(input);
905        assert_eq!(
906            tokens,
907            vec![Token::StartTag {
908                name: "img".to_string(),
909                attributes: vec![
910                    Attribute {
911                        name: "src".to_string(),
912                        value: "https://x.example/a.png?w=1&h=2&s=\"q\"".to_string(),
913                    },
914                    Attribute {
915                        name: "alt".to_string(),
916                        value: "it's 'x'".to_string(),
917                    },
918                ],
919                self_closing: false,
920            }]
921        );
922    }
923
924    #[test]
925    fn attribute_entities_close_normally_without_a_semicolon() {
926        // `&amp` with no `;` before the closing quote must flush literally and
927        // still let the quote terminate the attribute.
928        let input = r#"<a href="?a=1&amp">link</a>"#;
929        let tokens = collect_tokens(input);
930        assert_eq!(
931            tokens[0],
932            Token::StartTag {
933                name: "a".to_string(),
934                attributes: vec![Attribute {
935                    name: "href".to_string(),
936                    value: "?a=1&amp".to_string(),
937                }],
938                self_closing: false,
939            }
940        );
941        assert_eq!(tokens[1], Token::Text("link".to_string()));
942    }
943
944    #[test]
945    fn attribute_entity_at_start_of_unquoted_value() {
946        let input = r#"<a href=&amp;x>link</a>"#;
947        let tokens = collect_tokens(input);
948        assert_eq!(
949            tokens[0],
950            Token::StartTag {
951                name: "a".to_string(),
952                attributes: vec![Attribute {
953                    name: "href".to_string(),
954                    value: "&x".to_string(),
955                }],
956                self_closing: false,
957            }
958        );
959    }
960
961    #[test]
962    fn bare_ampersand_in_attribute_is_kept_literal() {
963        // `&` that does not begin a valid reference (e.g. the separator in a
964        // query string written raw) stays as-is.
965        let input = r#"<img src="https://x.example/a.png?a=1&b=2">"#;
966        let tokens = collect_tokens(input);
967        assert_eq!(
968            tokens[0],
969            Token::StartTag {
970                name: "img".to_string(),
971                attributes: vec![Attribute {
972                    name: "src".to_string(),
973                    value: "https://x.example/a.png?a=1&b=2".to_string(),
974                }],
975                self_closing: false,
976            }
977        );
978    }
979
980    /// Entities must decode regardless of how much text precedes them: the
981    /// tokenizer's internal buffer also carries the preceding text, so a
982    /// MAX_ENTITY_NAME_LEN check against the unprefixed buffer used to fire
983    /// mid-entity and mangle it (e.g. `&quot;` → `&uot;`).
984    #[test]
985    fn entity_after_long_text_still_decodes() {
986        for n in [26, 27, 30, 50, 100, 400] {
987            let input = format!("{} &quot;Q&quot; and &#39;9&#39; end", "x".repeat(n));
988            let tokens = collect_tokens(&input);
989            assert_eq!(
990                tokens,
991                vec![Token::Text(format!("{} \"Q\" and '9' end", "x".repeat(n)))],
992                "failed with {n} leading chars"
993            );
994        }
995    }
996
997    #[test]
998    fn test_nested_tags() {
999        let input = "<div><span>Text</span></div>";
1000        let tokens = collect_tokens(input);
1001        assert_eq!(
1002            tokens,
1003            vec![
1004                Token::StartTag {
1005                    name: "div".to_string(),
1006                    attributes: vec![],
1007                    self_closing: false
1008                },
1009                Token::StartTag {
1010                    name: "span".to_string(),
1011                    attributes: vec![],
1012                    self_closing: false
1013                },
1014                Token::Text("Text".to_string()),
1015                Token::EndTag {
1016                    name: "span".to_string()
1017                },
1018                Token::EndTag {
1019                    name: "div".to_string()
1020                },
1021            ]
1022        );
1023    }
1024}
orinium_browser/engine/html/tokenizer.rs

orinium_browser/engine/html/
tokenizer.rs
