orinium_browser/engine/html/
tokenizer.rs

1//! HTML tokenizer. Converts raw HTML input into token stream.
2
3use super::util::{MAX_ENTITY_NAME_LEN, decode_entity};
4
5/// Represents a single HTML attribute
6#[derive(Debug, Clone, PartialEq)]
7pub struct Attribute {
8    pub name: String,
9    pub value: String,
10}
11
12/// HTML tokens emitted by the tokenizer
13#[derive(Debug, Clone, PartialEq)]
14pub enum Token {
15    Doctype {
16        name: Option<String>,
17        public_id: Option<String>,
18        system_id: Option<String>,
19        force_quirks: bool,
20    },
21    StartTag {
22        name: String,
23        attributes: Vec<Attribute>,
24        self_closing: bool,
25    },
26    EndTag {
27        name: String,
28    },
29    Comment(String),
30    Text(String),
31}
32
33/// Represents the internal state of the tokenizer
34#[derive(Debug, PartialEq)]
35pub enum TokenizerState {
36    Data,
37    ScriptData,
38    StyleData,
39    EscapeDecoding,
40    TagOpen,
41    EndTagOpen,
42    TagName,
43    BeforeAttributeName,
44    AttributeName,
45    AfterAttributeName,
46    BeforeAttributeValue,
47    AttributeValueDoubleQuoted,
48    AttributeValueSingleQuoted,
49    AttributeValueUnquoted,
50    SelfClosingStartTag,
51    CommentStartDash,
52    Comment,
53    CommentEndDash,
54    CommentEnd,
55    BogusComment,
56    Doctype,
57    DoctypeName,
58    BeforeDoctypePublicId,
59    DoctypePublicIdWithSingleQuote,
60    DoctypePublicIdWithDoubleQuote,
61    AfterDoctypePublicId,
62    DoctypeSystemId,
63    BogusDoctype,
64}
65
66impl TokenizerState {
67    /// Returns true if the current state is a doctype-related state
68    fn is_doctype(&self) -> bool {
69        matches!(
70            self,
71            TokenizerState::Doctype
72                | TokenizerState::DoctypeName
73                | TokenizerState::BeforeDoctypePublicId
74                | TokenizerState::DoctypePublicIdWithSingleQuote
75                | TokenizerState::DoctypePublicIdWithDoubleQuote
76                | TokenizerState::AfterDoctypePublicId
77                | TokenizerState::DoctypeSystemId
78                | TokenizerState::BogusDoctype
79        )
80    }
81
82    /// Returns true if the current state is a comment-related state
83    fn is_comment(&self) -> bool {
84        matches!(
85            self,
86            TokenizerState::Comment
87                | TokenizerState::CommentStartDash
88                | TokenizerState::CommentEndDash
89                | TokenizerState::CommentEnd
90                | TokenizerState::BogusComment
91        )
92    }
93}
94
95/// HTML tokenizer implementation
96pub struct Tokenizer<'a> {
97    input: &'a str,
98    pos: usize,
99    token: Option<Token>,
100    state: TokenizerState,
101    current_token: Option<Token>,
102    current_attribute: Option<Attribute>,
103    buffer: String,
104}
105
106impl<'a> Tokenizer<'a> {
107    /// Creates a new tokenizer for the given input
108    pub fn new(input: &'a str) -> Self {
109        Self {
110            input,
111            pos: 0,
112            token: None,
113            state: TokenizerState::Data,
114            current_token: None,
115            current_attribute: None,
116            buffer: String::new(),
117        }
118    }
119
120    /// Returns the next character from input and advances the position
121    fn next_char(&mut self) -> Option<char> {
122        if self.pos >= self.input.len() {
123            None
124        } else {
125            let c = self.input[self.pos..].chars().next().unwrap();
126            self.pos += c.len_utf8();
127            Some(c)
128        }
129    }
130
131    /// Emits the current token and clears the buffer
132    fn commit_token(&mut self) {
133        self.token = self.current_token.take();
134        self.buffer.clear();
135    }
136
137    /// Pushes the current attribute to the start tag if exists
138    fn push_current_attribute(&mut self) {
139        if let (Some(attr), Some(Token::StartTag { attributes, .. })) =
140            (self.current_attribute.take(), &mut self.current_token)
141        {
142            attributes.push(attr);
143        }
144    }
145
146    fn handle_special_tag_state_transition(&mut self, token: &Token) {
147        if let Token::StartTag { name, .. } = token {
148            // Switch to ScriptData or StyleData state if we encounter <script> or <style> start tags
149            match name.to_lowercase().as_str() {
150                "script" => self.state = TokenizerState::ScriptData,
151                "style" => self.state = TokenizerState::StyleData,
152                _ => self.state = TokenizerState::Data,
153            }
154        } else {
155            // Do nothing. We only switch to ScriptData or StyleData on StartTag, and we return to Data on EndTag.
156        }
157    }
158
159    /// Debug log for emitted tokens
160    #[inline(always)]
161    fn debug_emit(&self, _token: &Token) {
162        #[cfg(debug_assertions)]
163        match _token {
164            Token::StartTag { name, .. } => {
165                log::debug!(target:"HtmlTokenizer::EmitToken::TagStart", "Emitting token: {name}, Pos: {}", self.pos)
166            }
167            Token::EndTag { name } => {
168                log::debug!(target:"HtmlTokenizer::EmitToken::TagEnd", "Emitting token: {name}, Pos: {}", self.pos)
169            }
170            Token::Comment(comment) => {
171                log::debug!(target:"HtmlTokenizer::EmitToken::Comment", "Emitting token: {}, Pos: {}", comment, self.pos)
172            }
173            Token::Text(text) => {
174                log::debug!(target:"HtmlTokenizer::EmitToken::Text", "Emitting token: `{text}`, Pos: {}", self.pos)
175            }
176            _ => {}
177        }
178    }
179
180    /// Returns the next token if available
181    pub fn next_token(&mut self) -> Option<Token> {
182        while let Some(c) = self.next_char() {
183            log::debug!(target:"HtmlTokenizer::Char", "State: {:?}, Char: '{}'", self.state, c);
184
185            match self.state {
186                TokenizerState::Data | TokenizerState::StyleData | TokenizerState::ScriptData => {
187                    self.state_data(c)
188                }
189                TokenizerState::EscapeDecoding => self.state_escape_decoding(c),
190                _ if self.state.is_doctype() => self.state_doctype(c),
191                TokenizerState::TagOpen => self.state_tag_open(c),
192                TokenizerState::TagName => self.state_tag_name(c),
193                TokenizerState::BeforeAttributeName => self.state_before_attribute_name(c),
194                TokenizerState::AttributeName => self.state_attribute_name(c),
195                TokenizerState::BeforeAttributeValue => self.state_before_attribute_value(c),
196                TokenizerState::AttributeValueDoubleQuoted
197                | TokenizerState::AttributeValueSingleQuoted => {
198                    self.state_attribute_value_quoted(c)
199                }
200                TokenizerState::AfterAttributeName => self.state_after_attribute_name(c),
201                TokenizerState::AttributeValueUnquoted => self.state_attribute_value_unquoted(c),
202                TokenizerState::SelfClosingStartTag => self.state_self_closing_start_tag(c),
203                TokenizerState::EndTagOpen => self.state_end_tag_open(c),
204                _ if self.state.is_comment() => self.state_comment(c),
205                _ => {
206                    log::error!(target:"HtmlTokenizer::State", "Unimplemented state: {:?}, returning to Data state", self.state);
207                    self.state = TokenizerState::Data;
208                }
209            }
210
211            if let Some(token) = self.token.take() {
212                self.debug_emit(&token);
213                self.handle_special_tag_state_transition(&token);
214                return Some(token);
215            }
216        }
217
218        // End of input: commit remaining current_token if exists
219        if self.current_token.is_some() {
220            self.commit_token();
221            return self.token.take();
222        }
223
224        // Emit BogusComment if input ended while in comment
225        if self.state.is_comment() {
226            self.state = TokenizerState::BogusComment;
227            self.commit_token();
228            return self.token.take();
229        }
230
231        None
232    }
233
234    // --- State handlers ---
235    fn state_data(&mut self, c: char) {
236        match c {
237            '<' => {
238                // In raw-text states (<script>, <style>) a `<` only opens a tag
239                // when it is `</` (the closing tag); any other `<` is literal
240                // text (e.g. the `<` inside a JS string like 'di<v').
241                let raw_text = matches!(
242                    self.state,
243                    TokenizerState::ScriptData | TokenizerState::StyleData
244                );
245                if raw_text && !self.input[self.pos..].starts_with('/') {
246                    self.buffer.push('<');
247                    match &mut self.current_token {
248                        Some(Token::Text(text)) => text.push('<'),
249                        _ => self.current_token = Some(Token::Text("<".to_string())),
250                    }
251                } else {
252                    self.commit_token();
253                    self.state = TokenizerState::TagOpen;
254                }
255            }
256            // Handle escape entities only in Data. Not in ScriptData, and StyleData states.
257            '&' if self.state == TokenizerState::Data => {
258                self.buffer.push('&');
259                self.state = TokenizerState::EscapeDecoding;
260            }
261            _ => {
262                self.buffer.push(c);
263                match &mut self.current_token {
264                    Some(Token::Text(text)) => text.push(c),
265                    _ => self.current_token = Some(Token::Text(c.to_string())),
266                }
267            }
268        }
269    }
270
271    fn state_escape_decoding(&mut self, c: char) {
272        if c == ';' {
273            let mut iter = self.buffer.rsplitn(2, '&');
274            let entity = iter.next().unwrap_or("");
275
276            let decoded = decode_entity(entity).unwrap_or_else(|| format!("&{};", entity));
277
278            match &mut self.current_token {
279                Some(Token::Text(text)) => text.push_str(&decoded),
280                _ => self.current_token = Some(Token::Text(decoded)),
281            }
282
283            self.buffer.clear();
284            self.state = TokenizerState::Data;
285        } else if self.buffer.len() > MAX_ENTITY_NAME_LEN || self.input[self.pos..].starts_with('<')
286        {
287            let mut iter = self.buffer.rsplitn(2, '&');
288            let entity = iter.next().unwrap_or("");
289            let decoded = format!("&{}", entity);
290
291            match &mut self.current_token {
292                Some(Token::Text(text)) => text.push_str(&decoded),
293                _ => self.current_token = Some(Token::Text(decoded)),
294            }
295
296            self.buffer.clear();
297            self.state = TokenizerState::Data;
298        } else {
299            self.buffer.push(c);
300        }
301    }
302
303    fn state_tag_open(&mut self, c: char) {
304        match c {
305            '/' => self.state = TokenizerState::EndTagOpen,
306            '!' => {
307                if self.input[self.pos..].starts_with('-') {
308                    self.pos += 1;
309                    self.state = TokenizerState::CommentStartDash;
310                } else if self.input[self.pos..].to_lowercase().starts_with("doctype") {
311                    self.pos += 7;
312                    self.state = TokenizerState::Doctype;
313                    self.current_token = Some(Token::Doctype {
314                        name: None,
315                        public_id: None,
316                        system_id: None,
317                        force_quirks: false,
318                    });
319                } else {
320                    self.state = TokenizerState::BogusComment;
321                }
322            }
323            c if c.is_ascii_alphabetic() => {
324                self.state = TokenizerState::TagName;
325                self.buffer.push(c);
326                self.current_token = Some(Token::StartTag {
327                    name: c.to_string(),
328                    attributes: Vec::new(),
329                    self_closing: false,
330                });
331            }
332            _ => {
333                self.buffer.push('<');
334                self.buffer.push(c);
335                match &mut self.current_token {
336                    Some(Token::Text(text)) => {
337                        text.push('<');
338                        text.push(c);
339                    }
340                    _ => self.current_token = Some(Token::Text(format!("<{c}"))),
341                }
342                self.state = TokenizerState::Data;
343            }
344        }
345    }
346
347    fn state_tag_name(&mut self, c: char) {
348        match c {
349            c if c.is_whitespace() => self.state = TokenizerState::BeforeAttributeName,
350            '/' => self.state = TokenizerState::SelfClosingStartTag,
351            '>' => {
352                self.commit_token();
353                self.state = TokenizerState::Data;
354            }
355            c if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == ':' => {
356                self.buffer.push(c);
357                match &mut self.current_token {
358                    Some(Token::StartTag { name, .. }) => name.push(c),
359                    Some(Token::EndTag { name }) => name.push(c),
360                    _ => {}
361                }
362            }
363            _ => {
364                self.commit_token();
365                self.state = TokenizerState::Data;
366            }
367        }
368    }
369
370    fn state_before_attribute_name(&mut self, c: char) {
371        match c {
372            c if c.is_whitespace() => {}
373            '/' => self.state = TokenizerState::SelfClosingStartTag,
374            '>' => {
375                self.commit_token();
376                self.state = TokenizerState::Data;
377            }
378            c if c.is_ascii_alphanumeric() => {
379                self.state = TokenizerState::AttributeName;
380                self.buffer.push(c);
381                self.current_attribute = Some(Attribute {
382                    name: c.to_string(),
383                    value: String::new(),
384                });
385            }
386            _ => {}
387        }
388    }
389
390    fn state_attribute_name(&mut self, c: char) {
391        match c {
392            c if c.is_whitespace() => self.state = TokenizerState::AfterAttributeName,
393            '=' => self.state = TokenizerState::BeforeAttributeValue,
394            '/' => {
395                self.push_current_attribute();
396                self.state = TokenizerState::SelfClosingStartTag;
397            }
398            '>' => {
399                self.push_current_attribute();
400                self.commit_token();
401                self.state = TokenizerState::Data;
402            }
403            c if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == ':' => {
404                self.buffer.push(c);
405                if let Some(attr) = &mut self.current_attribute {
406                    attr.name.push(c);
407                }
408            }
409            _ => {}
410        }
411    }
412
413    fn state_before_attribute_value(&mut self, c: char) {
414        match c {
415            c if c.is_whitespace() => {}
416            '"' => self.state = TokenizerState::AttributeValueDoubleQuoted,
417            '\'' => self.state = TokenizerState::AttributeValueSingleQuoted,
418            '>' => {
419                self.push_current_attribute();
420                self.commit_token();
421                self.state = TokenizerState::Data;
422            }
423            _ => {
424                self.state = TokenizerState::AttributeValueUnquoted;
425                if let Some(attr) = &mut self.current_attribute {
426                    attr.value.push(c);
427                }
428            }
429        }
430    }
431
432    fn state_attribute_value_quoted(&mut self, c: char) {
433        match (&self.state, c) {
434            (&TokenizerState::AttributeValueDoubleQuoted, '"')
435            | (&TokenizerState::AttributeValueSingleQuoted, '\'') => {
436                self.push_current_attribute();
437                self.state = TokenizerState::AfterAttributeName;
438            }
439            _ => {
440                if let Some(attr) = &mut self.current_attribute {
441                    attr.value.push(c);
442                }
443            }
444        }
445    }
446
447    fn state_after_attribute_name(&mut self, c: char) {
448        match c {
449            c if c.is_whitespace() => {}
450            '/' => {
451                self.push_current_attribute();
452                self.state = TokenizerState::SelfClosingStartTag;
453            }
454            '=' => self.state = TokenizerState::BeforeAttributeValue,
455            '>' => {
456                self.push_current_attribute();
457                self.commit_token();
458                self.state = TokenizerState::Data;
459            }
460            c if c.is_ascii_alphanumeric() => {
461                self.push_current_attribute();
462                self.state = TokenizerState::AttributeName;
463                self.buffer.push(c);
464                self.current_attribute = Some(Attribute {
465                    name: c.to_string(),
466                    value: String::new(),
467                });
468            }
469            _ => {}
470        }
471    }
472
473    fn state_attribute_value_unquoted(&mut self, c: char) {
474        match c {
475            c if c.is_whitespace() => {
476                self.push_current_attribute();
477                self.state = TokenizerState::BeforeAttributeName;
478            }
479            '>' => {
480                self.push_current_attribute();
481                self.commit_token();
482                self.state = TokenizerState::Data;
483            }
484            _ => {
485                if let Some(attr) = &mut self.current_attribute {
486                    attr.value.push(c);
487                }
488            }
489        }
490    }
491
492    fn state_self_closing_start_tag(&mut self, c: char) {
493        match c {
494            '>' => {
495                if let Some(Token::StartTag { self_closing, .. }) = &mut self.current_token {
496                    *self_closing = true;
497                }
498                self.commit_token();
499                self.state = TokenizerState::Data;
500            }
501            _ => self.state = TokenizerState::Data,
502        }
503    }
504
505    fn state_end_tag_open(&mut self, c: char) {
506        match c {
507            c if c.is_ascii_alphabetic() => {
508                self.state = TokenizerState::TagName;
509                self.buffer.push(c);
510                self.current_token = Some(Token::EndTag {
511                    name: c.to_string(),
512                });
513            }
514            _ => self.state = TokenizerState::Data,
515        }
516    }
517
518    fn state_comment(&mut self, c: char) {
519        match self.state {
520            TokenizerState::CommentStartDash => {
521                if c == '-' {
522                    self.state = TokenizerState::Comment;
523                    self.current_token = Some(Token::Comment(String::new()));
524                } else {
525                    self.state = TokenizerState::BogusComment;
526                }
527            }
528            TokenizerState::Comment => {
529                if c == '-' {
530                    self.state = TokenizerState::CommentEndDash;
531                } else if let Some(Token::Comment(comment)) = &mut self.current_token {
532                    comment.push(c);
533                }
534            }
535            TokenizerState::CommentEndDash => {
536                if c == '-' {
537                    self.state = TokenizerState::CommentEnd;
538                } else {
539                    self.state = TokenizerState::Comment;
540                    if let Some(Token::Comment(comment)) = &mut self.current_token {
541                        comment.push('-');
542                        comment.push(c);
543                    }
544                }
545            }
546            TokenizerState::CommentEnd => {
547                if c == '>' {
548                    self.commit_token();
549                    self.state = TokenizerState::Data;
550                } else {
551                    self.state = TokenizerState::Comment;
552                    if let Some(Token::Comment(comment)) = &mut self.current_token {
553                        comment.push_str("--");
554                        comment.push(c);
555                    }
556                }
557            }
558            _ => {}
559        }
560    }
561
562    fn state_doctype(&mut self, c: char) {
563        match c {
564            c if c.is_whitespace() => match self.state {
565                TokenizerState::Doctype => self.state = TokenizerState::DoctypeName,
566                TokenizerState::DoctypeName
567                    if (self.input[self.pos..].to_lowercase().starts_with("public")
568                        || self.input[self.pos..].to_lowercase().starts_with("system")) =>
569                {
570                    self.pos += 6;
571                    self.state = TokenizerState::BeforeDoctypePublicId;
572                }
573                TokenizerState::AfterDoctypePublicId => {
574                    self.state = TokenizerState::DoctypeSystemId;
575                }
576                _ => {}
577            },
578            '>' => {
579                if let Some(Token::Doctype { force_quirks, .. }) = &mut self.current_token
580                    && self.state == TokenizerState::BogusDoctype
581                {
582                    *force_quirks = true;
583                }
584                self.commit_token();
585                self.state = TokenizerState::Data;
586            }
587            _ => {
588                self.buffer.push(c);
589                match self.state {
590                    TokenizerState::Doctype => self.state = TokenizerState::BogusDoctype,
591                    TokenizerState::DoctypeName => {
592                        if let Some(Token::Doctype { name, .. }) = &mut self.current_token {
593                            if name.is_none() {
594                                *name = Some(c.to_string());
595                            } else if let Some(n) = name {
596                                n.push(c);
597                            }
598                        }
599                    }
600                    TokenizerState::BeforeDoctypePublicId => {
601                        match c {
602                            '"' => self.state = TokenizerState::DoctypePublicIdWithDoubleQuote,
603                            '\'' => self.state = TokenizerState::DoctypePublicIdWithSingleQuote,
604                            _ if c.is_whitespace() => {}
605                            _ => self.state = TokenizerState::BogusDoctype,
606                        }
607                        if let Some(Token::Doctype { public_id, .. }) = &mut self.current_token {
608                            *public_id = Some(c.to_string());
609                        }
610                    }
611                    TokenizerState::DoctypePublicIdWithSingleQuote
612                    | TokenizerState::DoctypePublicIdWithDoubleQuote => {
613                        if let Some(Token::Doctype { public_id, .. }) = &mut self.current_token
614                            && let Some(pid) = public_id
615                        {
616                            pid.push(c);
617                        }
618                        if (self.state == TokenizerState::DoctypePublicIdWithSingleQuote
619                            && c == '\'')
620                            || (self.state == TokenizerState::DoctypePublicIdWithDoubleQuote
621                                && c == '"')
622                        {
623                            self.state = TokenizerState::AfterDoctypePublicId;
624                        }
625                    }
626                    TokenizerState::DoctypeSystemId => {
627                        if let Some(Token::Doctype { system_id, .. }) = &mut self.current_token {
628                            if system_id.is_none() {
629                                *system_id = Some(c.to_string());
630                            } else if let Some(sid) = system_id {
631                                sid.push(c);
632                            }
633                        }
634                    }
635                    _ => {}
636                }
637            }
638        }
639    }
640}
641
642#[cfg(test)]
643mod tests {
644    use super::*;
645
646    fn collect_tokens(input: &str) -> Vec<Token> {
647        let mut tokenizer = Tokenizer::new(input);
648        let mut tokens = Vec::new();
649        while let Some(token) = tokenizer.next_token() {
650            tokens.push(token);
651        }
652        tokens
653    }
654
655    #[test]
656    fn test_text_node() {
657        let input = "Hello, world!";
658        let tokens = collect_tokens(input);
659        assert_eq!(tokens, vec![Token::Text("Hello, world!".to_string())]);
660    }
661
662    #[test]
663    fn test_script_data_less_than_is_literal() {
664        // A lone `<` inside a <script> body (e.g. `'di<v'` in a JS string) must
665        // stay as text; only `</` opens a tag.
666        let input = r#"<script>test('di<v');</script>"#;
667        let tokens = collect_tokens(input);
668        assert_eq!(
669            tokens,
670            vec![
671                Token::StartTag {
672                    name: "script".to_string(),
673                    attributes: vec![],
674                    self_closing: false
675                },
676                Token::Text("test('di<v');".to_string()),
677                Token::EndTag {
678                    name: "script".to_string()
679                }
680            ]
681        );
682    }
683
684    #[test]
685    fn test_simple_tag() {
686        let input = "<div></div>";
687        let tokens = collect_tokens(input);
688        assert_eq!(
689            tokens,
690            vec![
691                Token::StartTag {
692                    name: "div".to_string(),
693                    attributes: vec![],
694                    self_closing: false
695                },
696                Token::EndTag {
697                    name: "div".to_string()
698                }
699            ]
700        );
701    }
702
703    #[test]
704    fn test_tag_with_attributes() {
705        let input = r#"<a href="https://example.com" target='_blank'>Link</a>"#;
706        let tokens = collect_tokens(input);
707        assert_eq!(
708            tokens,
709            vec![
710                Token::StartTag {
711                    name: "a".to_string(),
712                    attributes: vec![
713                        Attribute {
714                            name: "href".to_string(),
715                            value: "https://example.com".to_string()
716                        },
717                        Attribute {
718                            name: "target".to_string(),
719                            value: "_blank".to_string()
720                        },
721                    ],
722                    self_closing: false
723                },
724                Token::Text("Link".to_string()),
725                Token::EndTag {
726                    name: "a".to_string()
727                }
728            ]
729        );
730    }
731
732    #[test]
733    fn test_boolean_attributes() {
734        let input = r#"<script async defer src="script.js"></script>"#;
735        let tokens = collect_tokens(input);
736        assert_eq!(
737            tokens,
738            vec![
739                Token::StartTag {
740                    name: "script".to_string(),
741                    attributes: vec![
742                        Attribute {
743                            name: "async".to_string(),
744                            value: String::new(),
745                        },
746                        Attribute {
747                            name: "defer".to_string(),
748                            value: String::new(),
749                        },
750                        Attribute {
751                            name: "src".to_string(),
752                            value: "script.js".to_string(),
753                        },
754                    ],
755                    self_closing: false,
756                },
757                Token::EndTag {
758                    name: "script".to_string(),
759                },
760            ]
761        );
762    }
763
764    #[test]
765    fn test_self_closing_tag() {
766        let input = "<img src='image.png'/>";
767        let tokens = collect_tokens(input);
768        assert_eq!(
769            tokens,
770            vec![Token::StartTag {
771                name: "img".to_string(),
772                attributes: vec![Attribute {
773                    name: "src".to_string(),
774                    value: "image.png".to_string()
775                }],
776                self_closing: true
777            }]
778        );
779    }
780
781    #[test]
782    fn test_comment() {
783        let input = "<!-- This is a comment -->";
784        let tokens = collect_tokens(input);
785        assert_eq!(
786            tokens,
787            vec![Token::Comment(" This is a comment ".to_string())]
788        );
789    }
790
791    #[test]
792    fn test_doctype() {
793        let input = "<!DOCTYPE html>";
794        let tokens = collect_tokens(input);
795        assert_eq!(
796            tokens,
797            vec![Token::Doctype {
798                name: Some("html".to_string()),
799                public_id: None,
800                system_id: None,
801                force_quirks: false
802            }]
803        );
804    }
805
806    #[test]
807    fn test_escape_entity() {
808        let input = "Hello &amp; goodbye";
809        let tokens = collect_tokens(input);
810        assert_eq!(tokens, vec![Token::Text("Hello & goodbye".to_string())]);
811    }
812
813    #[test]
814    fn test_nested_tags() {
815        let input = "<div><span>Text</span></div>";
816        let tokens = collect_tokens(input);
817        assert_eq!(
818            tokens,
819            vec![
820                Token::StartTag {
821                    name: "div".to_string(),
822                    attributes: vec![],
823                    self_closing: false
824                },
825                Token::StartTag {
826                    name: "span".to_string(),
827                    attributes: vec![],
828                    self_closing: false
829                },
830                Token::Text("Text".to_string()),
831                Token::EndTag {
832                    name: "span".to_string()
833                },
834                Token::EndTag {
835                    name: "div".to_string()
836                },
837            ]
838        );
839    }
840}
orinium_browser/engine/html/tokenizer.rs

orinium_browser/engine/html/
tokenizer.rs
