//! `angular-html-parser` 10.12: `ml_parser/lexer.ts`, without the syntax that Prettier doesn't enable. //! //! The text only has `\n` line breaks. A token's contents are always substrings of the text, so a token stores spans. use super::error::{ErrorKind, SyntaxError}; use crate::strings; #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum TokenType { TagOpenStart, TagOpenEnd, TagOpenEndVoid, TagClose, IncompleteTagOpen, Text, EscapableRawText, RawText, Interpolation, EncodedEntity, CommentStart, CommentEnd, InElementComment, CdataStart, CdataEnd, AttrName, AttrQuote, AttrValueText, AttrValueInterpolation, DocTypeStart, DocTypeEnd, ExpansionFormStart, ExpansionCaseValue, ExpansionCaseExpStart, ExpansionCaseExpEnd, ExpansionFormEnd, BlockOpenStart, BlockOpenEnd, BlockClose, BlockParameter, IncompleteBlockOpen, LetStart, LetValue, LetEnd, IncompleteLet, Eof, } #[derive(Debug, Copy, Clone, Default, PartialEq, Eq)] pub(crate) struct Span { pub(crate) start: u32, pub(crate) end: u32, } impl Span { pub(crate) fn new(start: u32, end: u32) -> Span { Span { start, end } } pub(crate) fn of(self, text: &[u8]) -> &[u8] { text.get(self.start as usize..self.end as usize).unwrap_or_default() } pub(crate) fn len(self) -> u32 { self.end.saturating_sub(self.start) } } /// `token.parts`, where that isn't simply the token's whole text. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum Parts { /// No parts, or just the token's text. None, /// The prefix and the name of a tag or an attribute. Name(Span, Span), /// The name of a block or a declaration, or the value of a case. Value(Span), /// `else if`, with any amount of whitespace in between. ElseIf, /// `default never`, with any amount of whitespace in between. DefaultNever, /// The contents of a comment in a start tag, and whether the comment ends at the end of the line. Comment(Span, bool), } #[derive(Debug, Copy, Clone)] pub(crate) struct Token { pub(crate) kind: TokenType, pub(crate) span: Span, pub(crate) parts: Parts, } #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub(crate) enum ContentType { RawText, EscapableRawText, ParsableData, } /// An attribute, as passed to the callback that decides the element's content type. pub(crate) struct LexedAttribute<'a> { pub(crate) name: &'a [u8], /// The last piece of text in the value. pub(crate) value: Option<&'a [u8]>, } #[derive(Copy, Clone, Default)] pub(crate) struct Options { /// `tokenizeExpansionForms` and `tokenizeBlocks` pub(crate) tokenize_angular_blocks: bool, pub(crate) tokenize_let: bool, pub(crate) can_self_close: bool, pub(crate) allow_htm_component_closing_tags: bool, pub(crate) allow_start_tag_comments: bool, } /// `getTagContentType(tagName, prefix, hasParent, attrs)`, without the prefix, which is never used. pub(crate) type GetTagContentType<'f> = &'f dyn Fn(&[u8], bool, &[LexedAttribute<'_>]) -> ContentType; /// The exceptions that the lexer throws. enum Fail { /// A `ParseError`, with the start of its span. Parse(u32), /// A `CursorError`: the text ended where more was expected. Cursor(u32), } type Result = std::result::Result; /// The condition that ends a text with interpolations. #[derive(Copy, Clone)] enum End { Text, TagStart, Quote(u32), Name, UnquotedValue, } /// The marker that ends a raw text. #[derive(Copy, Clone)] enum EndMarker<'a> { Str(&'static [u8]), GreaterThan, /// ` bool { (9..=32).contains(&code) || code == NBSP } fn is_not_whitespace(code: u32) -> bool { !is_whitespace(code) || code == 0 } fn is_ascii_letter(code: u32) -> bool { code < 128 && (code as u8).is_ascii_alphabetic() } fn is_digit(code: u32) -> bool { code < 128 && (code as u8).is_ascii_digit() } fn is_new_line(code: u32) -> bool { code == 10 || code == 13 } fn is_quote(code: u32) -> bool { matches!(code, 39 | 34 | 96) } fn is_name_end(code: u32) -> bool { is_whitespace(code) || matches!(code, 62 | 60 | 47 | 39 | 34 | 61 | 0) } /// HTML's "attribute value (unquoted) state". `=`, `<` and quotes are errors there but, like `/`, part of the value. fn is_unquoted_value_end(code: u32) -> bool { matches!(code, 9 | 10 | 12 | 13 | 32 | 62 | 0) } fn is_prefix_end(code: u32) -> bool { !is_ascii_letter(code) && !is_digit(code) } fn is_digit_entity_end(code: u32) -> bool { code == 59 || code == 0 || !(code < 128 && (code as u8).is_ascii_hexdigit()) } fn is_named_entity_end(code: u32) -> bool { code == 59 || code == 0 || !(is_ascii_letter(code) || is_digit(code)) } fn is_block_name_char(code: u32) -> bool { is_ascii_letter(code) || is_digit(code) || code == 95 } fn is_block_parameter_char(code: u32) -> bool { code != 59 && is_not_whitespace(code) } fn is_attribute_terminator(code: u32) -> bool { matches!(code, 47 | 62 | 60 | 0) } /// `/^a[^\S\r\n]+b/` fn is_two_words(text: &[u8], first: &[u8], second: &[u8]) -> bool { let Some(mut rest) = text.strip_prefix(first) else { return false; }; let before = rest.len(); while !matches!(rest.first(), Some(b'\n' | b'\r')) && let len @ 1.. = strings::js_whitespace_len(rest) { rest = &rest[len..]; } rest.len() < before && rest.starts_with(second) } struct Tokenizer<'a, 'f> { /// Where the next NUL is at or after `pos`, or the end of the text. nul: usize, text: &'a [u8], /// `_cursor` pos: usize, get_tag_content_type: GetTagContentType<'f>, options: Options, /// `_currentTokenStart` and `_currentTokenType` current: Option<(u32, TokenType)>, expansion_case_stack: Vec, /// Pairs of prefix and name. full_name_stack: Vec<(Span, Span)>, /// The attributes of the start tag being read. attributes: Vec>, tokens: Vec, errors: Vec>, /// Details of the last `Fail::Parse`: its kind, the span of the name in its message, and `_isInExpansionForm()`. thrown: (ErrorKind, Span, bool), } impl<'a> Tokenizer<'a, '_> { fn peek_at(&self, pos: usize) -> u32 { match self.text.get(pos) { None => 0, Some(&byte) if byte < 128 => u32::from(byte), Some(0xC2) if self.text.get(pos + 1) == Some(&0xA0) => NBSP, Some(_) => OTHER, } } fn peek(&self) -> u32 { self.peek_at(self.pos) } fn advance(&mut self) -> Result<()> { if self.pos >= self.text.len() { return Err(Fail::Cursor(self.pos as u32)); } self.pos += if self.peek() == NBSP { 2 } else { 1 }; Ok(()) } fn span_from(&self, start: usize) -> Span { Span::new(start as u32, self.pos as u32) } fn begin_token(&mut self, kind: TokenType) { self.current = Some((self.pos as u32, kind)); } fn begin_token_at(&mut self, kind: TokenType, start: usize) { self.current = Some((start as u32, kind)); } /// Returns the index of the token. fn end_token_at(&mut self, parts: Parts, end: usize) -> Result { let Some((start, kind)) = self.current.take() else { return Err(self.error_of(ErrorKind::Other, end, Span::new(0, 0))); }; self.tokens.push(Token { kind, span: Span::new(start, end as u32), parts }); Ok(self.tokens.len() - 1) } fn end_token(&mut self, parts: Parts) -> Result { self.end_token_at(parts, self.pos) } /// `_createError(_unexpectedCharacterErrorMsg(this._cursor.peek()), this._cursor.getSpan(start))` fn error(&mut self, start: usize) -> Fail { let rest = Span::new(self.pos as u32, self.text.len() as u32); self.error_of(ErrorKind::UnexpectedCharacter, start, rest) } /// `_createError`. `named` is the span of the name in the message. fn error_of(&mut self, kind: ErrorKind, start: usize, named: Span) -> Fail { self.current = None; self.thrown = (kind, named, self.is_in_expansion_form()); Fail::Parse(start as u32) } /// `handleError` fn handle_error( &mut self, at: u32, (kind, named, is_in_expansion_form): (ErrorKind, Span, bool), ) { self.errors.push(SyntaxError { is_in_expansion_form, ..SyntaxError::new(at, kind, named.of(self.text)) }); } fn attempt_char_code(&mut self, code: u32) -> bool { if self.peek() == code && self.pos < self.text.len() { self.pos += 1; return true; } false } fn require_char_code(&mut self, code: u32) -> Result<()> { match self.attempt_char_code(code) { true => Ok(()), false => Err(self.error(self.pos)), } } fn peek_str(&self, chars: &[u8]) -> bool { self.text.get(self.pos..).is_some_and(|rest| rest.starts_with(chars)) } fn attempt_str(&mut self, chars: &[u8]) -> bool { let is_next = self.peek_str(chars); if is_next { self.pos += chars.len(); } is_next } /// The part that matches is consumed even if the rest doesn't match. fn attempt_str_case_insensitive(&mut self, chars: &[u8]) -> bool { for expected in chars { match self.text.get(self.pos) { Some(byte) if byte.eq_ignore_ascii_case(expected) => self.pos += 1, _ => return false, } } true } fn require_str(&mut self, chars: &[u8]) -> Result<()> { match self.attempt_str(chars) { true => Ok(()), false => Err(self.error(self.pos)), } } fn attempt_char_code_until(&mut self, mut predicate: impl FnMut(u32) -> bool) -> Result<()> { while !predicate(self.peek()) { self.advance()?; } Ok(()) } fn skip_whitespace(&mut self) { while !is_not_whitespace(self.peek()) { self.pos += if self.peek() == NBSP { 2 } else { 1 }; } } fn require_char_code_until( &mut self, predicate: impl FnMut(u32) -> bool, len: usize, ) -> Result<()> { let start = self.pos; self.attempt_char_code_until(predicate)?; if self.pos - start < len { return Err(self.error(start)); } Ok(()) } fn attempt_until_ignore_quotes(&mut self, predicate: impl Fn(u32) -> bool) -> Result<()> { while self.peek() != 0 { let char = self.peek(); if predicate(char) { break; } if is_quote(char) { self.advance()?; loop { let inner = self.peek(); if inner == 92 { self.advance()?; } else if inner == char { break; } self.advance()?; } } self.advance()?; } Ok(()) } fn is_block_start(&self) -> bool { self.peek() == 64 && SUPPORTED_BLOCKS.iter().any(|name| self.peek_str(name)) } fn is_let_start(&self) -> bool { self.peek_str(b"@let") } fn is_in_expansion_case(&self) -> bool { self.expansion_case_stack.last() == Some(&TokenType::ExpansionCaseExpStart) } fn is_in_expansion_form(&self) -> bool { self.expansion_case_stack.last() == Some(&TokenType::ExpansionFormStart) } fn is_expansion_form_start(&self) -> bool { self.peek() == 123 && !self.peek_str(b"{{") } fn is_tag_start(&self) -> bool { self.peek() == 60 && { let code = self.peek_at(self.pos + 1); is_ascii_letter(code) || matches!(code, 47 | 33 | 63) } } fn is_text_end(&self) -> bool { if self.is_tag_start() || self.peek() == 0 { return true; } if !self.options.tokenize_angular_blocks { return false; } if self.is_expansion_form_start() || (self.peek() == 125 && self.is_in_expansion_case()) { return true; } !self.is_in_expansion_case() && !self.is_in_expansion_form() && (self.is_block_start() || self.is_let_start() || self.peek() == 125) } fn is_end(&self, end: End) -> bool { match end { End::Text => self.is_text_end(), End::TagStart => self.is_tag_start(), End::Quote(quote) => self.peek() == quote, End::Name => is_name_end(self.peek()), End::UnquotedValue => is_unquoted_value_end(self.peek()), } } /// Skips to the next character that can end a text terminated by `end`, or start something inside it. fn skip_plain_text(&mut self, end: End) { let rest = self.text.get(self.pos..).unwrap_or_default(); let next = match end { End::Text if self.options.tokenize_angular_blocks => strings::index_of_any(rest, b"<{&\0@}"), End::Text => { if self.nul < self.pos { self.nul = memchr::memchr(0, rest).map_or(self.text.len(), |at| self.pos + at); } let stop = memchr::memchr3(b'<', b'{', b'&', rest).map_or(self.text.len(), |at| self.pos + at); Some(stop.min(self.nul) - self.pos) } End::Quote(quote) => memchr::memchr3(quote as u8, b'{', b'&', rest), End::TagStart | End::Name | End::UnquotedValue => return, }; self.pos += next.unwrap_or(rest.len()); } fn tokenize(&mut self) { while self.peek() != 0 { let start = self.pos; match self.tokenize_one() { // The lexer didn't advance, so it would loop forever. Ok(()) if self.pos == start => { self.handle_error(start as u32, (ErrorKind::Other, Span::new(0, 0), false)); break; } Ok(()) => {} Err(Fail::Parse(at)) => self.handle_error(at, self.thrown), Err(Fail::Cursor(at)) => { self.current = None; let end = Span::new(self.text.len() as u32, self.text.len() as u32); let is_in_expansion_form = self.is_in_expansion_form(); self.handle_error(at, (ErrorKind::UnexpectedCharacter, end, is_in_expansion_form)); } } } self.begin_token(TokenType::Eof); let _ = self.end_token(Parts::None); } fn tokenize_one(&mut self) -> Result<()> { let start = self.pos; let blocks = self.options.tokenize_angular_blocks; if self.attempt_char_code(60) { if self.attempt_char_code(33) { if self.attempt_str(b"[CDATA[") { self.consume_delimited( start, TokenType::CdataStart, TokenType::CdataEnd, EndMarker::Str(b"]]>"), ) } else if self.attempt_str(b"--") { self.consume_delimited( start, TokenType::CommentStart, TokenType::CommentEnd, EndMarker::Str(b"-->"), ) } else if self.attempt_str_case_insensitive(b"doctype") { self.consume_delimited( start, TokenType::DocTypeStart, TokenType::DocTypeEnd, EndMarker::GreaterThan, ) } else { self.consume_delimited( start, TokenType::CommentStart, TokenType::CommentEnd, EndMarker::GreaterThan, ) } } else if self.attempt_char_code(47) { self.consume_tag_close(start) } else if self.peek() == 63 { self.consume_delimited( start, TokenType::CommentStart, TokenType::CommentEnd, EndMarker::GreaterThan, ) } else { self.consume_tag_open(start) } } else if self.options.tokenize_let && self.is_let_start() { self.consume_let_declaration(start) } else if blocks && self.is_block_start() { self.consume_block_start(start) } else if blocks && !self.is_in_expansion_case() && !self.is_in_expansion_form() && self.attempt_char_code(125) { self.begin_token_at(TokenType::BlockClose, start); self.end_token(Parts::None).map(|_| ()) } else if !(blocks && self.tokenize_expansion_form()?) { self .consume_with_interpolation( TokenType::Text, TokenType::Interpolation, End::Text, End::TagStart, ) .map(|_| ()) } else { Ok(()) } } fn block_name(&mut self) -> Result { let mut spaces_in_name_allowed = false; let start = self.pos; self.attempt_char_code_until(|code| { if is_whitespace(code) { return !spaces_in_name_allowed; } if is_block_name_char(code) { spaces_in_name_allowed = true; return false; } true })?; let name = trimmed(self.text, self.span_from(start)); Ok(if is_two_words(name.of(self.text), b"else", b"if") { Parts::ElseIf } else if is_two_words(name.of(self.text), b"default", b"never") { Parts::DefaultNever } else { Parts::Value(name) }) } fn empty_token(&mut self, kind: TokenType) -> Result<()> { self.begin_token(kind); self.end_token(Parts::None).map(|_| ()) } fn consume_block_start(&mut self, start: usize) -> Result<()> { self.require_char_code(64)?; self.begin_token_at(TokenType::BlockOpenStart, start); let name = self.block_name()?; let start_token = self.end_token(name)?; if self.peek() == 40 { self.advance()?; self.consume_block_parameters()?; self.skip_whitespace(); if self.attempt_char_code(41) { self.skip_whitespace(); } else { self.tokens[start_token].kind = TokenType::IncompleteBlockOpen; return Ok(()); } } if name == Parts::DefaultNever && self.attempt_char_code(59) { self.empty_token(TokenType::BlockOpenEnd)?; return self.empty_token(TokenType::BlockClose); } let is_case = matches!(name, Parts::Value(name) if matches!(name.of(self.text), b"case" | b"default")); if self.peek() == 123 { // The token starts after the brace. self.pos += 1; self.empty_token(TokenType::BlockOpenEnd)?; } else if self.is_block_start() && is_case { self.empty_token(TokenType::BlockOpenEnd)?; self.empty_token(TokenType::BlockClose)?; } else { self.tokens[start_token].kind = TokenType::IncompleteBlockOpen; } Ok(()) } fn consume_block_parameters(&mut self) -> Result<()> { self.attempt_char_code_until(is_block_parameter_char)?; while self.peek() != 41 && self.peek() != 0 { self.begin_token(TokenType::BlockParameter); let mut in_quote = None; let mut open_parens = 0u32; while (self.peek() != 59 && self.peek() != 0) || in_quote.is_some() { let char = self.peek(); if char == 92 { self.advance()?; } else if Some(char) == in_quote { in_quote = None; } else if in_quote.is_none() && is_quote(char) { in_quote = Some(char); } else if char == 40 && in_quote.is_none() { open_parens += 1; } else if char == 41 && in_quote.is_none() { if open_parens == 0 { break; } open_parens -= 1; } self.advance()?; } self.end_token(Parts::None)?; self.attempt_char_code_until(is_block_parameter_char)?; } Ok(()) } fn consume_let_declaration(&mut self, start: usize) -> Result<()> { self.require_str(b"@let")?; self.begin_token_at(TokenType::LetStart, start); if is_whitespace(self.peek()) { self.skip_whitespace(); } else { let parts = Parts::Value(self.span_from(start)); let token = self.end_token(parts)?; self.tokens[token].kind = TokenType::IncompleteLet; return Ok(()); } let name_start = self.pos; let mut allow_digit = false; self.attempt_char_code_until(|code| { if is_ascii_letter(code) || code == 36 || code == 95 || (allow_digit && is_digit(code)) { allow_digit = true; return false; } true })?; let name = Parts::Value(self.span_from(name_start)); let start_token = self.end_token(name)?; self.skip_whitespace(); if !self.attempt_char_code(61) { self.tokens[start_token].kind = TokenType::IncompleteLet; return Ok(()); } self.attempt_char_code_until(|code| is_not_whitespace(code) && !is_new_line(code))?; self.begin_token(TokenType::LetValue); self.attempt_until_ignore_quotes(|char| char == 59)?; self.end_token(Parts::None)?; if self.peek() == 59 { self.begin_token(TokenType::LetEnd); self.advance()?; self.end_token(Parts::None)?; } else { self.tokens[start_token].kind = TokenType::IncompleteLet; self.tokens[start_token].span = self.span_from(start); } Ok(()) } /// Returns true if it produced a token. fn tokenize_expansion_form(&mut self) -> Result { if self.is_expansion_form_start() { self.consume_expansion_form_start()?; return Ok(true); } if self.peek() != 125 && self.is_in_expansion_form() { self.consume_expansion_case_start()?; return Ok(true); } if self.peek() == 125 { if self.is_in_expansion_case() { self.begin_token(TokenType::ExpansionCaseExpEnd); self.require_char_code(125)?; self.end_token(Parts::None)?; self.skip_whitespace(); self.expansion_case_stack.pop(); return Ok(true); } if self.is_in_expansion_form() { self.begin_token(TokenType::ExpansionFormEnd); self.require_char_code(125)?; self.end_token(Parts::None)?; self.expansion_case_stack.pop(); return Ok(true); } } Ok(false) } fn read_until(&mut self, char: u32) -> Result { let start = self.pos; self.attempt_char_code_until(|code| code == char)?; Ok(self.span_from(start)) } fn consume_expansion_form_start(&mut self) -> Result<()> { self.begin_token(TokenType::ExpansionFormStart); self.require_char_code(123)?; self.end_token(Parts::None)?; self.expansion_case_stack.push(TokenType::ExpansionFormStart); for _ in 0..2 { self.begin_token(TokenType::RawText); self.read_until(44)?; self.end_token(Parts::None)?; self.require_char_code(44)?; self.skip_whitespace(); } Ok(()) } fn consume_expansion_case_start(&mut self) -> Result<()> { self.begin_token(TokenType::ExpansionCaseValue); let value = self.read_until(123)?; self.end_token(Parts::Value(trimmed(self.text, value)))?; self.skip_whitespace(); self.begin_token(TokenType::ExpansionCaseExpStart); self.require_char_code(123)?; self.end_token(Parts::None)?; self.skip_whitespace(); self.expansion_case_stack.push(TokenType::ExpansionCaseExpStart); Ok(()) } fn consume_entity(&mut self, text_token_type: TokenType) -> Result<()> { self.begin_token(TokenType::EncodedEntity); let start = self.pos; self.advance()?; if self.attempt_char_code(35) { let is_hex = self.attempt_char_code(120) || self.attempt_char_code(88); let code_start = self.pos; self.attempt_char_code_until(is_digit_entity_end)?; if self.peek() != 59 { self.advance()?; let kind = match is_hex { true => ErrorKind::HexadecimalEntityWithoutSemicolon, false => ErrorKind::DecimalEntityWithoutSemicolon, }; return Err(self.error_of(kind, self.pos, self.span_from(start))); } let digits = &self.text[code_start..self.pos]; self.advance()?; // `String.fromCodePoint(parseInt(digits, radix))` let radix = if is_hex { 16 } else { 10 }; let mut code: Option = None; for digit in digits.iter().map_while(|&byte| (byte as char).to_digit(radix)) { code = Some(code.unwrap_or(0).saturating_mul(radix).saturating_add(digit)); } if code.is_none_or(|code| code > 0x10FFFF) { return Err(self.error_of(ErrorKind::UnknownEntity, self.pos, self.span_from(start))); } self.end_token(Parts::None)?; return Ok(()); } let name_start = self.pos; self.attempt_char_code_until(is_named_entity_end)?; if self.peek() != 59 { self.begin_token_at(text_token_type, start); self.pos = name_start; self.end_token(Parts::None)?; return Ok(()); } self.advance()?; let entity = &self.text[start..self.pos]; if !super::entities::is_entity(&entity[1..entity.len() - 1]) { let name = Span::new(name_start as u32, self.pos as u32 - 1); return Err(self.error_of(ErrorKind::UnknownEntity, start, name)); } self.end_token(Parts::None)?; Ok(()) } /// True if the raw text's end marker comes next. fn is_at_end_marker(&self, marker: EndMarker<'_>) -> bool { match marker { EndMarker::Str(chars) => self.peek_str(chars), EndMarker::GreaterThan => self.peek() == 62, EndMarker::TagClose(name) => { if !self.peek_str(b") -> Result<()> { let kind = if consume_entities { TokenType::EscapableRawText } else { TokenType::RawText }; let first = match marker { EndMarker::Str(chars) => chars[0], EndMarker::GreaterThan => b'>', EndMarker::TagClose(_) => b'<', }; self.begin_token(kind); loop { // Skip to the next character that can start anything. let rest = self.text.get(self.pos..).unwrap_or_default(); let next = match consume_entities { true => strings::index_of_any(rest, &[first, b'&']), false => strings::index_of_char_usize(rest, first), }; self.pos += next.unwrap_or(rest.len()); if self.is_at_end_marker(marker) { break; } if consume_entities && self.peek() == 38 { self.end_token(Parts::None)?; self.consume_entity(TokenType::EscapableRawText)?; self.begin_token(TokenType::EscapableRawText); } else { self.advance()?; } } self.end_token(Parts::None)?; Ok(()) } /// A comment, a CDATA section or a document type. fn consume_delimited( &mut self, start: usize, start_kind: TokenType, end_kind: TokenType, marker: EndMarker<'_>, ) -> Result<()> { self.begin_token_at(start_kind, start); self.end_token(Parts::None)?; self.consume_raw_text(false, marker)?; self.begin_token(end_kind); match marker { EndMarker::Str(chars) => self.require_str(chars)?, _ => self.advance()?, } self.end_token(Parts::None)?; Ok(()) } fn consume_prefix_and_name(&mut self, end_predicate: impl FnMut(u32) -> bool) -> Result { let name_or_prefix_start = self.pos; while self.peek() != 58 && !is_prefix_end(self.peek()) { self.advance()?; } let (prefix, name_start) = if self.peek() == 58 { let prefix = self.span_from(name_or_prefix_start); self.advance()?; (prefix, self.pos) } else { (Span::new(name_or_prefix_start as u32, name_or_prefix_start as u32), name_or_prefix_start) }; self.require_char_code_until(end_predicate, usize::from(prefix.len() > 0))?; Ok(Parts::Name(prefix, self.span_from(name_start))) } fn consume_start_tag_comment(&mut self, start: usize, is_single_line: bool) -> Result<()> { let content_start = self.pos; if is_single_line { self.attempt_char_code_until(|code| is_new_line(code) || code == 0)?; let content = self.span_from(content_start); self.begin_token_at(TokenType::InElementComment, start); self.end_token(Parts::Comment(content, true))?; self.skip_whitespace(); return Ok(()); } while self.peek() != 0 && !self.peek_str(b"*/") { self.advance()?; } let content = self.span_from(content_start); let mut span_end = self.pos; if self.attempt_str(b"*/") { span_end = self.pos; self.skip_whitespace(); } self.begin_token_at(TokenType::InElementComment, start); self.end_token_at(Parts::Comment(content, false), span_end)?; Ok(()) } /// The body of the `try` in `_consumeTagOpen`. `open_token` is set to the index of the first token once it exists. fn consume_tag_open_tokens( &mut self, start: usize, open_token: &mut Option, ) -> Result<()> { if !is_ascii_letter(self.peek()) { return Err(self.error(start)); } self.begin_token_at(TokenType::TagOpenStart, start); let parts = self.consume_prefix_and_name(is_name_end)?; *open_token = Some(self.end_token(parts)?); self.skip_whitespace(); loop { if self.options.allow_start_tag_comments { let comment_start = self.pos; if self.attempt_str(b"//") { self.consume_start_tag_comment(comment_start, true)?; continue; } if self.attempt_str(b"/*") { self.consume_start_tag_comment(comment_start, false)?; continue; } } if is_attribute_terminator(self.peek()) { break; } let (name, value) = self.consume_attribute()?; let text = self.text; self .attributes .push(LexedAttribute { name: name.of(text), value: value.map(|value| value.of(text)) }); } let kind = if self.attempt_char_code(47) { TokenType::TagOpenEndVoid } else { TokenType::TagOpenEnd }; self.begin_token(kind); self.require_char_code(62)?; self.end_token(Parts::None)?; Ok(()) } fn consume_tag_open(&mut self, start: usize) -> Result<()> { let mut open_token = None; self.attributes.clear(); match self.consume_tag_open_tokens(start, &mut open_token) { Ok(()) => {} Err(Fail::Parse(_)) => { match open_token { Some(token) => self.tokens[token].kind = TokenType::IncompleteTagOpen, None => { self.begin_token_at(TokenType::Text, start); self.end_token(Parts::None)?; } } return Ok(()); } Err(fail) => return Err(fail), } if self.options.can_self_close && self.tokens.last().is_some_and(|token| token.kind == TokenType::TagOpenEndVoid) { return Ok(()); } let Some(Parts::Name(prefix, name)) = open_token.map(|token| self.tokens[token].parts) else { return Ok(()); }; let text = self.text; let content_type = (self.get_tag_content_type)( name.of(text), !self.full_name_stack.is_empty(), &self.attributes, ); // `_handleFullNameStackForTagOpen` if self.full_name_stack.last().is_none_or(|&last| self.is_same_name(last, (prefix, name))) { self.full_name_stack.push((prefix, name)); } if content_type == ContentType::ParsableData { return Ok(()); } // `_consumeRawTextWithTagClose` let full_name = Span::new(if prefix.len() > 0 { prefix.start } else { name.start }, name.end); self.consume_raw_text( content_type == ContentType::EscapableRawText, EndMarker::TagClose(full_name.of(text)), )?; self.begin_token(TokenType::TagClose); self.require_char_code_until(|code| code == 62, 3)?; self.advance()?; self.end_token(Parts::Name(prefix, name))?; self.handle_full_name_stack_for_tag_close(prefix, name); Ok(()) } fn is_same_name(&self, a: (Span, Span), b: (Span, Span)) -> bool { a.0.of(self.text) == b.0.of(self.text) && a.1.of(self.text) == b.1.of(self.text) } fn handle_full_name_stack_for_tag_close(&mut self, prefix: Span, name: Span) { if self.full_name_stack.last().is_some_and(|&last| self.is_same_name(last, (prefix, name))) { self.full_name_stack.pop(); } } /// Returns the name and the last piece of text in the value. fn consume_attribute(&mut self) -> Result<(Span, Option)> { let name_start = self.peek(); if name_start == 39 || name_start == 34 { return Err(self.error(self.pos)); } self.begin_token(TokenType::AttrName); let parts = if name_start == 91 { let mut open_brackets = 0i32; self.consume_prefix_and_name(|code| { if code == 91 { open_brackets += 1; } else if code == 93 { open_brackets -= 1; } if open_brackets <= 0 { is_name_end(code) } else { is_new_line(code) } })? } else { self.consume_prefix_and_name(is_name_end)? }; self.end_token(parts)?; let name = match parts { Parts::Name(_, name) => name, _ => Span::default(), }; let mut value = None; self.skip_whitespace(); if self.attempt_char_code(61) { self.skip_whitespace(); let (text, interpolation) = (TokenType::AttrValueText, TokenType::AttrValueInterpolation); let quote = self.peek(); value = Some(if quote == 39 || quote == 34 { self.consume_quote(quote)?; let value = self.consume_with_interpolation( text, interpolation, End::Quote(quote), End::Quote(quote), )?; self.consume_quote(quote)?; value } else { // This is a port of Angular's lexer, which ends the value where a name would end: `a=b=c` becomes // `a="b" ="c"`. That's right for Angular templates. Prettier parses HTML and Vue that way too. let end = match self.options.tokenize_angular_blocks { true => End::Name, false => End::UnquotedValue, }; self.consume_with_interpolation(text, interpolation, end, end)? }); } self.skip_whitespace(); Ok((name, value)) } fn consume_quote(&mut self, quote: u32) -> Result<()> { self.begin_token(TokenType::AttrQuote); self.require_char_code(quote)?; self.end_token(Parts::None)?; Ok(()) } fn consume_tag_close(&mut self, start: usize) -> Result<()> { self.begin_token_at(TokenType::TagClose, start); self.skip_whitespace(); if self.options.allow_htm_component_closing_tags && self.attempt_char_code(47) { self.skip_whitespace(); self.require_char_code(62)?; self.end_token(Parts::None)?; return Ok(()); } let parts = self.consume_prefix_and_name(is_name_end)?; self.skip_whitespace(); self.require_char_code(62)?; self.end_token(parts)?; if let Parts::Name(prefix, name) = parts { self.handle_full_name_stack_for_tag_close(prefix, name); } Ok(()) } /// Returns the last piece of text. fn consume_with_interpolation( &mut self, text_token_type: TokenType, interpolation_token_type: TokenType, end: End, end_interpolation: End, ) -> Result { self.begin_token(text_token_type); let mut text_start = self.pos; loop { self.skip_plain_text(end); if self.is_end(end) { break; } let current = self.pos; if self.attempt_str(b"{{") { self.end_token_at(Parts::None, current)?; self.consume_interpolation(interpolation_token_type, current, end_interpolation)?; self.begin_token(text_token_type); text_start = self.pos; } else if self.peek() == 38 { self.end_token(Parts::None)?; self.consume_entity(text_token_type)?; self.begin_token(text_token_type); text_start = self.pos; } else { self.advance()?; } } self.end_token(Parts::None)?; Ok(self.span_from(text_start)) } fn consume_interpolation( &mut self, kind: TokenType, start: usize, premature_end: End, ) -> Result<()> { self.begin_token_at(kind, start); let mut in_quote = None; let mut in_comment = false; while self.peek() != 0 && !self.is_end(premature_end) { if self.is_tag_start() { self.end_token(Parts::None)?; return Ok(()); } if in_quote.is_none() { if self.attempt_str(b"}}") { self.end_token(Parts::None)?; return Ok(()); } if self.attempt_str(b"//") { in_comment = true; } } let char = self.peek(); self.advance()?; if char == 92 { self.advance()?; } else if Some(char) == in_quote { in_quote = None; } else if !in_comment && in_quote.is_none() && is_quote(char) { in_quote = Some(char); } } self.end_token(Parts::None)?; Ok(()) } } /// `text.trim()` fn trimmed(text: &[u8], span: Span) -> Span { let all = span.of(text); let without_start = strings::trim_js_whitespace_start(all); let start = span.start + (all.len() - without_start.len()) as u32; Span::new(start, start + strings::trim_js_whitespace_end(without_start).len() as u32) } /// `tokenize`, from `start` on, into `tokens`. Returns the errors. pub(crate) fn tokenize<'a>( text: &'a [u8], start: usize, get_tag_content_type: GetTagContentType<'_>, options: Options, tokens: &mut Vec, ) -> Vec> { let mut tokenizer = Tokenizer { nul: memchr::memchr(0, &text[start..]).map_or(text.len(), |at| start + at), text, pos: start, get_tag_content_type, options, current: None, expansion_case_stack: Vec::new(), full_name_stack: Vec::new(), attributes: Vec::new(), tokens: std::mem::take(tokens), errors: Vec::new(), thrown: (ErrorKind::Other, Span::new(0, 0), false), }; // A token is typically about ten bytes. tokenizer.tokens.clear(); tokenizer.tokens.reserve(text.len().saturating_sub(start) / 8); tokenizer.tokenize(); // `mergeTextTokens` *tokens = tokenizer.tokens; tokens.dedup_by(|token, last| { let is_merged = last.kind == token.kind && matches!(token.kind, TokenType::Text | TokenType::AttrValueText); if is_merged { last.span.end = token.span.end; } is_merged }); tokenizer.errors }