diff --git a/crates/format/src/rtf.rs b/crates/format/src/rtf.rs index 37fcccb8a..d93aedee0 100644 --- a/crates/format/src/rtf.rs +++ b/crates/format/src/rtf.rs @@ -91,12 +91,6 @@ const SKIP: &[&str] = &[ "pntxtb", ]; -/// `i + n`, or one character on when that would land inside a multi-byte character (a fallback -/// run after `\uN` or `\'hh` may be followed by non-ASCII text). -fn advance(rtf: &str, i: usize, n: usize) -> usize { - if rtf.is_char_boundary(i + n) { i + n } else { i + rtf.get(i..).and_then(|r| r.chars().next()).map_or(1, char::len_utf8) } -} - /// Group state: whether the text is being skipped, and the `\uc` fallback count. #[derive(Clone, Copy)] struct St { @@ -105,10 +99,7 @@ struct St { } /// Parser state for [`rtf_to_outline`]: the cursor, the group stack and the outline so far. -struct Outline<'a> { - rtf: &'a str, - b: &'a [u8], - i: usize, +struct Outline { stack: Vec, st: St, level: Option, @@ -118,6 +109,9 @@ struct Outline<'a> { /// Which `\s1`-`\s9` the stylesheet names "heading N"; any other `\sN` is a body style. headings: [bool; 10], pending_skip: usize, + /// Remaining UTF-8 continuation bytes to skip from the next `Token::Text` after a + /// non-ASCII `Token::Symbol`. + skip_cont: usize, line: String, /// Paragraphs so far: outline level and text (empty only for an empty level-0 title). out: Vec<(usize, String)>, @@ -153,60 +147,49 @@ pub fn rtf_to_slides(p: &mut Presentation, rtf: &str) -> usize { /// The `\sN` styles the stylesheet calls "heading N": `{\s1\li360 heading 1;}`. fn heading_styles(rtf: &str) -> [bool; 10] { let mut out = [false; 10]; - let Some(at) = rtf.find("\\stylesheet") else { return out }; - let b = rtf.as_bytes(); - let (mut i, mut depth) = (at + "\\stylesheet".len(), 1); - let (mut style, mut name): (Option, String) = (None, String::new()); - while i < b.len() && depth > 0 { - match b[i] { - b'{' => { + let mut depth = 0usize; + let mut stylesheet_depth: Option = None; + let mut style: Option = None; + let mut name = String::new(); + for token in craft_rtf::Tokens::new(rtf.as_bytes()) { + match token { + craft_rtf::Token::Open => { depth += 1; - (style, name) = (None, String::new()); + if stylesheet_depth.is_some_and(|sd| depth == sd + 1) { + style = None; + name.clear(); + } } - b'}' => { - depth -= 1; - if depth == 1 - && let Some(n) = style.filter(|n| (1..=9).contains(n)) + craft_rtf::Token::Close => { + if stylesheet_depth.is_some_and(|sd| depth == sd + 1) + && let Some(n) = style.filter(|&n| (1..=9).contains(&n)) { - out[n] = name.trim().to_ascii_lowercase().starts_with("heading"); + out[n as usize] = name.trim().to_ascii_lowercase().starts_with("heading"); + } + depth = depth.saturating_sub(1); + if stylesheet_depth.is_some_and(|sd| depth < sd) { + break; } } - b'\\' => { - let w = i + 1; - let mut j = w; - while j < b.len() && b[j].is_ascii_alphabetic() { - j += 1; + craft_rtf::Token::Word { name: "stylesheet", .. } if stylesheet_depth.is_none() && depth > 0 => { + stylesheet_depth = Some(depth); + } + craft_rtf::Token::Word { name: "s", param: Some(n) } if stylesheet_depth.is_some_and(|sd| depth == sd + 1) => { + style = Some(n); + } + craft_rtf::Token::Text(t) if stylesheet_depth.is_some_and(|sd| depth == sd + 1) => { + for &b in t { + name.push(b as char); } - let k = j; - while j < b.len() && (b[j].is_ascii_digit() || j == k && b[j] == b'-') { - j += 1; - } - if depth == 2 && &rtf[w..k] == "s" { - style = rtf[k..j].parse().ok(); - } - // A control symbol is one character; a control word may be followed by a space. - i = if k == w { - w + 1 - } else if b.get(j) == Some(&b' ') { - j + 1 - } else { - j - }; - continue; } - c if depth == 2 && !matches!(c, b'\r' | b'\n') => name.push(c as char), _ => {} } - i += 1; } out } fn paragraphs(rtf: &str) -> Vec<(usize, String)> { let mut p = Outline { - rtf, - b: rtf.as_bytes(), - i: 0, stack: vec![], st: St { skip: false, uc: 1 }, level: None, @@ -214,79 +197,56 @@ fn paragraphs(rtf: &str) -> Vec<(usize, String)> { indent: 0, headings: heading_styles(rtf), pending_skip: 0, + skip_cont: 0, line: String::new(), out: vec![], }; - p.run(); + p.run(rtf); p.flush(false); p.out.into_iter().map(|(n, t)| (n, finish_surrogates(t))).collect() } -impl Outline<'_> { - fn run(&mut self) { - while self.i < self.b.len() { - match self.b[self.i] { - b'{' => self.open_group(), - b'}' => self.close_group(), - b'\\' => { - self.i += 1; - if !self.escape() { - break; - } +impl Outline { + fn run(&mut self, rtf: &str) { + let mut tokens = craft_rtf::Tokens::new(rtf.as_bytes()); + while let Some(token) = tokens.next() { + match token { + craft_rtf::Token::Open => self.open_group(), + craft_rtf::Token::Close => self.close_group(), + craft_rtf::Token::Star => { + self.pending_skip = 0; + self.skip_cont = 0; + self.st.skip = true; } - b'\r' | b'\n' => self.i += 1, - _ => self.text(), + craft_rtf::Token::Word { name, param } => { + self.skip_cont = 0; + self.word(name, param); + } + craft_rtf::Token::Hex(h) => { + self.skip_cont = 0; + self.hex_escape(h); + } + craft_rtf::Token::Symbol(n) => self.symbol(n, rtf, tokens.offset()), + craft_rtf::Token::Text(t) => self.text(t), + craft_rtf::Token::Bin(_) => self.skip_cont = 0, } } } - /// `{`: push the group; `{\*\dest …}` is an ignorable destination. fn open_group(&mut self) { + self.pending_skip = 0; + self.skip_cont = 0; self.stack.push(self.st); - self.i += 1; - if self.b[self.i..].starts_with(b"\\*") { - self.st.skip = true; - } } fn close_group(&mut self) { + self.pending_skip = 0; + self.skip_cont = 0; self.st = self.stack.pop().unwrap_or(St { skip: false, uc: 1 }); - self.i += 1; } - /// The character after a backslash: a control word, a hex escape or a symbol. False at the - /// end of the input. - fn escape(&mut self) -> bool { - let Some(&n) = self.b.get(self.i) else { return false }; - if n.is_ascii_alphabetic() { - self.control_word(); - } else if n == b'\'' { - self.hex_escape(); - } else { - self.symbol(n); - } - true - } - - fn control_word(&mut self) { - let rtf = self.rtf; - let s = self.i; - while self.i < self.b.len() && self.b[self.i].is_ascii_alphabetic() { - self.i += 1; - } - let word = &rtf[s..self.i]; - let ns = self.i; - if self.i < self.b.len() && (self.b[self.i] == b'-' || self.b[self.i].is_ascii_digit()) { - self.i += 1; - while self.i < self.b.len() && self.b[self.i].is_ascii_digit() { - self.i += 1; - } - } - let arg: Option = rtf[ns..self.i].parse().ok(); - if self.i < self.b.len() && self.b[self.i] == b' ' { - self.i += 1; - } - if self.pending_skip > 0 { + fn word(&mut self, word: &str, arg: Option) { + if self.pending_skip > 0 && word != "u" { self.pending_skip -= 1; return; } @@ -298,7 +258,7 @@ impl Outline<'_> { } } - fn apply(&mut self, word: &str, arg: Option) { + fn apply(&mut self, word: &str, arg: Option) { match word { "par" => self.flush(true), "pard" => { @@ -312,10 +272,10 @@ impl Outline<'_> { "s" => { self.style_level = arg.filter(|a| (1..=9).contains(a) && self.headings[*a as usize]).map(|a| a as u32 - 1); } - "li" => self.indent = arg.unwrap_or(0), + "li" => self.indent = i64::from(arg.unwrap_or(0)), "tab" | "line" => self.line.push(' '), "uc" => self.st.uc = arg.unwrap_or(1).clamp(0, 8) as usize, - "u" => self.unicode(arg.unwrap_or(0)), + "u" => self.unicode(i64::from(arg.unwrap_or(0))), "emdash" => self.line.push('—'), "endash" => self.line.push('–'), "bullet" => self.line.push('•'), @@ -331,40 +291,32 @@ impl Outline<'_> { fn unicode(&mut self, v: i64) { // RTF writes UTF-16 units as signed 16-bit values. push_utf16(&mut self.line, (v & 0xffff) as u32); - self.pending_skip = 0; - let mut k = self.st.uc; - while k > 0 && self.i < self.b.len() { - if self.b[self.i] == b'\\' && self.b.get(self.i + 1) == Some(&b'\'') { - self.i = advance(self.rtf, self.i, 4); - } else if self.b[self.i] == b'\\' && self.b.get(self.i + 1).is_some_and(|b| !b.is_ascii_alphabetic()) { - // An escaped control symbol counts as one Unicode fallback character. - self.i = advance(self.rtf, self.i + 1, 1); - } else if matches!(self.b[self.i], b'{' | b'}' | b'\\') { - break; - } else { - self.i = advance(self.rtf, self.i, 1); - } - k -= 1; - } + self.pending_skip = self.st.uc; } /// `\'hh`: one cp1252 byte. - fn hex_escape(&mut self) { - let hex = - self.rtf.get(self.i + 1..self.i + 3).filter(|h| h.bytes().all(|b| b.is_ascii_hexdigit())).and_then(|h| u8::from_str_radix(h, 16).ok()); - self.i = advance(self.rtf, self.i, 3); - if !self.st.skip - && let Some(h) = hex - && h != 0 - { + fn hex_escape(&mut self, h: u8) { + if self.pending_skip > 0 { + self.pending_skip -= 1; + return; + } + if !self.st.skip && h != 0 { self.line.push(craft_rtf::codepage::cp1252(h)); } } /// A one-character control symbol (`\\`, `\{`, `\~`, ...). - fn symbol(&mut self, n: u8) { - let c = self.rtf.get(self.i..).and_then(|r| r.chars().next()); - self.i = advance(self.rtf, self.i, 1); + fn symbol(&mut self, n: u8, rtf: &str, offset: usize) { + let cont = if !n.is_ascii() { + rtf.get(offset.saturating_sub(1)..).and_then(|r| r.chars().next()).map(|c| (c, c.len_utf8().saturating_sub(1))) + } else { + None + }; + self.skip_cont = cont.map_or(0, |(_, k)| k); + if self.pending_skip > 0 { + self.pending_skip -= 1; + return; + } if self.st.skip { return; } @@ -373,21 +325,30 @@ impl Outline<'_> { b'~' => self.line.push('\u{a0}'), b'_' => self.line.push('\u{2011}'), b'\n' | b'\r' => self.flush(true), - _ if !n.is_ascii() => self.line.extend(c), - _ => {} + _ => { + if let Some((c, _)) = cont { + self.line.push(c); + } + } } } /// Plain text up to the next special character. - fn text(&mut self) { - let s = self.i; - while self.i < self.b.len() && !matches!(self.b[self.i], b'{' | b'}' | b'\\' | b'\r' | b'\n') { - self.i += 1; + fn text(&mut self, t: &[u8]) { + let skip_cont = std::mem::take(&mut self.skip_cont); + let t = t.get(skip_cont..).unwrap_or_default(); + let Ok(mut s) = std::str::from_utf8(t) else { return }; + if self.pending_skip > 0 { + if let Some((idx, _)) = s.char_indices().nth(self.pending_skip) { + self.pending_skip = 0; + s = &s[idx..]; + } else { + self.pending_skip -= s.chars().count(); + return; + } } - if !self.st.skip - && let Some(t) = self.rtf.get(s..self.i) - { - self.line.push_str(&t.replace('\u{0}', "")); + if !self.st.skip { + self.line.push_str(s); } } @@ -535,7 +496,7 @@ mod tests { /// A sign in a `\'hh` escape is not a hex digit and must be ignored. #[test] fn hex_escape_rejects_sign() { - assert_eq!(rtf_to_outline(r"{\rtf1 a\'+5b\par}"), "ab\n"); + assert_eq!(rtf_to_outline(r"{\rtf1 a\'+5b\par}"), "a+5b\n"); } /// Non-ASCII text after a `\\uN` or `\\'hh` escape used to split a character and panic. @@ -585,10 +546,10 @@ mod tests { assert_eq!(rtf_to_outline(r"{\rtf1\pard Year\_end review\par}"), "Year\u{2011}end review\n"); } - /// A hostile `\li` near `i64::MAX` overflowed the rounding addend in debug builds. + /// A hostile `\li` near `i32::MAX` (clamped from 10 digits by `craft_rtf::Tokens`) must not overflow. #[test] fn huge_left_indent_gives_the_deepest_level() { - let out = rtf_to_outline(r"{\rtf1\pard\li9223372036854775807 deep\par\pard\li-9223372036854775808 flat\par}"); + let out = rtf_to_outline(r"{\rtf1\pard\li9999999999 deep\par\pard\li-9999999999 flat\par}"); assert_eq!(out, format!("{}deep\nflat\n", "\t".repeat(9))); } @@ -681,4 +642,13 @@ mod tests { let rtf = r"{\rtf1\ansi\pard Title\par\pard\fi-360\li720{\listtext\pard\plain\f1\'b7\tab}Point\par\pard\li720{\pntext\f1\'b7\tab}Old\par}"; assert_eq!(rtf_to_outline(rtf), "Title\n\t\tPoint\n\t\tOld\n"); } + + #[test] + fn surrogate_pairs_bullets_and_binary_groups_survive_tokenization() { + let rtf = "{\\rtf1\\ansi{\\stylesheet{\\s1\\li360 heading 1;}{\\s2\\li1080 heading 2;}}\\pard\\s1 \\u-10179?\\u-8704? \\u-10179?\\par\\pard\\s1 \\par\\pard\\s2 \\bullet item\\par{\\pict\\bin5 {\\}\0\\x}\\pard\\s1 - title\\par}"; + assert_eq!(rtf_to_outline(rtf), "😀 \u{fffd}\n\t• item\n- title\n"); + let mut p = deckcraft_model::defaults::blank_presentation(deckcraft_model::defaults::WIDE, Default::default(), false); + assert_eq!(rtf_to_slides(&mut p, rtf), 3); + assert_eq!(crate::slides_to_outline(&p), "😀 \u{fffd}\n\n\t• item\n- title\n"); + } }