feat: add unicode segmentation support for emoji handling and sanitize terminal output

Cargo.lock

@@ -3726,6 +3726,7 @@ dependencies = [
  "tiny_http",
  "tokio",
  "toml 1.1.8+spec-1.1.0",
+ "unicode-segmentation",
  "whatsapp-rust",
 ]
 

Cargo.toml

@@ -27,6 +27,8 @@ base64 = "0.23"
 chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
 # Minimal synchronous HTTP server for the optional web control panel (web_port).
 tiny_http = "0.12"
+# Grapheme segmentation, so the TUI sanitizes and wraps emoji without splitting them.
+unicode-segmentation = "1.13"
 
 [profile.release]
 # Remove symbols and debug info from the binary.

src/main.rs

@@ -188,13 +188,19 @@ fn clear_screen() -> std::io::Result<()> {
 /// Purpose: draw the UI and dispatch events until quit or the event channel closes.
 /// Inputs: `terminal`, the active terminal; `app`, the application state; `app_rx`, the event receiver.
 /// Output: `Ok(())` on quit or channel close, otherwise a draw or event-handling error.
-/// Side effects: draws to the terminal and mutates `app`; forces a full repaint when the list mode
-/// changes, so wide glyphs from the previous list cannot leave stale cells behind.
+/// Side effects: clears the whole screen once before the first draw, so text left by the shell on
+/// terminals with a non-blank alternate screen cannot show through; then draws to the terminal and
+/// mutates `app`.
 async fn event_loop(
     terminal: &mut DefaultTerminal,
     mut app: App,
     mut app_rx: UnboundedReceiver<AppEvent>,
 ) -> Result<()> {
+    // `Terminal::clear` reads the cursor position, which times out on some terminals;
+    // resizing to the current size clears the fullscreen viewport without that query.
+    let size = terminal.size().context("cannot read terminal size")?;
+    terminal.resize(size.into()).context("cannot clear terminal")?;
+
     while !app.should_quit {
         terminal
             .draw(|frame| ui::render(frame, &app))
@@ -202,15 +208,7 @@ async fn event_loop(
         let Some(event) = app_rx.recv().await else {
             break;
         };
-        let list_mode_before = app.list_mode;
         app.handle_event(event)?;
-        if app.list_mode != list_mode_before {
-
-            // `Terminal::clear` reads the cursor position, which times out on some terminals;
-            // resizing to the current size clears the fullscreen viewport without that query.
-            let size = terminal.size().context("cannot read terminal size")?;
-            terminal.resize(size.into()).context("cannot clear terminal")?;
-        }
     }
     Ok(())
 }

src/ui.rs

@@ -3,16 +3,19 @@
 //! Rendering is a pure function of `&App`: nothing here mutates application state.
 
 use std::fmt::Display;
+use std::ops::RangeInclusive;
 
 use chrono::{DateTime, Local, TimeZone};
 use qrcode::QrCode;
 use qrcode::render::unicode::Dense1x2;
 use qrcode::types::QrError;
 use ratatui::Frame;
+use ratatui::buffer::CellWidth;
 use ratatui::layout::{Alignment, Constraint, Layout, Rect};
 use ratatui::style::{Color, Modifier, Style};
 use ratatui::text::{Line, Span, Text};
 use ratatui::widgets::{Block, List, ListItem, ListState, Paragraph};
+use unicode_segmentation::UnicodeSegmentation;
 
 use crate::app::{App, Focus, ListMode, Screen};
 use crate::history::{ChatSummary, Contact, StoredMessage};
@@ -102,6 +105,21 @@ const AI_MESSAGE_COLOR: Color = Color::Cyan;
 /// Color of messages sent by the user.
 const OWN_MESSAGE_COLOR: Color = Color::Green;
 
+/// Zero width joiner, used to glue several emoji into one grapheme.
+const ZERO_WIDTH_JOINER: char = '\u{200D}';
+/// Variation selector 15 (text presentation).
+const VARIATION_SELECTOR_TEXT: char = '\u{FE0E}';
+/// Variation selector 16 (emoji presentation).
+const VARIATION_SELECTOR_EMOJI: char = '\u{FE0F}';
+/// Combining enclosing keycap, used by keycap emoji sequences.
+const COMBINING_KEYCAP: char = '\u{20E3}';
+/// Emoji skin tone modifiers (Fitzpatrick types 1-2 to 6).
+const SKIN_TONE_MODIFIERS: RangeInclusive<char> = '\u{1F3FB}'..='\u{1F3FF}';
+/// Tag characters, used by subdivision flag sequences.
+const TAG_CHARACTERS: RangeInclusive<char> = '\u{E0020}'..='\u{E007F}';
+/// Regional indicator symbols, used in pairs by flag emoji.
+const REGIONAL_INDICATORS: RangeInclusive<char> = '\u{1F1E6}'..='\u{1F1FF}';
+
 /// Purpose: draw the whole UI for the current application state.
 /// Inputs: the ratatui `frame` to draw on and the read-only `app` state.
 /// Output: none (the frame buffer is filled).
@@ -123,7 +141,7 @@ fn render_connecting(frame: &mut Frame, area: Rect, app: &App) {
     let lines = vec![
         Line::styled(TEXT_CONNECTING, Style::new().add_modifier(Modifier::BOLD)),
         Line::raw(""),
-        Line::raw(app.status.clone()),
+        Line::raw(sanitize_for_terminal(&app.status)),
         Line::styled(TEXT_SCREEN_KEYS, Style::new().add_modifier(Modifier::DIM)),
     ];
     render_centered(frame, area, lines);
@@ -144,7 +162,7 @@ fn render_pairing(frame: &mut Frame, area: Rect, app: &App) {
     lines.extend(qr_lines);
     lines.push(Line::raw(""));
     lines.push(Line::raw(TEXT_PAIRING_STEPS));
-    lines.push(Line::raw(app.status.clone()));
+    lines.push(Line::raw(sanitize_for_terminal(&app.status)));
     lines.push(Line::styled(TEXT_SCREEN_KEYS, Style::new().add_modifier(Modifier::DIM)));
     render_centered(frame, area, lines);
 }
@@ -275,7 +293,7 @@ fn list_title(base: &str, global_autoreply: bool, filter: &str) -> String {
         title.push_str(AI_GLOBAL_MARKER);
     }
     if !filter.is_empty() {
-        title.push_str(&format!("{FILTER_LABEL}{filter} "));
+        title.push_str(&format!("{FILTER_LABEL}{} ", sanitize_for_terminal(filter)));
     }
     title
 }
@@ -293,7 +311,7 @@ fn selected_index(len: usize, selected: usize) -> Option<usize> {
 /// Output: the list item.
 /// Side effects: none.
 fn chat_item(chat: &ChatSummary) -> ListItem<'static> {
-    let mut spans = vec![Span::raw(chat.name.clone())];
+    let mut spans = vec![Span::raw(sanitize_for_terminal(&chat.name))];
     if chat.autoreply {
         spans.push(Span::styled(AI_ROW_MARKER, Style::new().fg(AI_MARKER_COLOR)));
     }
@@ -306,7 +324,7 @@ fn chat_item(chat: &ChatSummary) -> ListItem<'static> {
 /// Output: the list item.
 /// Side effects: none.
 fn contact_item(contact: &Contact) -> ListItem<'static> {
-    ListItem::new(Line::raw(contact.name.clone()))
+    ListItem::new(Line::raw(sanitize_for_terminal(&contact.name)))
 }
 
 /// Purpose: build the title of the messages pane.
@@ -318,6 +336,7 @@ fn messages_title(open_chat: Option<&str>, name: &str, autoreply: bool) -> Strin
         return TITLE_NO_CHAT.to_string();
     }
     let state = if autoreply { AI_ON } else { AI_OFF };
+    let name = sanitize_for_terminal(name);
 
     format!(" {name} {TITLE_SEPARATOR} {AI_LABEL}: {state} ")
 }
@@ -353,7 +372,8 @@ fn visible_message_lines(messages: &[StoredMessage], width: u16, height: u16) ->
 
 /// Purpose: format, wrap and style a single message.
 /// Inputs: the message and the wrap `width` in cells.
-/// Output: the styled lines of the message.
+/// Output: the styled lines of the message; the text is sanitized by `wrap_text`, which cleans
+/// each paragraph after splitting on newlines so that explicit line breaks survive.
 /// Side effects: none.
 fn message_lines(message: &StoredMessage, width: usize) -> Vec<Line<'static>> {
     let style = message_style(message);
@@ -440,27 +460,61 @@ fn jid_user_part(jid: &str) -> &str {
     jid.split_once(JID_SEPARATOR).map_or(jid, |(user, _)| user)
 }
 
-/// Purpose: measure the display width of a string in terminal cells.
+/// Purpose: measure the display width of a string in terminal cells, exactly as the ratatui
+/// renderer does (sum of the cell width of each grapheme).
 /// Inputs: the text.
 /// Output: the width in cells.
 /// Side effects: none.
 fn text_width(text: &str) -> usize {
-    Line::raw(text).width()
+    text.graphemes(true).map(|grapheme| usize::from(grapheme.cell_width())).sum()
 }
 
-/// Purpose: measure the display width of a character in terminal cells.
+/// Purpose: tell whether a character takes part in composing emoji graphemes (joiners,
+/// variation selectors, skin tones, keycaps, tags, regional indicators).
 /// Inputs: the character.
-/// Output: the width in cells (0, 1 or 2).
+/// Output: `true` if the character is an emoji composition character.
+/// Side effects: none.
+fn is_emoji_composition(ch: char) -> bool {
+    matches!(
+        ch,
+        ZERO_WIDTH_JOINER | VARIATION_SELECTOR_TEXT | VARIATION_SELECTOR_EMOJI | COMBINING_KEYCAP
+    ) || SKIN_TONE_MODIFIERS.contains(&ch)
+        || TAG_CHARACTERS.contains(&ch)
+        || REGIONAL_INDICATORS.contains(&ch)
+}
+
+/// Purpose: simplify a single grapheme to what terminals draw with a predictable width.
+/// Inputs: one extended grapheme cluster.
+/// Output: `None` for graphemes with control characters or starting with an emoji composition
+/// character (regional indicators, joiners, selectors, skin tones, keycaps, tags: dropping them
+/// stops them from re-attaching to the previous grapheme after a re-segmentation); the first
+/// character only for composed emoji; the grapheme unchanged otherwise. No emitted grapheme
+/// starts with a composition character.
 /// Side effects: none.
-fn char_width(ch: char) -> usize {
-    let mut buffer = [0u8; 4];
+fn sanitize_grapheme(grapheme: &str) -> Option<&str> {
+    let first = grapheme.chars().next()?;
+    if grapheme.chars().any(char::is_control) || is_emoji_composition(first) {
+        return None;
+    }
+    let is_composed_emoji = grapheme.chars().count() > 1 && grapheme.chars().any(is_emoji_composition);
+
+    Some(if is_composed_emoji { &grapheme[..first.len_utf8()] } else { grapheme })
+}
 
-    text_width(ch.encode_utf8(&mut buffer))
+/// Purpose: make externally sourced text safe to draw: Windows consoles and `unicode-width`
+/// disagree on the width of composed emoji, which leaves stale cells on screen.
+/// Inputs: the `text` to clean.
+/// Output: the text without control characters, flags and graphemes starting with a composition
+/// character (lone joiners, selectors, skin tones, keycaps, tags, regional indicators), with each
+/// composed emoji reduced to its first character. Idempotent: sanitizing the output is a no-op.
+/// Side effects: none.
+fn sanitize_for_terminal(text: &str) -> String {
+    text.graphemes(true).filter_map(sanitize_grapheme).collect()
 }
 
 /// Purpose: wrap text to a width, honoring explicit newlines.
 /// Inputs: the `text` and the maximum `width` in cells (at least 1 is used).
-/// Output: the rows; control characters are dropped.
+/// Output: the rows; control characters are dropped and emoji are simplified.
 /// Side effects: none.
 fn wrap_text(text: &str, width: usize) -> Vec<String> {
     let width = width.max(1);
@@ -472,21 +526,22 @@ fn wrap_text(text: &str, width: usize) -> Vec<String> {
     rows
 }
 
-/// Purpose: wrap a single paragraph (no newlines) by characters.
+/// Purpose: wrap a single paragraph (no newlines) by graphemes, never splitting an emoji.
 /// Inputs: the `paragraph`, the `width` in cells and the output `rows` to append to.
 /// Output: none.
 /// Side effects: appends at least one row to `rows`.
 fn wrap_paragraph(paragraph: &str, width: usize, rows: &mut Vec<String>) {
+    let clean = sanitize_for_terminal(paragraph);
     let mut row = String::new();
     let mut used = 0;
-    for ch in paragraph.chars().filter(|ch| !ch.is_control()) {
-        let ch_width = char_width(ch);
-        if used + ch_width > width && !row.is_empty() {
+    for grapheme in clean.graphemes(true) {
+        let grapheme_width = text_width(grapheme);
+        if used + grapheme_width > width && !row.is_empty() {
             rows.push(std::mem::take(&mut row));
             used = 0;
         }
-        row.push(ch);
-        used += ch_width;
+        row.push_str(grapheme);
+        used += grapheme_width;
     }
     rows.push(row);
 }
@@ -499,7 +554,7 @@ fn render_input_pane(frame: &mut Frame, area: Rect, app: &App) {
     let focused = app.focus == Focus::Input;
     let block = pane_block(TITLE_INPUT.to_string(), focused);
     let inner = block.inner(area);
-    let typed = format!("{INPUT_PROMPT}{}", app.input);
+    let typed = sanitize_for_terminal(&format!("{INPUT_PROMPT}{}", app.input));
     let visible = tail_fitting(&typed, usize::from(inner.width).saturating_sub(1));
     let cursor_offset = u16::try_from(text_width(visible)).unwrap_or(u16::MAX);
 
@@ -511,13 +566,13 @@ fn render_input_pane(frame: &mut Frame, area: Rect, app: &App) {
 
 /// Purpose: keep the end of a text that fits a width (horizontal scroll of the input).
 /// Inputs: the `text` and the `max_width` in cells.
-/// Output: the longest suffix whose width is at most `max_width`.
+/// Output: the longest suffix of whole graphemes whose width is at most `max_width`.
 /// Side effects: none.
 fn tail_fitting(text: &str, max_width: usize) -> &str {
     let mut used = 0;
     let mut start = text.len();
-    for (index, ch) in text.char_indices().rev() {
-        used += char_width(ch);
+    for (index, grapheme) in text.grapheme_indices(true).rev() {
+        used += text_width(grapheme);
         if used > max_width {
             break;
         }
@@ -558,7 +613,7 @@ fn render_footer(frame: &mut Frame, area: Rect, app: &App) {
         Layout::horizontal([Constraint::Fill(1), Constraint::Length(version_width)]).areas(area);
     let mut spans = vec![
         Span::raw(FOOTER_MARGIN),
-        Span::styled(app.status.clone(), Style::new().add_modifier(Modifier::BOLD)),
+        Span::styled(sanitize_for_terminal(&app.status), Style::new().add_modifier(Modifier::BOLD)),
         Span::raw(FOOTER_GAP),
     ];
     spans.extend(key_help_spans(app.global_autoreply));
@@ -695,6 +750,85 @@ mod tests {
         assert_eq!(wrap_text("", 10), [""]);
     }
 
+    #[test]
+    fn sanitize_reduces_zwj_sequences_to_the_first_emoji() {
+        assert_eq!(sanitize_for_terminal("\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}"), "\u{1F468}");
+    }
+
+    #[test]
+    fn sanitize_drops_variation_selectors_skin_tones_and_keycaps() {
+        assert_eq!(sanitize_for_terminal("\u{2764}\u{FE0F}"), "\u{2764}");
+        assert_eq!(sanitize_for_terminal("\u{1F44D}\u{1F3FD}"), "\u{1F44D}");
+        assert_eq!(sanitize_for_terminal("1\u{FE0F}\u{20E3}"), "1");
+    }
+
+    #[test]
+    fn sanitize_removes_flags_and_lone_regional_indicators() {
+        assert_eq!(sanitize_for_terminal("a\u{1F1EE}\u{1F1F9}b"), "ab");
+        assert_eq!(sanitize_for_terminal("a\u{1F1EE}b"), "ab");
+    }
+
+    #[test]
+    fn sanitize_keeps_plain_text_and_combining_accents() {
+        assert_eq!(sanitize_for_terminal("Ciao mondo \u{1F600}"), "Ciao mondo \u{1F600}");
+        assert_eq!(sanitize_for_terminal("e\u{301}"), "e\u{301}");
+    }
+
+    #[test]
+    fn sanitize_drops_control_characters() {
+        assert_eq!(sanitize_for_terminal("a\tb\u{7}c"), "abc");
+    }
+
+    #[test]
+    fn sanitize_does_not_recompose_after_dropping_a_control_grapheme() {
+        assert_eq!(sanitize_for_terminal("\u{1F468}\u{7}\u{200D}\u{1F469}"), "\u{1F468}\u{1F469}");
+        assert_eq!(sanitize_for_terminal("\u{1F44D}\r\u{1F3FD}"), "\u{1F44D}");
+    }
+
+    #[test]
+    fn sanitize_drops_leading_composition_characters() {
+        assert_eq!(sanitize_for_terminal("\u{FE0F}a"), "a");
+        assert_eq!(sanitize_for_terminal("\u{2764}\u{FE0E}"), "\u{2764}");
+        assert_eq!(sanitize_for_terminal("\u{1F3F4}\u{E0067}\u{E0062}\u{E007F}"), "\u{1F3F4}");
+        assert_eq!(sanitize_for_terminal(""), "");
+    }
+
+    #[test]
+    fn sanitize_is_idempotent() {
+        let inputs = [
+            "\u{1F468}\u{7}\u{200D}\u{1F469}",
+            "\u{1F44D}\r\u{1F3FD}",
+            "\u{FE0F}a",
+            "\u{2764}\u{FE0E}",
+            "\u{1F3F4}\u{E0067}\u{E0062}\u{E007F}",
+            "",
+        ];
+        for input in inputs {
+            let once = sanitize_for_terminal(input);
+
+            assert_eq!(sanitize_for_terminal(&once), once);
+        }
+    }
+
+    #[test]
+    fn text_width_matches_the_renderer_cell_width() {
+        assert_eq!(text_width("\u{644}\u{627}"), 2);
+        assert_eq!(text_width("\u{FF76}\u{FF9E}"), 2);
+    }
+
+    #[test]
+    fn wrap_never_splits_a_wide_emoji() {
+        assert_eq!(wrap_text("a\u{1F600}b", 2), ["a", "\u{1F600}", "b"]);
+        assert_eq!(wrap_text("\u{1F468}\u{200D}\u{1F469}", 2), ["\u{1F468}"]);
+    }
+
+    #[test]
+    fn tail_fitting_keeps_whole_graphemes() {
+        assert_eq!(tail_fitting("ab\u{1F600}", 1), "");
+        assert_eq!(tail_fitting("ab\u{1F600}", 2), "\u{1F600}");
+        assert_eq!(tail_fitting("ae\u{301}", 1), "e\u{301}");
+    }
+
     #[test]
     fn visible_lines_keep_the_latest() {
         let messages = [incoming("one"), incoming("two"), incoming("three")];