From 6965ba573e9561f938c96f0862054f59e89fe4ce Mon Sep 17 00:00:00 2001 From: HueByte Date: Thu, 19 Feb 2026 21:11:06 +0100 Subject: [PATCH] fix: enhance chat message handling by implementing full Unicode support and collapsing excessive newlines --- docs/changelog/v0.2.3.md | 3 +- src/EchoHub.Client/UI/ChatRenderer.cs | 164 +++++++++------------ src/EchoHub.Server/Services/ChatService.cs | 82 +---------- 3 files changed, 69 insertions(+), 180 deletions(-) diff --git a/docs/changelog/v0.2.3.md b/docs/changelog/v0.2.3.md index 902b244..e428fad 100644 --- a/docs/changelog/v0.2.3.md +++ b/docs/changelog/v0.2.3.md @@ -47,8 +47,7 @@ - Fixed channels disappearing when creating a new channel — replaced full re-fetch with incremental updates - Wired `OnChannelUpdated` SignalR event so new public channels appear for all connected users in real time - Fixed color tag parser using wrong regex group numbers (6,7,8 instead of 1,2,3) — new ASCII art was rendering without colors -- Fixed crash when receiving emoji or other non-BMP Unicode characters — all list renderers now use `EnumerateRunes()` instead of `char` iteration -- Emoji and non-BMP characters are now converted to text shortcodes server-side (e.g. `:smile:`, `:fire:`) for reliable TUI rendering +- Full Unicode/emoji support — renderers use Terminal.Gui v2 grapheme cluster API (`GraphemeHelper`, `AddStr`) for proper wide character handling - Fixed `/send` and `/avatar` commands not handling file paths with spaces correctly, even when quoted - Profile avatar now renders with full color tag support instead of showing raw tags - Server-side newline spam protection — consecutive blank/whitespace-only lines collapsed to 1 and total lines capped at 30 diff --git a/src/EchoHub.Client/UI/ChatRenderer.cs b/src/EchoHub.Client/UI/ChatRenderer.cs index 605d3cd..ef6330a 100644 --- a/src/EchoHub.Client/UI/ChatRenderer.cs +++ b/src/EchoHub.Client/UI/ChatRenderer.cs @@ -1,6 +1,5 @@ using System.Collections; using System.Collections.Specialized; -using System.Text; using System.Text.RegularExpressions; using Terminal.Gui.Drawing; using Terminal.Gui.Text; @@ -27,21 +26,15 @@ public partial class ChatLine public ChatLine(string plainText) { Segments = [new ChatSegment(plainText, null)]; - TextLength = DisplayWidth(plainText); + TextLength = plainText.GetColumns(); } public ChatLine(List segments) { Segments = segments; - TextLength = segments.Sum(s => DisplayWidth(s.Text)); + TextLength = segments.Sum(s => s.Text.GetColumns()); } - /// - /// Compute the display column width of a string, accounting for wide characters (emoji, CJK). - /// - private static int DisplayWidth(string text) => - text.EnumerateRunes().Sum(r => Math.Max(r.GetColumns(), 1)); - public override string ToString() => string.Concat(Segments.Select(s => s.Text)); /// @@ -60,20 +53,18 @@ public partial class ChatLine foreach (var segment in Segments) { var text = segment.Text; - int chunkStart = 0; // char index where current chunk starts + int chunkStart = 0; int charPos = 0; - foreach (var rune in text.EnumerateRunes()) + foreach (var grapheme in GraphemeHelper.GetGraphemes(text)) { - var runeCols = Math.Max(rune.GetColumns(), 1); + var graphemeCols = Math.Max(grapheme.GetColumns(), 1); - if (col + runeCols > width) + if (col + graphemeCols > width) { - // Flush accumulated text from this segment chunk if (charPos > chunkStart) currentSegments.Add(new ChatSegment(text[chunkStart..charPos], segment.Color)); - // Emit current line and start a new one results.Add(new ChatLine(currentSegments)); currentSegments = []; @@ -90,11 +81,10 @@ public partial class ChatLine chunkStart = charPos; } - col += runeCols; - charPos += rune.Utf16SequenceLength; + col += graphemeCols; + charPos += grapheme.Length; } - // Flush remaining chunk of this segment if (chunkStart < text.Length) currentSegments.Add(new ChatSegment(text[chunkStart..], segment.Color)); } @@ -143,7 +133,6 @@ public partial class ChatLine if (match.Groups[1].Success) { - // Reset {X} currentFg = null; currentBg = null; } @@ -172,13 +161,12 @@ public partial class ChatLine return segments.Count > 0 ? new ChatLine(segments) : new ChatLine(""); } - // {X} (reset), {F:RRGGBB} (foreground), {B:RRGGBB} (background) [GeneratedRegex(@"\{(?:(X)|(?:(F|B):([0-9A-Fa-f]{6})))\}")] private static partial Regex ColorTagRegex(); } /// -/// Custom list data source for chat messages with per-character coloring. +/// Custom list data source for chat messages with per-segment coloring. /// public class ChatListSource : IListDataSource { @@ -232,7 +220,6 @@ public class ChatListSource : IListDataSource listView.Move(Math.Max(col - viewportX, 0), row); var chatLine = _lines[item]; - // Always use Normal — chat messages should not show focus/selection highlight var normalAttr = listView.GetAttributeForRole(VisualRole.Normal); var mentionBg = chatLine.IsMention ? ChatColors.MentionHighlightAttr.Background : (Color?)null; @@ -242,32 +229,26 @@ public class ChatListSource : IListDataSource foreach (var segment in chatLine.Segments) { var attr = segment.Color ?? normalAttr; - // Override background for mention-highlighted lines if (mentionBg.HasValue) attr = new Attribute(attr.Foreground, mentionBg.Value); listView.SetAttribute(attr); - foreach (var rune in segment.Text.EnumerateRunes()) + foreach (var grapheme in GraphemeHelper.GetGraphemes(segment.Text)) { - var cols = rune.GetColumns(); - if (cols < 1) cols = 1; + var cols = Math.Max(grapheme.GetColumns(), 1); if (charPos >= viewportX && drawnChars + cols <= width) { - listView.AddRune(rune); + listView.AddStr(grapheme); drawnChars += cols; } charPos += cols; } } - // Fill remaining width with spaces var fillAttr = mentionBg.HasValue ? new Attribute(normalAttr.Foreground, mentionBg.Value) : normalAttr; listView.SetAttribute(fillAttr); - while (drawnChars < width) - { - listView.AddRune(new Rune(' ')); - drawnChars++; - } + for (int i = drawnChars; i < width; i++) + listView.AddStr(" "); } private void UpdateMaxLength(ChatLine line) @@ -338,56 +319,30 @@ public class ChannelListSource : IListDataSource int drawnChars = 0; - // Use focus attr if this row is selected if (selected) { listView.SetAttribute(focusAttr); - foreach (var rune in (prefix + channelText + badge).EnumerateRunes()) - { - var cols = Math.Max(rune.GetColumns(), 1); - if (drawnChars + cols <= width) { listView.AddRune(rune); drawnChars += cols; } - } + drawnChars = RenderHelpers.WriteText(listView, prefix + channelText + badge, drawnChars, width); } else { - // Prefix - var prefixAttr = isActive ? ActiveAttr : NormalAttr; - listView.SetAttribute(prefixAttr); - foreach (var rune in prefix.EnumerateRunes()) - { - var cols = Math.Max(rune.GetColumns(), 1); - if (drawnChars + cols <= width) { listView.AddRune(rune); drawnChars += cols; } - } + listView.SetAttribute(isActive ? ActiveAttr : NormalAttr); + drawnChars = RenderHelpers.WriteText(listView, prefix, drawnChars, width); - // Channel name - var nameAttr = isActive ? ActiveAttr : hasUnread ? UnreadAttr : NormalAttr; - listView.SetAttribute(nameAttr); - foreach (var rune in channelText.EnumerateRunes()) - { - var cols = Math.Max(rune.GetColumns(), 1); - if (drawnChars + cols <= width) { listView.AddRune(rune); drawnChars += cols; } - } + listView.SetAttribute(isActive ? ActiveAttr : hasUnread ? UnreadAttr : NormalAttr); + drawnChars = RenderHelpers.WriteText(listView, channelText, drawnChars, width); - // Unread badge if (hasUnread) { listView.SetAttribute(BadgeAttr); - foreach (var rune in badge.EnumerateRunes()) - { - var cols = Math.Max(rune.GetColumns(), 1); - if (drawnChars + cols <= width) { listView.AddRune(rune); drawnChars += cols; } - } + drawnChars = RenderHelpers.WriteText(listView, badge, drawnChars, width); } } - // Fill rest var fillAttr = selected ? focusAttr : listView.GetAttributeForRole(VisualRole.Normal); listView.SetAttribute(fillAttr); - while (drawnChars < width) - { - listView.AddRune(new Rune(' ')); - drawnChars++; - } + for (int i = drawnChars; i < width; i++) + listView.AddStr(" "); } public void Dispose() { } @@ -409,7 +364,7 @@ public class UserListSource : IListDataSource { _users.Clear(); _users.AddRange(users); - MaxItemLength = users.Count > 0 ? users.Max(u => u.Text.Length) : 0; + MaxItemLength = users.Count > 0 ? users.Max(u => u.Text.GetColumns()) : 0; if (!SuspendCollectionChangedEvent) CollectionChanged?.Invoke(this, new NotifyCollectionChangedEventArgs(NotifyCollectionChangedAction.Reset)); } @@ -425,56 +380,72 @@ public class UserListSource : IListDataSource var (text, nameColor) = _users[item]; var normalAttr = listView.GetAttributeForRole(selected ? VisualRole.Focus : VisualRole.Normal); - // Convert to runes for safe surrogate handling - var runes = text.EnumerateRunes().ToArray(); - // Find where the name starts (after status icon + space + optional role badge) // Format: "● ★Username" or "● Username" + var graphemes = GraphemeHelper.GetGraphemes(text).ToList(); int nameStart = 0; - while (nameStart < runes.Length && !Rune.IsLetterOrDigit(runes[nameStart]) && runes[nameStart].Value != '_') + while (nameStart < graphemes.Count) + { + var g = graphemes[nameStart]; + if (g.Length > 0 && (char.IsLetterOrDigit(g[0]) || g[0] == '_')) + break; nameStart++; + } int drawnChars = 0; // Draw prefix (status icon + role badge) in normal color - var prefixAttr = normalAttr; - for (int c = 0; c < nameStart && c < runes.Length; c++) + listView.SetAttribute(normalAttr); + for (int i = 0; i < nameStart; i++) { - var cols = Math.Max(runes[c].GetColumns(), 1); - if (drawnChars + cols <= width) - { - listView.SetAttribute(prefixAttr); - listView.AddRune(runes[c]); - drawnChars += cols; - } + var cols = Math.Max(graphemes[i].GetColumns(), 1); + if (drawnChars + cols > width) break; + listView.AddStr(graphemes[i]); + drawnChars += cols; } // Draw name in nickname color - var userAttr = nameColor ?? normalAttr; - if (selected) userAttr = normalAttr; // use focus attr when selected - for (int c = nameStart; c < runes.Length; c++) + var userAttr = selected ? normalAttr : nameColor ?? normalAttr; + listView.SetAttribute(userAttr); + for (int i = nameStart; i < graphemes.Count; i++) { - var cols = Math.Max(runes[c].GetColumns(), 1); - if (drawnChars + cols <= width) - { - listView.SetAttribute(userAttr); - listView.AddRune(runes[c]); - drawnChars += cols; - } + var cols = Math.Max(graphemes[i].GetColumns(), 1); + if (drawnChars + cols > width) break; + listView.AddStr(graphemes[i]); + drawnChars += cols; } // Fill rest listView.SetAttribute(normalAttr); - while (drawnChars < width) - { - listView.AddRune(new Rune(' ')); - drawnChars++; - } + for (int i = drawnChars; i < width; i++) + listView.AddStr(" "); } public void Dispose() { } } +/// +/// Shared rendering helpers for IListDataSource implementations. +/// +static class RenderHelpers +{ + /// + /// Write text grapheme-by-grapheme to a ListView, respecting a width limit. + /// Returns the updated drawn-columns count. + /// + public static int WriteText(ListView lv, string text, int drawn, int maxWidth) + { + foreach (var grapheme in GraphemeHelper.GetGraphemes(text)) + { + var cols = Math.Max(grapheme.GetColumns(), 1); + if (drawn + cols > maxWidth) break; + lv.AddStr(grapheme); + drawn += cols; + } + return drawn; + } +} + /// /// Shared color attributes for chat rendering (timestamps, system messages). /// @@ -509,7 +480,6 @@ public static partial class ChatColors return segments; } - // Matches @username (letters, digits, underscores, hyphens — same as channel name chars) [GeneratedRegex(@"@[\w-]+")] private static partial Regex MentionRegex(); } diff --git a/src/EchoHub.Server/Services/ChatService.cs b/src/EchoHub.Server/Services/ChatService.cs index 5203176..3232033 100644 --- a/src/EchoHub.Server/Services/ChatService.cs +++ b/src/EchoHub.Server/Services/ChatService.cs @@ -1,4 +1,3 @@ -using System.Text; using EchoHub.Core.Constants; using EchoHub.Core.Contracts; using EchoHub.Core.DTOs; @@ -145,8 +144,7 @@ public class ChatService : IChatService if (content.Length > HubConstants.MaxMessageLength) return $"Message exceeds maximum length of {HubConstants.MaxMessageLength} characters."; - // Sanitize: convert emoji to text, collapse newlines - content = ConvertEmoji(content); + // Sanitize: collapse excessive newlines content = SanitizeNewlines(content); using var scope = _scopeFactory.CreateScope(); @@ -348,84 +346,6 @@ public class ChatService : IChatService return (user.Id, user.Username); } - /// - /// Replace emoji with text shortcodes. TUI terminals can't render wide chars reliably. - /// - private static string ConvertEmoji(string content) - { - var sb = new StringBuilder(content.Length); - foreach (var rune in content.EnumerateRunes()) - { - if (EmojiMap.TryGetValue(rune.Value, out var name)) - sb.Append(name); - else if (rune.Value >= 0x1F000) // supplementary emoji planes - sb.Append($"[?]"); - else if (rune.Value is 0x200D or 0xFE0F or 0xFE0E) // ZWJ, variation selectors - { } // strip silently - else - sb.Append(rune.ToString()); - } - return sb.ToString(); - } - - private static readonly Dictionary EmojiMap = new() - { - [0x1F600] = ":grinning:", [0x1F601] = ":grin:", [0x1F602] = ":joy:", - [0x1F603] = ":smiley:", [0x1F604] = ":smile:", [0x1F605] = ":sweat_smile:", - [0x1F606] = ":laughing:", [0x1F607] = ":angel:", [0x1F608] = ":imp:", - [0x1F609] = ":wink:", [0x1F60A] = ":blush:", [0x1F60B] = ":yum:", - [0x1F60C] = ":relieved:", [0x1F60D] = ":heart_eyes:", [0x1F60E] = ":sunglasses:", - [0x1F60F] = ":smirk:", [0x1F610] = ":neutral:", [0x1F611] = ":expressionless:", - [0x1F612] = ":unamused:", [0x1F613] = ":sweat:", [0x1F614] = ":pensive:", - [0x1F615] = ":confused:", [0x1F616] = ":confounded:", [0x1F617] = ":kiss:", - [0x1F618] = ":kissing_heart:", [0x1F619] = ":kissing:", [0x1F61A] = ":kissing_closed_eyes:", - [0x1F61B] = ":tongue:", [0x1F61C] = ":wink_tongue:", [0x1F61D] = ":squint_tongue:", - [0x1F61E] = ":disappointed:", [0x1F61F] = ":worried:", [0x1F620] = ":angry:", - [0x1F621] = ":rage:", [0x1F622] = ":cry:", [0x1F623] = ":persevere:", - [0x1F624] = ":triumph:", [0x1F625] = ":disappointed_relieved:", [0x1F626] = ":frowning:", - [0x1F627] = ":anguished:", [0x1F628] = ":fearful:", [0x1F629] = ":weary:", - [0x1F62A] = ":sleepy:", [0x1F62B] = ":tired:", [0x1F62C] = ":grimacing:", - [0x1F62D] = ":sob:", [0x1F62E] = ":open_mouth:", [0x1F62F] = ":hushed:", - [0x1F630] = ":cold_sweat:", [0x1F631] = ":scream:", [0x1F632] = ":astonished:", - [0x1F633] = ":flushed:", [0x1F634] = ":sleeping:", [0x1F635] = ":dizzy_face:", - [0x1F636] = ":no_mouth:", [0x1F637] = ":mask:", [0x1F638] = ":smile_cat:", - [0x1F642] = ":slight_smile:", [0x1F643] = ":upside_down:", - [0x1F644] = ":roll_eyes:", [0x1F910] = ":zipper_mouth:", - [0x1F911] = ":money_mouth:", [0x1F912] = ":thermometer_face:", - [0x1F913] = ":nerd:", [0x1F914] = ":thinking:", [0x1F915] = ":head_bandage:", - [0x1F920] = ":cowboy:", [0x1F921] = ":clown:", [0x1F923] = ":rofl:", - [0x1F924] = ":drooling:", [0x1F925] = ":lying:", - [0x1F970] = ":smiling_hearts:", [0x1F971] = ":yawning:", - [0x1F972] = ":smiling_tear:", [0x1F973] = ":party:", - [0x1F974] = ":woozy:", [0x1F975] = ":hot:", [0x1F976] = ":cold:", - [0x1F978] = ":disguised:", [0x1F979] = ":holding_back_tears:", - [0x1F97A] = ":pleading:", [0x1F92A] = ":zany:", [0x1F92B] = ":shushing:", - [0x1F92C] = ":censored:", [0x1F92D] = ":hand_over_mouth:", - [0x1F92E] = ":vomiting:", [0x1F92F] = ":exploding_head:", - // Gestures - [0x1F44D] = ":+1:", [0x1F44E] = ":-1:", [0x1F44F] = ":clap:", - [0x1F44B] = ":wave:", [0x1F44C] = ":ok_hand:", [0x1F44A] = ":punch:", - [0x1F4AA] = ":muscle:", [0x1F64F] = ":pray:", [0x1F91D] = ":handshake:", - [0x1F90C] = ":pinched_fingers:", [0x1F918] = ":metal:", [0x1F919] = ":call_me:", - // Hearts - [0x2764] = "<3", [0x1F494] = " /// Collapse consecutive newlines and cap total line count to prevent newline spam. ///