Implement grapheme clusters

Author: lhecker

.github/actions/spelling/expect/expect.txt

@@ -144,6 +144,7 @@ bytebuffer
 cac
 cacafire
 CALLCONV
+CANDRABINDU
 capslock
 CARETBLINKINGENABLED
 CARRIAGERETURN
@@ -155,6 +156,7 @@ cbiex
 CBN
 CBoolean
 cbt
+Ccc
 CCCBB
 cch
 CCHAR
@@ -180,6 +182,7 @@ chaof
 charinfo
 CHARSETINFO
 chh
+chonker
 chshdng
 CHT
 Cic
@@ -598,7 +601,9 @@ FEEF
 fesb
 FFAF
 FFDE
+FFFD
 FFFDb
+FFrom
 fgbg
 FGCOLOR
 FGHIJ
@@ -617,6 +622,7 @@ FINDDOWN
 FINDSTRINGEXACT
 FINDUP
 FIter
+FITZPATRICK
 FIXEDCONVERTED
 FIXEDFILEINFO
 Flg
@@ -888,11 +894,13 @@ jconcpp
 JLO
 JOBOBJECT
 JOBOBJECTINFOCLASS
+JONGSEONG
 JPN
 jsoncpp
 Jsons
 jsprovider
 jumplist
+JUNGSEONG
 KAttrs
 kawa
 Kazu
@@ -911,6 +919,7 @@ keyups
 KILLACTIVE
 KILLFOCUS
 kinda
+KIYEOK
 KLF
 KLMNO
 KLMNOPQRST
@@ -1020,6 +1029,7 @@ luma
 lval
 LVB
 LVERTICAL
+LVT
 LWA
 LWIN
 lwkmvj
@@ -1049,6 +1059,7 @@ mdmerge
 MDs
 MEASUREITEM
 megamix
+Meh
 memallocator
 meme
 MENUCHAR
@@ -1164,6 +1175,7 @@ NOMINMAX
 NOMOVE
 NONALERT
 nonbreaking
+noncharacter
 nonclient
 NONINFRINGEMENT
 NONPREROTATED
@@ -1212,6 +1224,7 @@ ntuser
 NTVDM
 ntverp
 nugetversions
+NUKTA
 nullness
 nullonfailure
 nullopts
@@ -1489,6 +1502,7 @@ renderengine
 rendersize
 reparented
 reparenting
+REPH
 replatformed
 Replymessage
 repositorypath
@@ -1517,6 +1531,7 @@ rgw
 RIGHTALIGN
 RIGHTBUTTON
 riid
+ris
 RIS
 roadmap
 robomac
@@ -1883,6 +1898,7 @@ UPDATEDISPLAY
 UPDOWN
 UPKEY
 upss
+UPSS
 uregex
 URegular
 usebackq
@@ -1925,6 +1941,7 @@ vga
 vgaoem
 viewkind
 viewports
+VIRAMA
 Virt
 VIRTTERM
 vkey
@@ -2165,6 +2182,7 @@ Zabcdefghijklmn
 Zabcdefghijklmnopqrstuvwxyz
 ZCmd
 ZCtrl
+ZWJs
 zxcvbnm
 ZYXWVU
 ZYXWVUTd

doc/cascadia/profiles.schema.json

@@ -2344,6 +2344,11 @@
           "description": "Force the terminal to use the legacy input encoding. Certain keys in some applications may stop working when enabling this setting.",
           "type": "boolean"
         },
+        "experimental.graphemes": {
+          "default": true,
+          "description": "When set to true, the terminal will use grapheme cluster boundaries for cursor movement. Otherwise, the terminal will use codepoint boundaries.",
+          "type": "boolean"
+        },
         "experimental.useBackgroundImageForWindow": {
           "default": false,
           "description": "When set to true, the background image for the currently focused profile is expanded to encompass the entire window, beneath other panes.",

src/buffer/out/Row.cpp

@@ -5,10 +5,8 @@
 #include "Row.hpp"
 
 #include <isa_availability.h>
-#include <til/unicode.h>
 
-#include "textBuffer.hpp"
-#include "../../types/inc/GlyphWidth.hpp"
+#include "../../types/inc/CodepointWidthDetector.hpp"
 
 // It would be nice to add checked array access in the future, but it's a little annoying to do so without impacting
 // performance (including Debug performance). Other languages are a little bit more ergonomic there than C++.
@@ -646,60 +644,45 @@ catch (...)
     //
     // We can infer the "end" from the amount of columns we're given (colLimit - colBeg),
     // because ASCII is always 1 column wide per character.
-    auto it = chars.begin();
-    const auto end = it + std::min<size_t>(chars.size(), colLimit - colBeg);
+    const auto len = std::min<size_t>(chars.size(), colLimit - colBeg);
     size_t ch = chBeg;
 
-    while (it != end)
+    for (size_t off = 0; off < len; ++off)
     {
-        if (*it >= 0x80) [[unlikely]]
+        if (chars[off] >= 0x80) [[unlikely]]
         {
-            _replaceTextUnicode(ch, it);
+            _replaceTextUnicode(ch, off);
             return;
         }
 
         til::at(row._charOffsets, colEnd) = gsl::narrow_cast<uint16_t>(ch);
         ++colEnd;
         ++ch;
-        ++it;
     }
 
     colEndDirty = colEnd;
     charsConsumed = ch - chBeg;
 }
 
-[[msvc::forceinline]] void ROW::WriteHelper::_replaceTextUnicode(size_t ch, std::wstring_view::const_iterator it) noexcept
+[[msvc::forceinline]] void ROW::WriteHelper::_replaceTextUnicode(size_t ch, size_t off)
 {
-    const auto end = chars.end();
+    auto& cwd = CodepointWidthDetector::Singleton();
+    const auto len = chars.size();
 
-    while (it != end)
+    // The non-ASCII character we have encountered may be a combining mark, like "a^" which is then displayed as "â".
+    // In order to recognize both characters as a single grapheme, we need to back up by 1 ASCII character
+    // and let MeasureNext() find the next proper grapheme boundary.
+    if (off != 0)
     {
-        unsigned int width = 1;
-        auto ptr = &*it;
-        const auto wch = *ptr;
-        size_t advance = 1;
-
-        ++it;
-
-        // Even in our slow-path we can avoid calling IsGlyphFullWidth if the current character is ASCII.
-        // It also allows us to skip the surrogate pair decoding at the same time.
-        if (wch >= 0x80)
-        {
-            if (til::is_surrogate(wch))
-            {
-                if (it != end && til::is_leading_surrogate(wch) && til::is_trailing_surrogate(*it))
-                {
-                    advance = 2;
-                    ++it;
-                }
-                else
-                {
-                    ptr = &UNICODE_REPLACEMENT;
-                }
-            }
+        --colEnd;
+        --ch;
+        --off;
+    }
 
-            width = IsGlyphFullWidth({ ptr, advance }) + 1u;
-        }
+    while (off < len)
+    {
+        int width;
+        const auto end = cwd.GraphemeNext(chars, off, &width);
 
         const auto colEndNew = gsl::narrow_cast<uint16_t>(colEnd + width);
         if (colEndNew > colLimit)
@@ -719,7 +702,8 @@ catch (...)
             til::at(row._charOffsets, colEnd++) = gsl::narrow_cast<uint16_t>(ch | CharOffsetsTrailer);
         }
 
-        ch += advance;
+        ch += end - off;
+        off = end;
     }
 
     colEndDirty = colEnd;
@@ -1062,7 +1046,7 @@ std::wstring_view ROW::GetText() const noexcept
 
 std::wstring_view ROW::GetText(til::CoordType columnBegin, til::CoordType columnEnd) const noexcept
 {
-    const til::CoordType columns = _columnCount;
+    const auto columns = GetReadableColumnCount();
     const auto colBeg = clamp(columnBegin, 0, columns);
     const auto colEnd = clamp(columnEnd, colBeg, columns);
     const size_t chBeg = _uncheckedCharOffset(gsl::narrow_cast<size_t>(colBeg));

src/buffer/out/Row.hpp

@@ -181,7 +181,7 @@ class ROW final
         bool IsValid() const noexcept;
         void ReplaceCharacters(til::CoordType width) noexcept;
         void ReplaceText() noexcept;
-        void _replaceTextUnicode(size_t ch, std::wstring_view::const_iterator it) noexcept;
+        void _replaceTextUnicode(size_t ch, size_t off);
         void CopyTextFrom(const std::span<const uint16_t>& charOffsets) noexcept;
         static void _copyOffsets(uint16_t* dst, const uint16_t* src, uint16_t size, uint16_t offset) noexcept;
         void Finish();

src/buffer/out/textBuffer.cpp

@@ -2,16 +2,13 @@
 // Licensed under the MIT license.
 
 #include "precomp.h"
-
 #include "textBuffer.hpp"
 
 #include <til/hash.h>
-#include <til/unicode.h>
 
 #include "UTextAdapter.h"
-#include "../../types/inc/GlyphWidth.hpp"
+#include "../../types/inc/CodepointWidthDetector.hpp"
 #include "../renderer/base/renderer.hpp"
-#include "../types/inc/convert.hpp"
 #include "../types/inc/utils.hpp"
 
 using namespace Microsoft::Console;
@@ -408,17 +405,17 @@ void TextBuffer::_PrepareForDoubleByteSequence(const DbcsAttribute dbcsAttribute
 // Given the character offset `position` in the `chars` string, this function returns the starting position of the next grapheme.
 // For instance, given a `chars` of L"x\uD83D\uDE42y" and a `position` of 1 it'll return 3.
 // GraphemePrev would do the exact inverse of this operation.
-// In the future, these functions are expected to also deliver information about how many columns a grapheme occupies.
-// (I know that mere UTF-16 code point iteration doesn't handle graphemes, but that's what we're working towards.)
 size_t TextBuffer::GraphemeNext(const std::wstring_view& chars, size_t position) noexcept
 {
-    return til::utf16_iterate_next(chars, position);
+    auto& cwd = CodepointWidthDetector::Singleton();
+    return cwd.GraphemeNext(chars, position, nullptr);
 }
 
 // It's the counterpart to GraphemeNext. See GraphemeNext.
 size_t TextBuffer::GraphemePrev(const std::wstring_view& chars, size_t position) noexcept
 {
-    return til::utf16_iterate_prev(chars, position);
+    auto& cwd = CodepointWidthDetector::Singleton();
+    return cwd.GraphemePrev(chars, position, nullptr);
 }
 
 // Ever wondered how much space a piece of text needs before inserting it? This function will tell you!
@@ -445,7 +442,7 @@ size_t TextBuffer::FitTextIntoColumns(const std::wstring_view& chars, til::Coord
     {
     }
 
-    const auto dist = gsl::narrow_cast<size_t>(it - beg);
+    auto dist = gsl::narrow_cast<size_t>(it - beg);
     auto col = gsl::narrow_cast<til::CoordType>(dist);
 
     if (it == asciiEnd) [[likely]]
@@ -455,33 +452,23 @@ size_t TextBuffer::FitTextIntoColumns(const std::wstring_view& chars, til::Coord
     }
 
     // Unicode slow-path where we need to count text and columns separately.
-    for (;;)
-    {
-        auto ptr = &*it;
-        const auto wch = *ptr;
-        size_t len = 1;
-
-        col++;
+    auto& cwd = CodepointWidthDetector::Singleton();
+    const auto len = chars.size();
 
-        // Even in our slow-path we can avoid calling IsGlyphFullWidth if the current character is ASCII.
-        // It also allows us to skip the surrogate pair decoding at the same time.
-        if (wch >= 0x80)
-        {
-            if (til::is_surrogate(wch))
-            {
-                const auto it2 = it + 1;
-                if (til::is_leading_surrogate(wch) && it2 != end && til::is_trailing_surrogate(*it2))
-                {
-                    len = 2;
-                }
-                else
-                {
-                    ptr = &UNICODE_REPLACEMENT;
-                }
-            }
+    // The non-ASCII character we have encountered may be a combining mark, like "a^" which is then displayed as "â".
+    // In order to recognize both characters as a single grapheme, we need to back up by 1 ASCII character
+    // and let GraphemeNext() find the next proper grapheme boundary.
+    if (dist != 0)
+    {
+        dist--;
+        col--;
+    }
 
-            col += IsGlyphFullWidth({ ptr, len });
-        }
+    while (dist < len)
+    {
+        int width;
+        dist = cwd.GraphemeNext(chars, dist, &width);
+        col += width;
 
         // If we ran out of columns, we need to always return `columnLimit` and not `cols`,
         // because if we tried inserting a wide glyph into just 1 remaining column it will
@@ -490,17 +477,13 @@ size_t TextBuffer::FitTextIntoColumns(const std::wstring_view& chars, til::Coord
         if (col > columnLimit)
         {
             columns = columnLimit;
-            return gsl::narrow_cast<size_t>(it - beg);
-        }
-
-        // But if we simply ran out of text we just need to return the actual number of columns.
-        it += len;
-        if (it == end)
-        {
-            columns = col;
-            return chars.size();
+            return dist;
         }
     }
+
+    // But if we simply ran out of text we just need to return the actual number of columns.
+    columns = col;
+    return chars.size();
 }
 
 // Pretend as if `position` is a regular cursor in the TextBuffer.

src/cascadia/TerminalCore/ICoreSettings.idl

@@ -20,6 +20,7 @@ namespace Microsoft.Terminal.Core
         String WordDelimiters;
 
         Boolean ForceVTInput;
+        Boolean Graphemes;
         Boolean TrimBlockSelection;
         Boolean DetectURLs;
         Boolean VtPassthrough;