wip

Signed-off-by: Christian Parpart <[email protected]>

Author: christianparpart

src/libunicode/grapheme_line_segmenter.cpp

@@ -84,27 +84,25 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
     if (_buffer.empty())
         return result_type { .text = _buffer.substr(0, 0), .width = 0 };
 
-    char const* start = _next;
-    char const* const resultStart = _utf8.expectedLength ? start - _utf8.currentLength : start;
+    if (_next == _buffer.data() + _buffer.size())
+        return result_type { .text = _buffer.substr(0, 0), .width = 0 };
+
+    // Points to the beginning of a grapheme cluster.
+    char const* const resultStart = _utf8.expectedLength ? _next - _utf8.currentLength : _next;
 
     // Number of bytes used in the current line.
     size_t totalByteCountProcessed = 0;
 
     // Number of width used in the current line.
     unsigned totalWidthProcessed = 0;
 
-    auto const makeResult = [&]() -> result_type {
-        return result_type { .text = std::string_view(resultStart, totalByteCountProcessed),
-                             .width = totalWidthProcessed };
-    };
-
     enum class State
     {
         ASCII,
         ComplexUnicode,
     };
 
-    while (maxWidth > 0 && !_buffer.empty())
+    while (maxWidth > 0 && _next != _buffer.data() + _buffer.size())
     {
         State const state =
             (_utf8.expectedLength != 0 || is_complex(_buffer.front())) ? State::ComplexUnicode : State::ASCII;
@@ -114,7 +112,8 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
             case State::ASCII: {
                 auto const count = process_ascii(maxWidth);
                 if (count == 0)
-                    return makeResult();
+                    return result_type { .text = std::string_view { resultStart, totalByteCountProcessed },
+                                         .width = totalWidthProcessed };
                 _events.on_ascii(_buffer.substr(0, count));
                 maxWidth -= count;
                 totalWidthProcessed += count;
@@ -127,7 +126,8 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
                 if (sub.graphemeClusterCount == 0)
                 {
                     _next += sub.byteCount;
-                    return makeResult();
+                    return result_type { .text = std::string_view { resultStart, totalByteCountProcessed },
+                                         .width = totalWidthProcessed };
                 }
                 maxWidth -= sub.graphemeClusterCount;
                 totalWidthProcessed += sub.graphemeClusterCount;
@@ -138,7 +138,8 @@ grapheme_line_segmenter::result_type grapheme_line_segmenter::process(unsigned m
         }
     }
 
-    return makeResult();
+    return result_type { .text = std::string_view { resultStart, totalByteCountProcessed },
+                         .width = totalWidthProcessed };
 }
 
 unsigned grapheme_line_segmenter::process_ascii(unsigned maxWidth) const noexcept
@@ -231,10 +232,12 @@ auto grapheme_line_segmenter::process_complex_unicode(unsigned maxWidth) noexcep
         {
             auto const prevCodepoint = _lastCodepointHint;
             auto const nextCodepoint = std::get<Success>(result).value;
-            auto const nextWidth = std::max(_currentClusterWidth, static_cast<unsigned>(unicode::width(nextCodepoint)));
+            auto const nextWidth =
+                std::max(_currentClusterWidth, static_cast<unsigned>(unicode::width(nextCodepoint)));
             _lastCodepointHint = nextCodepoint;
             if (grapheme_segmenter::breakable(prevCodepoint, nextCodepoint))
             {
+                printf("breakable: 0x%04X -> 0x%04X\n", prevCodepoint, nextCodepoint);
                 // Flush out current grapheme cluster's East Asian Width.
                 consumedWidth += _currentClusterWidth;
                 maxWidth -= _currentClusterWidth;
@@ -259,6 +262,7 @@ auto grapheme_line_segmenter::process_complex_unicode(unsigned maxWidth) noexcep
             }
             else
             {
+                printf("NOT breakable: 0x%04X -> 0x%04X\n", prevCodepoint, nextCodepoint);
                 lastClusterEnd = input;
                 // Increase width on VS16 but do not decrease on VS15.
                 if (nextCodepoint == 0xFE0F) // VS16

src/libunicode/grapheme_line_segmenter_test.cpp

@@ -17,7 +17,7 @@
 
 #include <fmt/format.h>
 
-#include <catch2/catch.hpp>
+#include <catch2/catch_test_macros.hpp>
 
 #include <string_view>
 #include <variant>

src/tools/unicode-query.cpp

@@ -15,8 +15,10 @@
 #include <libunicode/codepoint_properties_loader.h>
 #include <libunicode/convert.h>
 #include <libunicode/grapheme_segmenter.h>
+#include <libunicode/run_segmenter.h>
 #include <libunicode/ucd.h>
 #include <libunicode/ucd_enums.h>
+#include <libunicode/ucd_fmt.h>
 #include <libunicode/ucd_ostream.h>
 #include <libunicode/utf8_grapheme_segmenter.h>
 
@@ -33,27 +35,62 @@ using namespace std;
 namespace
 {
 
-std::string quotedAndEscaped(std::string const& text)
+std::string escapeControlCodes(std::string const& text)
+{
+    auto result = stringstream {};
+    for (char const ch: text)
+    {
+        if (ch < 0x20)
+            result << "\\x" << setw(2) << std::hex << (unsigned(ch) & 0xFF);
+        else
+            result << ch;
+    }
+    return result.str();
+}
+
+std::string escaped(std::string const& text)
 {
     auto result = stringstream {};
-    result << '"';
     for (char const ch: text)
     {
         if (std::isprint(ch) && ch != '"')
             result << ch;
         else
             result << "\\x" << setw(2) << std::hex << (unsigned(ch) & 0xFF);
     }
-    result << "\"";
     return result.str();
 }
 
+std::string quotedAndEscaped(std::string const& text)
+{
+    return '"' + escaped(text) + '"';
+}
+
 int printUsage(int exitCode)
 {
-    cout << "unicode-query [properties] U+XXXX [...]\n";
+    cout << "unicode-query [properties] U+XXXX [...]\n"
+         << "              gc [-e] [--] \"Text string\"\n"
+         << "              runs [-e] [--] \"Text string\"\n";
     return exitCode;
 }
 
+std::string_view seq(std::string_view const& text)
+{
+    static const bool isTTY = []() {
+        #if !defined(_WIN32)
+        auto const isPTY = isatty(STDOUT_FILENO);
+        return isPTY;
+        #else
+        return false;
+        #endif
+    }();
+    if (isTTY)
+        return text;
+    else
+        return {};
+}
+
+// {{{ properties
 optional<char32_t> parseChar(std::string_view text)
 {
     if (text.size() >= 3 && text[0] == 'U' && text[1] == '+')
@@ -116,7 +153,7 @@ void showCodepointProperties(char32_t codepoint)
     cout << "Emoji Segmentation Category : " << properties.emoji_segmentation_category << '\n';
     cout << "Grapheme Cluster Break      : " << properties.grapheme_cluster_break << '\n';
     cout << "\n";
-    // clang-format off
+    // clang-format on
 }
 
 int showCodepointProperties(int argc, char const* argv[])
@@ -134,7 +171,100 @@ int showCodepointProperties(int argc, char const* argv[])
     }
     return EXIT_SUCCESS;
 }
+// }}}
+
+// {{{ grapheme clusters
+int showGraphemeClusters(int argc, char const* argv[])
+{
+    int i = 0;
+    bool escapeText = false;
+    for (; i < argc; ++i)
+    {
+        auto const arg = string_view(argv[i]);
+        if (arg == "-e")
+            escapeText = true;
+        else if (arg == "--")
+        {
+            ++i;
+            break;
+        }
+        else if (arg.starts_with('-'))
+            return printUsage(EXIT_FAILURE);
+        else
+            break;
+    }
+    for (; i < argc; ++i)
+    {
+        auto const text = string_view(argv[i]);
+        auto const gcs = unicode::utf8_grapheme_segmenter(text);
+        for (auto const& gc: gcs)
+        {
+            auto const text32 = std::u32string_view(gc);
+            auto const text8 = unicode::convert_to<char>(text32);
+            std::cout << (escapeText ? escaped(text8) : escapeControlCodes(text8)) << "\n";
+        }
+    }
+    return EXIT_SUCCESS;
+}
+// }}}
 
+// {{{ runs
+int showRuns(istream& in, bool escapeRunText)
+{
+    string bytes((istreambuf_iterator<char>(in)), istreambuf_iterator<char>());
+    u32string const codepoints = unicode::convert_to<char32_t>(string_view(bytes));
+
+    unicode::run_segmenter rs(codepoints);
+    unicode::run_segmenter::range run;
+
+    while (rs.consume(unicode::out(run)))
+    {
+        auto const script = get<unicode::Script>(run.properties);
+        auto const presentationStyle = get<unicode::PresentationStyle>(run.properties);
+
+        auto const text32 = u32string_view(codepoints.data() + run.start, run.end - run.start);
+        auto const text8 = unicode::convert_to<char>(text32);
+        auto const textEscaped = escapeRunText ? escaped(text8) : escapeControlCodes(text8);
+
+        cout << run.start << "-" << run.end - 1 << " (" << run.end - run.start << "): " << script << " "
+             << presentationStyle << "\n"
+             << '"' << seq("\033[32m") << textEscaped << seq("\033[m")
+             << "\"\n\n";
+    }
+
+    return EXIT_SUCCESS;
+}
+
+int showRuns(int argc, char const* argv[])
+{
+    // [-e]
+    int i = 0;
+    bool escaped = false;
+    for (; i < argc; ++i)
+    {
+        auto const arg = string_view(argv[i]);
+        if (arg == "-e")
+            escaped = true;
+        else if (arg == "--")
+        {
+            ++i;
+            break;
+        }
+        else if (arg.starts_with('-'))
+            return printUsage(EXIT_FAILURE);
+        else
+            break;
+    }
+
+    for (; i < argc; ++i)
+    {
+        auto in = std::istringstream(argv[i]);
+        showRuns(in, escaped);
+    }
+
+    return EXIT_SUCCESS;
+}
+// }}}
 } // namespace
 
 // Example usage:
@@ -154,6 +284,18 @@ int main(int argc, char const* argv[])
     if (string_view(argv[argIndex]) == "help")
         return printUsage(EXIT_SUCCESS);
 
+    if (string_view(argv[argIndex]) == "runs")
+    {
+        ++argIndex;
+        return showRuns(argc - argIndex, argv + argIndex);
+    }
+
+    if (string_view(argv[argIndex]) == "gc")
+    {
+        ++argIndex;
+        return showGraphemeClusters(argc - argIndex, argv + argIndex);
+    }
+
     if (string_view(argv[argIndex]) == "properties")
         ++argIndex;