uic/Python: Handle surrogates correctly

When a QString has surrogates (2 consecutive QChar's in UTF-16),
convert to UCS4 first.

Rewrite the formatStringSequence() helper (which operates
on sequences of char/QChar) and related helpers to operate
on uint instead of ushort and remove the non-type template parameter
specifying the encoding by introducing a characterCode() converter
helper to convert to uint.

Check the QString passed in for surrogates and convert to UCS-4
if they are found (using the 32bit \U escape marker for Python).

Pick-to: 6.10 6.9 6.8
Fixes: PYSIDE-3173
Change-Id: Ie1d1282d78ad80894cf67dd47ea6a332d7dfda25
Reviewed-by: Cristian Maureira-Fredes <cristian.maureira-fredes@qt.io>

src/tools/uic/shared/language.cpp

@@ -6,6 +6,8 @@
 #include <QtCore/qtextstream.h>
 #include <QtCore/QList>
 
+#include <algorithm>
+
 namespace language {
 
 using namespace Qt::StringLiterals;
@@ -182,7 +184,7 @@ QLatin1StringView paletteColorRole(int v)
 // Helpers for formatting a character sequences
 
 // Format a special character like '\x0a'
-static int formatEscapedNumber(QTextStream &str, ushort value, int base, int width,
+static int formatEscapedNumber(QTextStream &str, uint value, int base, int width,
                                char prefix = 0)
 {
     int length = 1 + width;
@@ -236,24 +238,23 @@ static int formatSpecialCharacter(QTextStream &str, ushort value)
 
 enum : int { maxSegmentSize = 1024 };
 
-template <Encoding e>
-struct FormattingTraits
-{
-};
 
-template <>
+static uint characterCode(char c)
-struct FormattingTraits<Encoding::Utf8>
 {
-    static ushort code(char c) { return uchar(c); }
+    return uchar(c);
-};
+}
 
-template <>
+static uint characterCode(QChar c)
-struct FormattingTraits<Encoding::Unicode>
 {
-    static ushort code(QChar c) { return c.unicode(); }
+    return c.unicode();
-};
+}
 
-template <Encoding e, class Iterator>
+static uint characterCode(uint c)
+{
+    return c;
+}
+
+template <class Iterator>
 static void formatStringSequence(QTextStream &str, Iterator it, Iterator end,
                                  const QString &indent,
                                  int escapeIntegerBase, int escapeWidth,
@@ -262,13 +263,13 @@ static void formatStringSequence(QTextStream &str, Iterator it, Iterator end,
     str << '"';
     int length = 0;
     while (it != end) {
-        const auto code = FormattingTraits<e>::code(*it);
+        const auto code = characterCode(*it);
         if (code >= 0x80) {
             length += formatEscapedNumber(str, code, escapeIntegerBase, escapeWidth, escapePrefix);
         } else if (const int l = formatSpecialCharacter(str, code)) {
             length += l;
         } else if (code != '\r') {
-            str << *it;
+            str << char(code);
             ++length;
         }
         ++it;
@@ -280,6 +281,11 @@ static void formatStringSequence(QTextStream &str, Iterator it, Iterator end,
     str << '"';
 }
 
+static bool isSurrogate(QChar c)
+{
+    return c.isSurrogate();
+}
+
 void _formatString(QTextStream &str, const QString &value, const QString &indent,
                    bool qString)
 {
@@ -289,17 +295,20 @@ void _formatString(QTextStream &str, const QString &value, const QString &indent
         if (qString && _language == Language::Cpp)
             str << "QString::fromUtf8(";
         const QByteArray utf8 = value.toUtf8();
-        formatStringSequence<Encoding::Utf8>(str, utf8.cbegin(), utf8.cend(), indent,
+        formatStringSequence(str, utf8.cbegin(), utf8.cend(), indent, 8, 3);
-                                             8, 3);
         if (qString && _language == Language::Cpp)
             str << ')';
     }
         break;
     // Special characters as 4 digit hex Unicode points (u8"\u00dcmlaut")
     case Encoding::Unicode:
-        str << 'u'; // Python Unicode literal (would be UTF-16 in C++)
+        str << 'u'; // Python Unicode literal
-        formatStringSequence<Encoding::Unicode>(str, value.cbegin(), value.cend(), indent,
+        if (std::any_of(value.cbegin(), value.cend(), isSurrogate)) {
-                                                16, 4, 'u');
+            const auto ucs4 = value.toUcs4();
+            formatStringSequence(str, ucs4.cbegin(), ucs4.cend(), indent, 16, 8, 'U');
+        } else {
+            formatStringSequence(str, value.cbegin(), value.cend(), indent, 16, 4, 'u');
+        }
         break;
     }
 }

tests/auto/tools/uic/baseline/config.ui

@@ -329,7 +329,7 @@
    <item>
     <widget class="QCheckBox" name="touchScreen">
      <property name="text">
-      <string>Emulate touch screen (no mouse move)</string>
+      <string>Emulate touch screen (no mouse move)🚀</string>
      </property>
     </widget>
    </item>

tests/auto/tools/uic/baseline/config.ui.h

@@ -704,7 +704,7 @@ public:
         TextLabel1_3->setText(QCoreApplication::translate("Config", "Skin", nullptr));
         skin->setItemText(0, QCoreApplication::translate("Config", "None", nullptr));
 
-        touchScreen->setText(QCoreApplication::translate("Config", "Emulate touch screen (no mouse move)", nullptr));
+        touchScreen->setText(QCoreApplication::translate("Config", "Emulate touch screen (no mouse move)\360\237\232\200", nullptr));
         lcdScreen->setText(QCoreApplication::translate("Config", "Emulate LCD screen (Only with fixed zoom of 3.0 times magnification)", nullptr));
         TextLabel1->setText(QCoreApplication::translate("Config", "<p>Note that any applications using the virtual framebuffer will be terminated if you change the Size or Depth <i>above</i>. You may freely modify the Gamma <i>below</i>.", nullptr));
         GroupBox1->setTitle(QCoreApplication::translate("Config", "Gamma", nullptr));

tests/auto/tools/uic/baseline/config.ui.py

@@ -638,7 +638,7 @@ class Ui_Config(object):
         self.TextLabel1_3.setText(QCoreApplication.translate("Config", u"Skin", None))
         self.skin.setItemText(0, QCoreApplication.translate("Config", u"None", None))
 
-        self.touchScreen.setText(QCoreApplication.translate("Config", u"Emulate touch screen (no mouse move)", None))
+        self.touchScreen.setText(QCoreApplication.translate("Config", u"Emulate touch screen (no mouse move)\U0001f680", None))
         self.lcdScreen.setText(QCoreApplication.translate("Config", u"Emulate LCD screen (Only with fixed zoom of 3.0 times magnification)", None))
         self.TextLabel1.setText(QCoreApplication.translate("Config", u"<p>Note that any applications using the virtual framebuffer will be terminated if you change the Size or Depth <i>above</i>. You may freely modify the Gamma <i>below</i>.", None))
         self.GroupBox1.setTitle(QCoreApplication.translate("Config", u"Gamma", None))