Add protobuf message registry and raw QProtobufMessage serialization

This allows to create messages using the full protobuf message
name, if the message was registered.

Using this mapping between QMetaType and QProtobufOrdering it's
now possible to serialize and deserialize pointers to QProtobufMessage
without casting QProtobufMessage to the protobuf message type.

This is partially required for conformance testing and Any type
handling.

Change-Id: I58fd2d821b76ce3f59f121b65ad28c2b28a189e7
Reviewed-by: Mårten Nordheim <marten.nordheim@qt.io>

src/protobuf/qabstractprotobufserializer.cpp

@@ -107,4 +107,25 @@ bool QAbstractProtobufSerializer::doDeserialize(QProtobufMessage *message,
     assumed to not be \nullptr.
     Returns \c true if deserialization was successful, otherwise \c false.
 */
+
+namespace QtProtobufPrivate {
+extern QtProtobufPrivate::QProtobufPropertyOrdering getOrderingByMetaType(QMetaType type);
+}
+
+QByteArray QAbstractProtobufSerializer::serializeRawMessage(const QProtobufMessage *message) const
+{
+    Q_ASSERT(message != nullptr && message->metaObject() != nullptr);
+    auto ordering = QtProtobufPrivate::getOrderingByMetaType(message->metaObject()->metaType());
+    Q_ASSERT(ordering.data != nullptr);
+    return serializeMessage(message, ordering);
+}
+
+bool QAbstractProtobufSerializer::deserializeRawMessage(QProtobufMessage *message, QByteArrayView data) const
+{
+    Q_ASSERT(message != nullptr && message->metaObject() != nullptr);
+    auto ordering = QtProtobufPrivate::getOrderingByMetaType(message->metaObject()->metaType());
+    Q_ASSERT(ordering.data != nullptr);
+    return deserializeMessage(message, ordering, data);
+}
+
 QT_END_NAMESPACE

src/protobuf/qabstractprotobufserializer.h

@@ -52,6 +52,9 @@ public:
     virtual QAbstractProtobufSerializer::DeserializationError deserializationError() const = 0;
     virtual QString deserializationErrorString() const = 0;
 
+    QByteArray serializeRawMessage(const QProtobufMessage *message) const;
+    bool deserializeRawMessage(QProtobufMessage *message, QByteArrayView data) const;
+
 protected:
     virtual QByteArray
     serializeMessage(const QProtobufMessage *message,

src/protobuf/qprotobufmessage.cpp

@@ -125,6 +125,27 @@ bool QProtobufMessage::isEqual(const QProtobufMessage &lhs, const QProtobufMessa
     return lhs.d_func()->unknownEntries == rhs.d_func()->unknownEntries;
 }
 
+namespace QtProtobufPrivate {
+/*!
+    \internal
+*/
+extern QProtobufMessage *constructMessageByName(const QString &messageType);
+}
+
+/*!
+    Constructs QProtobufMessage using \a messageType.
+    Returns a pointer to the constructed QProtobufMessage.
+
+    This function attempts to create a message whose type matches \a messageType. If \a messageType
+    is unknown, the function returns \nullptr. If the message is not found in the registry, the
+    function returns \nullptr.
+    Ownership of the constructed message is given to the function caller.
+*/
+QProtobufMessage *QProtobufMessage::constructByName(const QString &messageType)
+{
+    return QtProtobufPrivate::constructMessageByName(messageType);
+}
+
 QT_END_NAMESPACE
 
 #include "moc_qprotobufmessage.cpp"

src/protobuf/qprotobufmessage.h

@@ -21,6 +21,8 @@ public:
     QVariant property(QAnyStringView propertyName) const;
     bool setProperty(QAnyStringView propertyName, const QVariant &value);
 
+    Q_REQUIRED_RESULT static QProtobufMessage *constructByName(const QString &messageType);
+
 protected:
     explicit QProtobufMessage(const QMetaObject *metaObject);
     QProtobufMessage(const QProtobufMessage &other);
@@ -37,6 +39,7 @@ private:
     const QMetaObject *metaObject() const;
 
     friend class QProtobufSerializer;
+    friend class QAbstractProtobufSerializer;
     friend class QProtobufSerializerPrivate;
     friend class QAbstractProtobufSerializer;
 

src/protobuf/qprotobufserializer.h

@@ -276,6 +276,7 @@ void deserializeEnumList(const QProtobufSerializer *serializer, QProtobufSelfche
 template<typename T>
 inline void qRegisterProtobufType() {
     T::registerTypes();
+    QtProtobufPrivate::registerOrdering(QMetaType::fromType<T>(), T::propertyOrdering);
     QtProtobufPrivate::registerHandler(QMetaType::fromType<T *>(), { QtProtobufPrivate::serializeObject<T>,
             QtProtobufPrivate::deserializeObject<T> });
     QtProtobufPrivate::registerHandler(QMetaType::fromType<QList<std::shared_ptr<T>>>(), { QtProtobufPrivate::serializeList<T>,

src/protobuf/qtprotobuftypes.cpp

@@ -4,7 +4,11 @@
 
 #include <QtProtobuf/qtprotobufglobal.h>
 
+#include <QtCore/qreadwritelock.h>
 #include <QtCore/qmutex.h>
+#include <QtCore/qpair.h>
+
+#include <QtProtobuf/private/qtprotobuflogging_p.h>
 
 #include "qtprotobuftypes.h"
 #include "qprotobufobject.h"
@@ -14,7 +18,60 @@
 
 QT_BEGIN_NAMESPACE
 
-#define registerProtobufType(X) qRegisterMetaType<X>(#X)
+namespace {
+/*
+    \internal
+    \brief The ProtobufOrderingRegistry stores the mapping between
+           QtProtobufPrivate::QProtobufPropertyOrdering and QMetaType.
+*/
+struct ProtobufOrderingRegistry
+{
+    using ProtobufOrderingRegistryRecord =
+                                    QPair<QMetaType, QtProtobufPrivate::QProtobufPropertyOrdering>;
+
+    void registerOrdering(QMetaType type, QtProtobufPrivate::QProtobufPropertyOrdering ordering)
+    {
+        QWriteLocker locker(&m_lock);
+        m_registry[ordering.getMessageFullName().toString()] =
+                ProtobufOrderingRegistryRecord(type, ordering);
+    }
+
+    QtProtobufPrivate::QProtobufPropertyOrdering getOrdering(QMetaType type) const
+    {
+        QtProtobufPrivate::QProtobufPropertyOrdering ordering{ nullptr };
+
+        QReadLocker locker(&m_lock);
+        for (auto it = m_registry.constBegin(); it != m_registry.constEnd(); ++it) {
+            if (type == it.value().first) {
+                ordering = it.value().second;
+                break;
+            }
+        }
+
+        return ordering;
+    }
+
+    ProtobufOrderingRegistryRecord
+    findMessageByName(const QString &messageName) const
+    {
+        ProtobufOrderingRegistryRecord record = {
+            QMetaType(QMetaType::UnknownType), { nullptr }
+        };
+
+        QReadLocker locker(&m_lock);
+        auto it = m_registry.constFind(messageName);
+        if (it != m_registry.constEnd())
+            record = it.value();
+        return record;
+    }
+
+private:
+    mutable QReadWriteLock m_lock;
+    QHash<QString, ProtobufOrderingRegistryRecord> m_registry;
+};
+
+Q_GLOBAL_STATIC(ProtobufOrderingRegistry, orderingRegistry)
+}
 
 namespace QtProtobufPrivate {
     constexpr uint jsonNameOffsetsOffset = 0;
@@ -109,6 +166,31 @@ namespace QtProtobufPrivate {
                  || (offset == jsonNameOffsetsOffset && uint(index) == data->numFields));
         return *(uint_data() + offset + index);
     }
+
+    void registerOrdering(QMetaType type, QProtobufPropertyOrdering ordering)
+    {
+        orderingRegistry->registerOrdering(type, ordering);
+    }
+
+    QProtobufPropertyOrdering getOrderingByMetaType(QMetaType type)
+    {
+        return orderingRegistry->getOrdering(type);
+    }
+
+    QProtobufMessage *constructMessageByName(const QString &messageType)
+    {
+        qRegisterProtobufTypes();
+        ProtobufOrderingRegistry::ProtobufOrderingRegistryRecord messageOrderingRecord =
+                orderingRegistry->findMessageByName(messageType);
+        QMetaType type = messageOrderingRecord.first;
+        if (type.id() != QMetaType::UnknownType) {
+            void *msg = type.create();
+            return reinterpret_cast<QProtobufMessage *>(msg);
+        }
+        qProtoWarning() << "Unable to find protobuf message with name" << messageType
+                        << ". Message is not registered.";
+        return nullptr;
+    }
 }
 
 namespace QtProtobuf {
@@ -155,32 +237,32 @@ static void qRegisterBaseTypes()
 {
     [[maybe_unused]] // definitely unused
     static bool registered = [] {
-        registerProtobufType(QtProtobuf::int32);
+        qRegisterMetaType<QtProtobuf::int32>();
-        registerProtobufType(QtProtobuf::int64);
+        qRegisterMetaType<QtProtobuf::int64>();
-        registerProtobufType(QtProtobuf::uint32);
+        qRegisterMetaType<QtProtobuf::uint32>();
-        registerProtobufType(QtProtobuf::uint64);
+        qRegisterMetaType<QtProtobuf::uint64>();
-        registerProtobufType(QtProtobuf::sint32);
+        qRegisterMetaType<QtProtobuf::sint32>();
-        registerProtobufType(QtProtobuf::sint64);
+        qRegisterMetaType<QtProtobuf::sint64>();
-        registerProtobufType(QtProtobuf::fixed32);
+        qRegisterMetaType<QtProtobuf::fixed32>();
-        registerProtobufType(QtProtobuf::fixed64);
+        qRegisterMetaType<QtProtobuf::fixed64>();
-        registerProtobufType(QtProtobuf::sfixed32);
+        qRegisterMetaType<QtProtobuf::sfixed32>();
-        registerProtobufType(QtProtobuf::sfixed64);
+        qRegisterMetaType<QtProtobuf::sfixed64>();
-        registerProtobufType(QtProtobuf::boolean);
+        qRegisterMetaType<QtProtobuf::boolean>();
 
-        registerProtobufType(QtProtobuf::int32List);
+        qRegisterMetaType<QtProtobuf::int32List>();
-        registerProtobufType(QtProtobuf::int64List);
+        qRegisterMetaType<QtProtobuf::int64List>();
-        registerProtobufType(QtProtobuf::uint32List);
+        qRegisterMetaType<QtProtobuf::uint32List>();
-        registerProtobufType(QtProtobuf::uint64List);
+        qRegisterMetaType<QtProtobuf::uint64List>();
-        registerProtobufType(QtProtobuf::sint32List);
+        qRegisterMetaType<QtProtobuf::sint32List>();
-        registerProtobufType(QtProtobuf::sint64List);
+        qRegisterMetaType<QtProtobuf::sint64List>();
-        registerProtobufType(QtProtobuf::fixed32List);
+        qRegisterMetaType<QtProtobuf::fixed32List>();
-        registerProtobufType(QtProtobuf::fixed64List);
+        qRegisterMetaType<QtProtobuf::fixed64List>();
-        registerProtobufType(QtProtobuf::sfixed32List);
+        qRegisterMetaType<QtProtobuf::sfixed32List>();
-        registerProtobufType(QtProtobuf::sfixed64List);
+        qRegisterMetaType<QtProtobuf::sfixed64List>();
 
-        registerProtobufType(QtProtobuf::doubleList);
+        qRegisterMetaType<QtProtobuf::doubleList>();
-        registerProtobufType(QtProtobuf::floatList);
+        qRegisterMetaType<QtProtobuf::floatList>();
-        registerProtobufType(QtProtobuf::boolList);
+        qRegisterMetaType<QtProtobuf::boolList>();
 
         QtProtobuf::registerBasicConverters<QtProtobuf::int32>();
         QtProtobuf::registerBasicConverters<QtProtobuf::int64>();

src/protobuf/qtprotobuftypes.h

@@ -11,6 +11,7 @@
 #include <QtCore/QHash>
 #include <QtCore/QMetaType>
 #include <QtCore/QtEndian>
+#include <QtProtobuf/QProtobufMessage>
 
 #include <memory>
 #include <functional>
@@ -49,6 +50,8 @@ private:
 };
 static_assert(std::is_trivially_destructible_v<QProtobufPropertyOrdering>);
 
+extern Q_PROTOBUF_EXPORT void registerOrdering(QMetaType type, QProtobufPropertyOrdering ordering);
+
 // Convenience structure to hold a reference to a single entry
 struct QProtobufPropertyOrderingInfo
 {
@@ -207,6 +210,7 @@ struct qMakeUnsignedImpl<int64> {
 };
 template <typename T>
 using qMakeUnsigned = typename qMakeUnsignedImpl<T>::type;
+
 } //namespace QtProtobuf
 
 Q_PROTOBUF_EXPORT void qRegisterProtobufTypes();

tests/auto/protobuf/basic/CMakeLists.txt

@@ -162,6 +162,15 @@ qt6_add_protobuf(tst_protobuf_non_packed_repeatedtypes
 qt_autogen_tools_initial_setup(tst_protobuf_non_packed_repeatedtypes)
 
 
+qt_internal_add_test(tst_protobuf_raw_serializers
+    SOURCES
+        tst_protobuf_raw_serializers.cpp
+    LIBRARIES
+        Qt::Test
+        Qt::ProtobufPrivate
+        tst_protobuf_basictypes_gen
+)
+
 if(UNIX AND NOT CMAKE_CROSSCOMPILING)
     qt_internal_add_test(tst_protobuf_internals
         SOURCES

tests/auto/protobuf/basic/tst_protobuf_basictypes.cpp

@@ -8,6 +8,8 @@
 #include <QSignalSpy>
 #include <QTest>
 
+#include <memory>
+
 #include <qtprotobuftestscommon.h>
 
 class QtProtobufTypesGenerationTest : public QObject
@@ -35,6 +37,8 @@ private slots:
     void AssignmentOperatorTest();
     void MoveOperatorTest();
     void AccessMessageFieldsFromGetter();
+
+    void InvalidMessageConstructorTest();
 };
 
 using namespace qtprotobufnamespace::tests;
@@ -45,6 +49,10 @@ void QtProtobufTypesGenerationTest::EmptyMessageTest()
     QCOMPARE(qtprotobufnamespace::tests::EmptyMessage::staticMetaObject.propertyCount(), 0);
     QCOMPARE(qtprotobufnamespace::tests::EmptyMessage::propertyOrdering.getMessageFullName(),
              "qtprotobufnamespace.tests.EmptyMessage");
+
+    std::unique_ptr<QProtobufMessage> rawMessage(
+            QProtobufMessage::constructByName("qtprotobufnamespace.tests.EmptyMessage"));
+    QVERIFY(reinterpret_cast<qtprotobufnamespace::tests::EmptyMessage*>(rawMessage.get()) != nullptr);
 }
 
 void QtProtobufTypesGenerationTest::BoolMessageTest()
@@ -284,6 +292,15 @@ void QtProtobufTypesGenerationTest::ComplexMessageTest()
                        .value<qtprotobufnamespace::tests::SimpleStringMessage *>()),
              stringMsg);
     QCOMPARE(test.testComplexField(), stringMsg);
+
+    std::unique_ptr<QProtobufMessage> rawObject(
+            QProtobufMessage::constructByName("qtprotobufnamespace.tests.ComplexMessage"));
+    auto *rawMessage = reinterpret_cast<qtprotobufnamespace::tests::ComplexMessage*>(rawObject.get());
+    QVERIFY(rawMessage);
+    QCOMPARE(rawMessage->testFieldInt(), 0);
+    qtprotobufnamespace::tests::SimpleStringMessage embeddedStringMessage =
+            rawMessage->testComplexField();
+    QCOMPARE(embeddedStringMessage.testFieldString(), QString());
 }
 
 void QtProtobufTypesGenerationTest::BytesMessageTest()
@@ -350,5 +367,13 @@ void QtProtobufTypesGenerationTest::AccessMessageFieldsFromGetter()
     QCOMPARE(actual, expected);
 }
 
+void QtProtobufTypesGenerationTest::InvalidMessageConstructorTest()
+{
+    std::unique_ptr<QProtobufMessage> message(QProtobufMessage::constructByName(
+            "qtprotobufnamespace.tests.InvalidMessageConstructorTestNotExists"));
+    QCOMPARE(message, nullptr);
+}
+
+
 QTEST_MAIN(QtProtobufTypesGenerationTest)
 #include "tst_protobuf_basictypes.moc"

tests/auto/protobuf/basic/tst_protobuf_raw_serializers.cpp

@@ -0,0 +1,54 @@
+// Copyright (C) 2022 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include "basicmessages.qpb.h"
+#include "fieldindexrange.qpb.h"
+
+#include <QtTest/QtTest>
+#include <QProtobufSerializer>
+
+#include <limits>
+
+class QtProtobufRawSerializersTest : public QObject
+{
+    Q_OBJECT
+private slots:
+    void init() { m_serializer.reset(new QProtobufSerializer); }
+    void ComplexMessageSerializeTest();
+    void ComplexMessageDeserializeTest();
+private:
+    std::unique_ptr<QProtobufSerializer> m_serializer;
+};
+
+class NonMessageQObject : public QObject
+{
+    Q_OBJECT
+};
+
+void QtProtobufRawSerializersTest::ComplexMessageSerializeTest()
+{
+    QProtobufMessage *rawMessage =
+            QProtobufMessage::constructByName("qtprotobufnamespace.tests.ComplexMessage");
+    m_serializer->deserializeRawMessage(
+                rawMessage, QByteArray::fromHex("1208320671776572747908d3ffffffffffffffff01"));
+    auto *message = reinterpret_cast<qtprotobufnamespace::tests::ComplexMessage *>(rawMessage);
+    QCOMPARE(message->testFieldInt(), -45);
+    QCOMPARE(message->testComplexField().testFieldString(), QLatin1String("qwerty"));
+    delete rawMessage;
+}
+
+void QtProtobufRawSerializersTest::ComplexMessageDeserializeTest()
+{
+    QProtobufMessage *rawMessage =
+            QProtobufMessage::constructByName("qtprotobufnamespace.tests.ComplexMessage");
+    auto *message = reinterpret_cast<qtprotobufnamespace::tests::ComplexMessage *>(rawMessage);
+    message->setTestFieldInt(-45);
+    message->testComplexField().setTestFieldString(QLatin1String("qwerty"));
+
+    QByteArray buffer = m_serializer->serializeRawMessage(rawMessage);
+    QCOMPARE(buffer.toHex(), "08d3ffffffffffffffff0112083206717765727479");
+    delete rawMessage;
+}
+
+QTEST_MAIN(QtProtobufRawSerializersTest)
+#include "tst_protobuf_raw_serializers.moc"