From 660608d6b0dd120592c1100a081a5e759f0adf7f Mon Sep 17 00:00:00 2001 From: Alexey Edelev Date: Wed, 5 Oct 2022 19:11:20 +0200 Subject: [PATCH] Add protobuf message registry and raw QProtobufMessage serialization MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This allows to create messages using the full protobuf message name, if the message was registered. Using this mapping between QMetaType and QProtobufOrdering it's now possible to serialize and deserialize pointers to QProtobufMessage without casting QProtobufMessage to the protobuf message type. This is partially required for conformance testing and Any type handling. Change-Id: I58fd2d821b76ce3f59f121b65ad28c2b28a189e7 Reviewed-by: Mårten Nordheim --- src/protobuf/qabstractprotobufserializer.cpp | 21 +++ src/protobuf/qabstractprotobufserializer.h | 3 + src/protobuf/qprotobufmessage.cpp | 21 +++ src/protobuf/qprotobufmessage.h | 3 + src/protobuf/qprotobufserializer.h | 1 + src/protobuf/qtprotobuftypes.cpp | 132 ++++++++++++++---- src/protobuf/qtprotobuftypes.h | 4 + tests/auto/protobuf/basic/CMakeLists.txt | 9 ++ .../basic/tst_protobuf_basictypes.cpp | 25 ++++ .../basic/tst_protobuf_raw_serializers.cpp | 54 +++++++ 10 files changed, 248 insertions(+), 25 deletions(-) create mode 100644 tests/auto/protobuf/basic/tst_protobuf_raw_serializers.cpp diff --git a/src/protobuf/qabstractprotobufserializer.cpp b/src/protobuf/qabstractprotobufserializer.cpp index ca75440e..2a5b242f 100644 --- a/src/protobuf/qabstractprotobufserializer.cpp +++ b/src/protobuf/qabstractprotobufserializer.cpp @@ -107,4 +107,25 @@ bool QAbstractProtobufSerializer::doDeserialize(QProtobufMessage *message, assumed to not be \nullptr. Returns \c true if deserialization was successful, otherwise \c false. */ + +namespace QtProtobufPrivate { +extern QtProtobufPrivate::QProtobufPropertyOrdering getOrderingByMetaType(QMetaType type); +} + +QByteArray QAbstractProtobufSerializer::serializeRawMessage(const QProtobufMessage *message) const +{ + Q_ASSERT(message != nullptr && message->metaObject() != nullptr); + auto ordering = QtProtobufPrivate::getOrderingByMetaType(message->metaObject()->metaType()); + Q_ASSERT(ordering.data != nullptr); + return serializeMessage(message, ordering); +} + +bool QAbstractProtobufSerializer::deserializeRawMessage(QProtobufMessage *message, QByteArrayView data) const +{ + Q_ASSERT(message != nullptr && message->metaObject() != nullptr); + auto ordering = QtProtobufPrivate::getOrderingByMetaType(message->metaObject()->metaType()); + Q_ASSERT(ordering.data != nullptr); + return deserializeMessage(message, ordering, data); +} + QT_END_NAMESPACE diff --git a/src/protobuf/qabstractprotobufserializer.h b/src/protobuf/qabstractprotobufserializer.h index 06a7d610..cc8b6f20 100644 --- a/src/protobuf/qabstractprotobufserializer.h +++ b/src/protobuf/qabstractprotobufserializer.h @@ -52,6 +52,9 @@ public: virtual QAbstractProtobufSerializer::DeserializationError deserializationError() const = 0; virtual QString deserializationErrorString() const = 0; + QByteArray serializeRawMessage(const QProtobufMessage *message) const; + bool deserializeRawMessage(QProtobufMessage *message, QByteArrayView data) const; + protected: virtual QByteArray serializeMessage(const QProtobufMessage *message, diff --git a/src/protobuf/qprotobufmessage.cpp b/src/protobuf/qprotobufmessage.cpp index 6f99fd65..112eab3d 100644 --- a/src/protobuf/qprotobufmessage.cpp +++ b/src/protobuf/qprotobufmessage.cpp @@ -125,6 +125,27 @@ bool QProtobufMessage::isEqual(const QProtobufMessage &lhs, const QProtobufMessa return lhs.d_func()->unknownEntries == rhs.d_func()->unknownEntries; } +namespace QtProtobufPrivate { +/*! + \internal +*/ +extern QProtobufMessage *constructMessageByName(const QString &messageType); +} + +/*! + Constructs QProtobufMessage using \a messageType. + Returns a pointer to the constructed QProtobufMessage. + + This function attempts to create a message whose type matches \a messageType. If \a messageType + is unknown, the function returns \nullptr. If the message is not found in the registry, the + function returns \nullptr. + Ownership of the constructed message is given to the function caller. +*/ +QProtobufMessage *QProtobufMessage::constructByName(const QString &messageType) +{ + return QtProtobufPrivate::constructMessageByName(messageType); +} + QT_END_NAMESPACE #include "moc_qprotobufmessage.cpp" diff --git a/src/protobuf/qprotobufmessage.h b/src/protobuf/qprotobufmessage.h index e3a7eea3..86070b37 100644 --- a/src/protobuf/qprotobufmessage.h +++ b/src/protobuf/qprotobufmessage.h @@ -21,6 +21,8 @@ public: QVariant property(QAnyStringView propertyName) const; bool setProperty(QAnyStringView propertyName, const QVariant &value); + Q_REQUIRED_RESULT static QProtobufMessage *constructByName(const QString &messageType); + protected: explicit QProtobufMessage(const QMetaObject *metaObject); QProtobufMessage(const QProtobufMessage &other); @@ -37,6 +39,7 @@ private: const QMetaObject *metaObject() const; friend class QProtobufSerializer; + friend class QAbstractProtobufSerializer; friend class QProtobufSerializerPrivate; friend class QAbstractProtobufSerializer; diff --git a/src/protobuf/qprotobufserializer.h b/src/protobuf/qprotobufserializer.h index d343e55f..ce092d72 100644 --- a/src/protobuf/qprotobufserializer.h +++ b/src/protobuf/qprotobufserializer.h @@ -276,6 +276,7 @@ void deserializeEnumList(const QProtobufSerializer *serializer, QProtobufSelfche template inline void qRegisterProtobufType() { T::registerTypes(); + QtProtobufPrivate::registerOrdering(QMetaType::fromType(), T::propertyOrdering); QtProtobufPrivate::registerHandler(QMetaType::fromType(), { QtProtobufPrivate::serializeObject, QtProtobufPrivate::deserializeObject }); QtProtobufPrivate::registerHandler(QMetaType::fromType>>(), { QtProtobufPrivate::serializeList, diff --git a/src/protobuf/qtprotobuftypes.cpp b/src/protobuf/qtprotobuftypes.cpp index 91695486..2972dec8 100644 --- a/src/protobuf/qtprotobuftypes.cpp +++ b/src/protobuf/qtprotobuftypes.cpp @@ -4,7 +4,11 @@ #include +#include #include +#include + +#include #include "qtprotobuftypes.h" #include "qprotobufobject.h" @@ -14,7 +18,60 @@ QT_BEGIN_NAMESPACE -#define registerProtobufType(X) qRegisterMetaType(#X) +namespace { +/* + \internal + \brief The ProtobufOrderingRegistry stores the mapping between + QtProtobufPrivate::QProtobufPropertyOrdering and QMetaType. +*/ +struct ProtobufOrderingRegistry +{ + using ProtobufOrderingRegistryRecord = + QPair; + + void registerOrdering(QMetaType type, QtProtobufPrivate::QProtobufPropertyOrdering ordering) + { + QWriteLocker locker(&m_lock); + m_registry[ordering.getMessageFullName().toString()] = + ProtobufOrderingRegistryRecord(type, ordering); + } + + QtProtobufPrivate::QProtobufPropertyOrdering getOrdering(QMetaType type) const + { + QtProtobufPrivate::QProtobufPropertyOrdering ordering{ nullptr }; + + QReadLocker locker(&m_lock); + for (auto it = m_registry.constBegin(); it != m_registry.constEnd(); ++it) { + if (type == it.value().first) { + ordering = it.value().second; + break; + } + } + + return ordering; + } + + ProtobufOrderingRegistryRecord + findMessageByName(const QString &messageName) const + { + ProtobufOrderingRegistryRecord record = { + QMetaType(QMetaType::UnknownType), { nullptr } + }; + + QReadLocker locker(&m_lock); + auto it = m_registry.constFind(messageName); + if (it != m_registry.constEnd()) + record = it.value(); + return record; + } + +private: + mutable QReadWriteLock m_lock; + QHash m_registry; +}; + +Q_GLOBAL_STATIC(ProtobufOrderingRegistry, orderingRegistry) +} namespace QtProtobufPrivate { constexpr uint jsonNameOffsetsOffset = 0; @@ -109,6 +166,31 @@ namespace QtProtobufPrivate { || (offset == jsonNameOffsetsOffset && uint(index) == data->numFields)); return *(uint_data() + offset + index); } + + void registerOrdering(QMetaType type, QProtobufPropertyOrdering ordering) + { + orderingRegistry->registerOrdering(type, ordering); + } + + QProtobufPropertyOrdering getOrderingByMetaType(QMetaType type) + { + return orderingRegistry->getOrdering(type); + } + + QProtobufMessage *constructMessageByName(const QString &messageType) + { + qRegisterProtobufTypes(); + ProtobufOrderingRegistry::ProtobufOrderingRegistryRecord messageOrderingRecord = + orderingRegistry->findMessageByName(messageType); + QMetaType type = messageOrderingRecord.first; + if (type.id() != QMetaType::UnknownType) { + void *msg = type.create(); + return reinterpret_cast(msg); + } + qProtoWarning() << "Unable to find protobuf message with name" << messageType + << ". Message is not registered."; + return nullptr; + } } namespace QtProtobuf { @@ -155,32 +237,32 @@ static void qRegisterBaseTypes() { [[maybe_unused]] // definitely unused static bool registered = [] { - registerProtobufType(QtProtobuf::int32); - registerProtobufType(QtProtobuf::int64); - registerProtobufType(QtProtobuf::uint32); - registerProtobufType(QtProtobuf::uint64); - registerProtobufType(QtProtobuf::sint32); - registerProtobufType(QtProtobuf::sint64); - registerProtobufType(QtProtobuf::fixed32); - registerProtobufType(QtProtobuf::fixed64); - registerProtobufType(QtProtobuf::sfixed32); - registerProtobufType(QtProtobuf::sfixed64); - registerProtobufType(QtProtobuf::boolean); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); - registerProtobufType(QtProtobuf::int32List); - registerProtobufType(QtProtobuf::int64List); - registerProtobufType(QtProtobuf::uint32List); - registerProtobufType(QtProtobuf::uint64List); - registerProtobufType(QtProtobuf::sint32List); - registerProtobufType(QtProtobuf::sint64List); - registerProtobufType(QtProtobuf::fixed32List); - registerProtobufType(QtProtobuf::fixed64List); - registerProtobufType(QtProtobuf::sfixed32List); - registerProtobufType(QtProtobuf::sfixed64List); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); - registerProtobufType(QtProtobuf::doubleList); - registerProtobufType(QtProtobuf::floatList); - registerProtobufType(QtProtobuf::boolList); + qRegisterMetaType(); + qRegisterMetaType(); + qRegisterMetaType(); QtProtobuf::registerBasicConverters(); QtProtobuf::registerBasicConverters(); diff --git a/src/protobuf/qtprotobuftypes.h b/src/protobuf/qtprotobuftypes.h index d4a5720c..c6aaa2ae 100644 --- a/src/protobuf/qtprotobuftypes.h +++ b/src/protobuf/qtprotobuftypes.h @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -49,6 +50,8 @@ private: }; static_assert(std::is_trivially_destructible_v); +extern Q_PROTOBUF_EXPORT void registerOrdering(QMetaType type, QProtobufPropertyOrdering ordering); + // Convenience structure to hold a reference to a single entry struct QProtobufPropertyOrderingInfo { @@ -207,6 +210,7 @@ struct qMakeUnsignedImpl { }; template using qMakeUnsigned = typename qMakeUnsignedImpl::type; + } //namespace QtProtobuf Q_PROTOBUF_EXPORT void qRegisterProtobufTypes(); diff --git a/tests/auto/protobuf/basic/CMakeLists.txt b/tests/auto/protobuf/basic/CMakeLists.txt index 68d968e6..b7a172b5 100644 --- a/tests/auto/protobuf/basic/CMakeLists.txt +++ b/tests/auto/protobuf/basic/CMakeLists.txt @@ -162,6 +162,15 @@ qt6_add_protobuf(tst_protobuf_non_packed_repeatedtypes qt_autogen_tools_initial_setup(tst_protobuf_non_packed_repeatedtypes) +qt_internal_add_test(tst_protobuf_raw_serializers + SOURCES + tst_protobuf_raw_serializers.cpp + LIBRARIES + Qt::Test + Qt::ProtobufPrivate + tst_protobuf_basictypes_gen +) + if(UNIX AND NOT CMAKE_CROSSCOMPILING) qt_internal_add_test(tst_protobuf_internals SOURCES diff --git a/tests/auto/protobuf/basic/tst_protobuf_basictypes.cpp b/tests/auto/protobuf/basic/tst_protobuf_basictypes.cpp index 725fa9ea..08a651a9 100644 --- a/tests/auto/protobuf/basic/tst_protobuf_basictypes.cpp +++ b/tests/auto/protobuf/basic/tst_protobuf_basictypes.cpp @@ -8,6 +8,8 @@ #include #include +#include + #include class QtProtobufTypesGenerationTest : public QObject @@ -35,6 +37,8 @@ private slots: void AssignmentOperatorTest(); void MoveOperatorTest(); void AccessMessageFieldsFromGetter(); + + void InvalidMessageConstructorTest(); }; using namespace qtprotobufnamespace::tests; @@ -45,6 +49,10 @@ void QtProtobufTypesGenerationTest::EmptyMessageTest() QCOMPARE(qtprotobufnamespace::tests::EmptyMessage::staticMetaObject.propertyCount(), 0); QCOMPARE(qtprotobufnamespace::tests::EmptyMessage::propertyOrdering.getMessageFullName(), "qtprotobufnamespace.tests.EmptyMessage"); + + std::unique_ptr rawMessage( + QProtobufMessage::constructByName("qtprotobufnamespace.tests.EmptyMessage")); + QVERIFY(reinterpret_cast(rawMessage.get()) != nullptr); } void QtProtobufTypesGenerationTest::BoolMessageTest() @@ -284,6 +292,15 @@ void QtProtobufTypesGenerationTest::ComplexMessageTest() .value()), stringMsg); QCOMPARE(test.testComplexField(), stringMsg); + + std::unique_ptr rawObject( + QProtobufMessage::constructByName("qtprotobufnamespace.tests.ComplexMessage")); + auto *rawMessage = reinterpret_cast(rawObject.get()); + QVERIFY(rawMessage); + QCOMPARE(rawMessage->testFieldInt(), 0); + qtprotobufnamespace::tests::SimpleStringMessage embeddedStringMessage = + rawMessage->testComplexField(); + QCOMPARE(embeddedStringMessage.testFieldString(), QString()); } void QtProtobufTypesGenerationTest::BytesMessageTest() @@ -350,5 +367,13 @@ void QtProtobufTypesGenerationTest::AccessMessageFieldsFromGetter() QCOMPARE(actual, expected); } +void QtProtobufTypesGenerationTest::InvalidMessageConstructorTest() +{ + std::unique_ptr message(QProtobufMessage::constructByName( + "qtprotobufnamespace.tests.InvalidMessageConstructorTestNotExists")); + QCOMPARE(message, nullptr); +} + + QTEST_MAIN(QtProtobufTypesGenerationTest) #include "tst_protobuf_basictypes.moc" diff --git a/tests/auto/protobuf/basic/tst_protobuf_raw_serializers.cpp b/tests/auto/protobuf/basic/tst_protobuf_raw_serializers.cpp new file mode 100644 index 00000000..702911de --- /dev/null +++ b/tests/auto/protobuf/basic/tst_protobuf_raw_serializers.cpp @@ -0,0 +1,54 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 + +#include "basicmessages.qpb.h" +#include "fieldindexrange.qpb.h" + +#include +#include + +#include + +class QtProtobufRawSerializersTest : public QObject +{ + Q_OBJECT +private slots: + void init() { m_serializer.reset(new QProtobufSerializer); } + void ComplexMessageSerializeTest(); + void ComplexMessageDeserializeTest(); +private: + std::unique_ptr m_serializer; +}; + +class NonMessageQObject : public QObject +{ + Q_OBJECT +}; + +void QtProtobufRawSerializersTest::ComplexMessageSerializeTest() +{ + QProtobufMessage *rawMessage = + QProtobufMessage::constructByName("qtprotobufnamespace.tests.ComplexMessage"); + m_serializer->deserializeRawMessage( + rawMessage, QByteArray::fromHex("1208320671776572747908d3ffffffffffffffff01")); + auto *message = reinterpret_cast(rawMessage); + QCOMPARE(message->testFieldInt(), -45); + QCOMPARE(message->testComplexField().testFieldString(), QLatin1String("qwerty")); + delete rawMessage; +} + +void QtProtobufRawSerializersTest::ComplexMessageDeserializeTest() +{ + QProtobufMessage *rawMessage = + QProtobufMessage::constructByName("qtprotobufnamespace.tests.ComplexMessage"); + auto *message = reinterpret_cast(rawMessage); + message->setTestFieldInt(-45); + message->testComplexField().setTestFieldString(QLatin1String("qwerty")); + + QByteArray buffer = m_serializer->serializeRawMessage(rawMessage); + QCOMPARE(buffer.toHex(), "08d3ffffffffffffffff0112083206717765727479"); + delete rawMessage; +} + +QTEST_MAIN(QtProtobufRawSerializersTest) +#include "tst_protobuf_raw_serializers.moc"