diff --git a/src/corelib/CMakeLists.txt b/src/corelib/CMakeLists.txt index bc0870e8b61..36ffbedbbe8 100644 --- a/src/corelib/CMakeLists.txt +++ b/src/corelib/CMakeLists.txt @@ -569,6 +569,26 @@ qt_internal_extend_target(Core CONDITION QT_FEATURE_animation animation/qvariantanimation.cpp animation/qvariantanimation.h animation/qvariantanimation_p.h ) +if(QT_FEATURE_async_io) + qt_internal_extend_target(Core + SOURCES + io/qiooperation.cpp io/qiooperation_p.h io/qiooperation_p_p.h + io/qrandomaccessasyncfile.cpp io/qrandomaccessasyncfile_p.h io/qrandomaccessasyncfile_p_p.h + ) + + # TODO: This should become the last (fallback) condition later. + # We migth also want to rewrite it so that it does not depend on + # QT_FEATURE_future. + if(QT_FEATURE_thread AND QT_FEATURE_future) + qt_internal_extend_target(Core + SOURCES + io/qrandomaccessasyncfile_threadpool.cpp + DEFINES + QT_RANDOMACCESSASYNCFILE_THREAD + ) + endif() +endif() + # This needs to be done before one below adds kernel32 because the symbols we use # from synchronization also appears in kernel32 in the version of MinGW we use in CI. # However, when picking the symbols from libkernel32.a it will try to load the symbols diff --git a/src/corelib/configure.cmake b/src/corelib/configure.cmake index a589e535668..535e3742cd2 100644 --- a/src/corelib/configure.cmake +++ b/src/corelib/configure.cmake @@ -1230,6 +1230,11 @@ qt_feature("openssl-hash" PRIVATE CONDITION QT_FEATURE_openssl_linked AND QT_FEATURE_opensslv30 PURPOSE "Uses OpenSSL based implementation of cryptographic hash algorithms." ) +qt_feature("async-io" PRIVATE + LABEL "Async File I/O" + PURPOSE "Provides support for asynchronous file I/O." + CONDITION QT_FEATURE_thread AND QT_FEATURE_future +) qt_configure_add_summary_section(NAME "Qt Core") qt_configure_add_summary_entry(ARGS "backtrace") diff --git a/src/corelib/io/qiooperation.cpp b/src/corelib/io/qiooperation.cpp new file mode 100644 index 00000000000..b9cdb63e554 --- /dev/null +++ b/src/corelib/io/qiooperation.cpp @@ -0,0 +1,183 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#include "qiooperation_p.h" +#include "qiooperation_p_p.h" + +#include +#include + +QT_BEGIN_NAMESPACE + +QIOOperationPrivate::QIOOperationPrivate(QtPrivate::QIOOperationDataStorage *storage) + : dataStorage(storage) +{ + Q_ASSERT(storage); +} + +QIOOperationPrivate::~QIOOperationPrivate() +{ +} + +void QIOOperationPrivate::appendBytesProcessed(qint64 num) +{ + processed += num; +} + +void QIOOperationPrivate::operationComplete(QIOOperation::Error err) +{ + Q_Q(QIOOperation); + + error = err; + state = State::Finished; + if (err != QIOOperation::Error::None) + Q_EMIT q->errorOccurred(err); + Q_EMIT q->finished(); +} + +void QIOOperationPrivate::setError(QIOOperation::Error err) +{ + Q_Q(QIOOperation); + error = err; + if (err != QIOOperation::Error::None) { + state = State::Finished; + Q_EMIT q->errorOccurred(err); + Q_EMIT q->finished(); + } +} + +QIOOperation::~QIOOperation() +{ + ensureCompleteOrCanceled(); +} + +QIOOperation::Type QIOOperation::type() const +{ + Q_D(const QIOOperation); + return d->type; +} + +QIOOperation::Error QIOOperation::error() const +{ + Q_D(const QIOOperation); + return d->error; +} + +bool QIOOperation::isFinished() const +{ + Q_D(const QIOOperation); + return d->state == QIOOperationPrivate::State::Finished; +} + +QIOOperation::QIOOperation(QIOOperationPrivate &dd, QObject *parent) + : QObject(dd, parent) +{ + if (auto file = qobject_cast(parent)) + d_func()->file = file; +} + +void QIOOperation::ensureCompleteOrCanceled() +{ + // Block until the operation is either complete or canceled. + // Otherwise the write/read might use removed data + Q_D(QIOOperation); + if (d->state != QIOOperationPrivate::State::Finished) { + if (d->file) { + auto *filePriv = QRandomAccessAsyncFilePrivate::get(d->file); + filePriv->cancelAndWait(this); + } + } +} + +QIOReadWriteOperationBase::~QIOReadWriteOperationBase() + = default; + +qint64 QIOReadWriteOperationBase::offset() const +{ + Q_D(const QIOOperation); + return d->offset; +} + +qint64 QIOReadWriteOperationBase::numBytesProcessed() const +{ + if (!isFinished()) + return -1; + Q_D(const QIOOperation); + return d->processed; +} + +QIOReadWriteOperationBase::QIOReadWriteOperationBase(QIOOperationPrivate &dd, QObject *parent) + : QIOOperation(dd, parent) +{ +} + +QIOReadOperation::~QIOReadOperation() = default; + +QByteArray QIOReadOperation::data() const +{ + if (!isFinished()) + return {}; + Q_D(const QIOOperation); + return d->dataStorage->getValue(); +} + +QIOReadOperation::QIOReadOperation(QIOOperationPrivate &dd, QObject *parent) + : QIOReadWriteOperationBase(dd, parent) +{ + Q_ASSERT(dd.type == QIOOperation::Type::Read); + Q_ASSERT(dd.dataStorage->containsByteArray()); +} + +QIOWriteOperation::~QIOWriteOperation() = default; + +QByteArray QIOWriteOperation::data() const +{ + if (!isFinished()) + return {}; + Q_D(const QIOOperation); + return d->dataStorage->getValue(); +} + +QIOWriteOperation::QIOWriteOperation(QIOOperationPrivate &dd, QObject *parent) + : QIOReadWriteOperationBase(dd, parent) +{ + Q_ASSERT(dd.type == QIOOperation::Type::Write); + Q_ASSERT(dd.dataStorage->containsByteArray()); +} + +QIOVectoredReadOperation::~QIOVectoredReadOperation() = default; + +QSpan> QIOVectoredReadOperation::data() const +{ + if (!isFinished()) + return {}; + Q_D(const QIOOperation); + return d->dataStorage->getValue>>(); +} + +QIOVectoredReadOperation::QIOVectoredReadOperation(QIOOperationPrivate &dd, QObject *parent) + : QIOReadWriteOperationBase(dd, parent) +{ + Q_ASSERT(dd.type == QIOOperation::Type::Read); + Q_ASSERT(dd.dataStorage->containsReadSpans()); +} + +QIOVectoredWriteOperation::~QIOVectoredWriteOperation() = default; + +QSpan> QIOVectoredWriteOperation::data() const +{ + if (!isFinished()) + return {}; + Q_D(const QIOOperation); + return d->dataStorage->getValue>>(); +} + +QIOVectoredWriteOperation::QIOVectoredWriteOperation(QIOOperationPrivate &dd, QObject *parent) + : QIOReadWriteOperationBase(dd, parent) +{ + Q_ASSERT(dd.type == QIOOperation::Type::Write); + Q_ASSERT(dd.dataStorage->containsWriteSpans()); +} + +QT_END_NAMESPACE diff --git a/src/corelib/io/qiooperation_p.h b/src/corelib/io/qiooperation_p.h new file mode 100644 index 00000000000..5e745189a1b --- /dev/null +++ b/src/corelib/io/qiooperation_p.h @@ -0,0 +1,149 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#ifndef QIOOPERATION_P_H +#define QIOOPERATION_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include + +#include +#include + +QT_BEGIN_NAMESPACE + +class QIOOperationPrivate; +class Q_CORE_EXPORT QIOOperation : public QObject +{ + Q_OBJECT +public: + // TODO: more specific error codes? + enum class Error + { + None, + FileNotOpen, + IncorrectOffset, + Read, + Write, + Aborted, + }; + Q_ENUM(Error) + + enum class Type : quint8 + { + Unknown, + Read, + Write, + }; + Q_ENUM(Type) + + ~QIOOperation() override; + + Type type() const; + Error error() const; + bool isFinished() const; + +Q_SIGNALS: + void finished(); + void errorOccurred(Error err); + +protected: + QIOOperation() = delete; + Q_DISABLE_COPY_MOVE(QIOOperation) + explicit QIOOperation(QIOOperationPrivate &dd, QObject *parent = nullptr); + + void ensureCompleteOrCanceled(); + + Q_DECLARE_PRIVATE(QIOOperation) + + friend class QRandomAccessAsyncFilePrivate; +}; + +class Q_CORE_EXPORT QIOReadWriteOperationBase : public QIOOperation +{ +public: + ~QIOReadWriteOperationBase() override; + + qint64 offset() const; + qint64 numBytesProcessed() const; + +protected: + QIOReadWriteOperationBase() = delete; + Q_DISABLE_COPY_MOVE(QIOReadWriteOperationBase) + explicit QIOReadWriteOperationBase(QIOOperationPrivate &dd, QObject *parent = nullptr); +}; + +class Q_CORE_EXPORT QIOReadOperation : public QIOReadWriteOperationBase +{ +public: + ~QIOReadOperation() override; + + QByteArray data() const; + +protected: + QIOReadOperation() = delete; + Q_DISABLE_COPY_MOVE(QIOReadOperation) + explicit QIOReadOperation(QIOOperationPrivate &dd, QObject *parent = nullptr); + + friend class QRandomAccessAsyncFilePrivate; +}; + +class Q_CORE_EXPORT QIOWriteOperation : public QIOReadWriteOperationBase +{ +public: + ~QIOWriteOperation() override; + + QByteArray data() const; + +protected: + QIOWriteOperation() = delete; + Q_DISABLE_COPY_MOVE(QIOWriteOperation) + explicit QIOWriteOperation(QIOOperationPrivate &dd, QObject *parent = nullptr); + + friend class QRandomAccessAsyncFilePrivate; +}; + +class Q_CORE_EXPORT QIOVectoredReadOperation : public QIOReadWriteOperationBase +{ +public: + ~QIOVectoredReadOperation() override; + + QSpan> data() const; + +protected: + QIOVectoredReadOperation() = delete; + Q_DISABLE_COPY_MOVE(QIOVectoredReadOperation) + explicit QIOVectoredReadOperation(QIOOperationPrivate &dd, QObject *parent = nullptr); + + friend class QRandomAccessAsyncFilePrivate; +}; + +class Q_CORE_EXPORT QIOVectoredWriteOperation : public QIOReadWriteOperationBase +{ +public: + ~QIOVectoredWriteOperation() override; + + QSpan> data() const; + +protected: + QIOVectoredWriteOperation() = delete; + Q_DISABLE_COPY_MOVE(QIOVectoredWriteOperation) + explicit QIOVectoredWriteOperation(QIOOperationPrivate &dd, QObject *parent = nullptr); + + friend class QRandomAccessAsyncFilePrivate; +}; + +QT_END_NAMESPACE + +#endif // QIOOPERATION_P_H diff --git a/src/corelib/io/qiooperation_p_p.h b/src/corelib/io/qiooperation_p_p.h new file mode 100644 index 00000000000..d6fef439a85 --- /dev/null +++ b/src/corelib/io/qiooperation_p_p.h @@ -0,0 +1,183 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#ifndef QIOOPERATION_P_P_H +#define QIOOPERATION_P_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qiooperation_p.h" +#include "qrandomaccessasyncfile_p.h" + +#include + +#include +#include + +#include + +QT_BEGIN_NAMESPACE + +namespace QtPrivate { + +class QIOOperationDataStorage +{ +public: + // When passing QSpan>, we'd better have an underlying storage + // for an outer span, so that users could pass in a temporary object. + // We'd use QVarLengthArray for that. Having 256 elements (the default) + // seems to be unneeded for vectored IO. For now I picked 10 as a reasonable + // default. But maybe even less? + static constexpr qsizetype DefaultNumOfBuffers = 10; + using ReadSpans = QVarLengthArray, DefaultNumOfBuffers>; + using WriteSpans = QVarLengthArray, DefaultNumOfBuffers>; + + explicit QIOOperationDataStorage() + : data(std::monostate{}) + {} + explicit QIOOperationDataStorage(QSpan> s) + : data(ReadSpans(s.begin(), s.end())) + {} + explicit QIOOperationDataStorage(QSpan> s) + : data(WriteSpans(s.begin(), s.end())) + {} + explicit QIOOperationDataStorage(const QByteArray &a) + : data(a) + {} + explicit QIOOperationDataStorage(QByteArray &&a) + : data(std::move(a)) + {} + + bool isEmpty() const + { return std::holds_alternative(data); } + + bool containsReadSpans() const + { return std::holds_alternative(data); } + + bool containsWriteSpans() const + { return std::holds_alternative(data); } + + bool containsByteArray() const + { return std::holds_alternative(data); } + + ReadSpans &getReadSpans() + { + Q_ASSERT(containsReadSpans()); + return std::get(data); + } + const ReadSpans &getReadSpans() const + { + Q_ASSERT(containsReadSpans()); + return std::get(data); + } + + WriteSpans &getWriteSpans() + { + Q_ASSERT(containsWriteSpans()); + return std::get(data); + } + const WriteSpans &getWriteSpans() const + { + Q_ASSERT(containsWriteSpans()); + return std::get(data); + } + + QByteArray &getByteArray() + { + Q_ASSERT(containsByteArray()); + return std::get(data); + } + const QByteArray &getByteArray() const + { + Q_ASSERT(containsByteArray()); + return std::get(data); + } + + // Potentially can be extended to return a QVariant::value(). + template + T getValue() const = delete; + +private: + std::variant data; +}; + +template <> +inline QSpan> QIOOperationDataStorage::getValue() const +{ + Q_ASSERT(std::holds_alternative(data)); + const auto *val = std::get_if(&data); + if (val) + return QSpan(*val); + return {}; +} + +template <> +inline QSpan> QIOOperationDataStorage::getValue() const +{ + Q_ASSERT(std::holds_alternative(data)); + const auto *val = std::get_if(&data); + if (val) + return QSpan(*val); + return {}; +} + +template <> +inline QByteArray QIOOperationDataStorage::getValue() const +{ + Q_ASSERT(std::holds_alternative(data)); + const auto *val = std::get_if(&data); + if (val) + return *val; + return {}; +} + +} // namespace QtPrivate + +class QIOOperationPrivate : public QObjectPrivate +{ +public: + Q_DECLARE_PUBLIC(QIOOperation) + + enum class State : quint8 + { + Running, + Finished, + }; + + explicit QIOOperationPrivate(QtPrivate::QIOOperationDataStorage *storage); + ~QIOOperationPrivate(); + + static QIOOperationPrivate *get(QIOOperation *op) + { return op->d_func(); } + + void appendBytesProcessed(qint64 num); + void operationComplete(QIOOperation::Error err); + void setError(QIOOperation::Error err); + + QPointer file; + + qint64 offset = 0; + qint64 processed = 0; + + QIOOperation::Error error = QIOOperation::Error::None; + QIOOperation::Type type = QIOOperation::Type::Unknown; + + State state = State::Running; + + // takes ownership + std::unique_ptr dataStorage; +}; + +QT_END_NAMESPACE + +#endif // QIOOPERATION_P_P_H diff --git a/src/corelib/io/qrandomaccessasyncfile.cpp b/src/corelib/io/qrandomaccessasyncfile.cpp new file mode 100644 index 00000000000..92cf2278b69 --- /dev/null +++ b/src/corelib/io/qrandomaccessasyncfile.cpp @@ -0,0 +1,183 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#include "qrandomaccessasyncfile_p.h" +#include "qrandomaccessasyncfile_p_p.h" + +QT_BEGIN_NAMESPACE + +QRandomAccessAsyncFile::QRandomAccessAsyncFile(QObject *parent) + : QObject{*new QRandomAccessAsyncFilePrivate, parent} +{ + d_func()->init(); +} + +QRandomAccessAsyncFile::~QRandomAccessAsyncFile() +{ + close(); +} + +bool QRandomAccessAsyncFile::open(const QString &filePath, QIODeviceBase::OpenMode mode) +{ + Q_D(QRandomAccessAsyncFile); + return d->open(filePath, mode); +} + +void QRandomAccessAsyncFile::close() +{ + Q_D(QRandomAccessAsyncFile); + d->close(); +} + +qint64 QRandomAccessAsyncFile::size() const +{ + Q_D(const QRandomAccessAsyncFile); + return d->size(); +} + +/*! + \internal + + Reads at maximum \a maxSize bytes, starting from \a offset. + + The data is written to the internal buffer managed by the returned + QIOOperation object. + +//! [returns-qiooperation] + Returns a QIOOperation object that would emit a QIOOperation::finished() + signal once the operation is complete. +//! [returns-qiooperation] +*/ +QIOReadOperation *QRandomAccessAsyncFile::read(qint64 offset, qint64 maxSize) +{ + Q_D(QRandomAccessAsyncFile); + return d->read(offset, maxSize); +} + +/*! + \internal + \overload + + Reads the data from the file, starting from \a offset, and stores it into + \a buffer. + + The amount of bytes to be read from the file is determined by the size of + the buffer. Note that the actual amount of read bytes can be less than that. + + This operation does not take ownership of the provided buffer, so it is the + user's responsibility to make sure that the buffer is valid until the + returned QIOOperation completes. + + \note The buffer might be populated from different threads, so the user + application should not access it until the returned QIOOperation completes. + + \include qrandomaccessasyncfile.cpp returns-qiooperation +*/ +QIOVectoredReadOperation * +QRandomAccessAsyncFile::readInto(qint64 offset, QSpan buffer) +{ + Q_D(QRandomAccessAsyncFile); + return d->readInto(offset, buffer); +} + +/*! + \internal + + Reads the data from the file, starting from \a offset, and stores it into + \a buffers. + + The amount of bytes to be read from the file is determined by the sum of + sizes of all buffers. Note that the actual amount of read bytes can be less + than that. + + This operation does not take ownership of the provided buffers, so it is the + user's responsibility to make sure that the buffers are valid until the + returned QIOOperation completes. + + \note The buffers might be populated from different threads, so the user + application should not access them until the returned QIOOperation completes. + + \include qrandomaccessasyncfile.cpp returns-qiooperation +*/ +QIOVectoredReadOperation * +QRandomAccessAsyncFile::readInto(qint64 offset, QSpan> buffers) +{ + Q_D(QRandomAccessAsyncFile); + return d->readInto(offset, buffers); +} + +/*! + \internal + + Writes \a data into the file, starting from \a offset. + + The \a data array is copied into the returned QIOOperation object. + + \include qrandomaccessasyncfile.cpp returns-qiooperation +*/ +QIOWriteOperation *QRandomAccessAsyncFile::write(qint64 offset, const QByteArray &data) +{ + Q_D(QRandomAccessAsyncFile); + return d->write(offset, data); +} + +/*! + \internal + \overload + + Writes \a data into the file, starting from \a offset. + + The \a data array is moved into the returned QIOOperation object. + + \include qrandomaccessasyncfile.cpp returns-qiooperation +*/ +QIOWriteOperation *QRandomAccessAsyncFile::write(qint64 offset, QByteArray &&data) +{ + Q_D(QRandomAccessAsyncFile); + return d->write(offset, std::move(data)); +} + +/*! + \internal + \overload + + Writes the content of \a buffer into the file, starting from \a offset. + + This operation does not take ownership of the provided buffer, so it is the + user's responsibility to make sure that the buffer is valid until the + returned QIOOperation completes. + + \note The buffer might be accessed from different threads, so the user + application should not modify it until the returned QIOOperation completes. +*/ +QIOVectoredWriteOperation * +QRandomAccessAsyncFile::writeFrom(qint64 offset, QSpan buffer) +{ + Q_D(QRandomAccessAsyncFile); + return d->writeFrom(offset, buffer); +} + +/*! + \internal + + Writes the content of \a buffers into the file, starting from \a offset. + + This operation does not take ownership of the provided buffers, so it is the + user's responsibility to make sure that the buffers are valid until the + returned QIOOperation completes. + + \note The buffers might be accessed from different threads, so the user + application should not modify them until the returned QIOOperation + completes. +*/ +QIOVectoredWriteOperation * +QRandomAccessAsyncFile::writeFrom(qint64 offset, QSpan> buffers) +{ + Q_D(QRandomAccessAsyncFile); + return d->writeFrom(offset, buffers); +} + +QT_END_NAMESPACE + +#include "moc_qrandomaccessasyncfile_p.cpp" diff --git a/src/corelib/io/qrandomaccessasyncfile_p.h b/src/corelib/io/qrandomaccessasyncfile_p.h new file mode 100644 index 00000000000..afd46ca8e9a --- /dev/null +++ b/src/corelib/io/qrandomaccessasyncfile_p.h @@ -0,0 +1,65 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#ifndef QRANDOMACCESSASYNCFILE_P_H +#define QRANDOMACCESSASYNCFILE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qiooperation_p.h" + +#include +#include + +QT_BEGIN_NAMESPACE + +class QRandomAccessAsyncFilePrivate; +class Q_CORE_EXPORT QRandomAccessAsyncFile : public QObject +{ + Q_OBJECT +public: + explicit QRandomAccessAsyncFile(QObject *parent = nullptr); + ~QRandomAccessAsyncFile() override; + + // sync APIs + bool open(const QString &filePath, QIODeviceBase::OpenMode mode); + void close(); + qint64 size() const; + + // owning APIs: we are responsible for storing the data + [[nodiscard]] QIOReadOperation *read(qint64 offset, qint64 maxSize); + [[nodiscard]] QIOWriteOperation *write(qint64 offset, const QByteArray &data); + [[nodiscard]] QIOWriteOperation *write(qint64 offset, QByteArray &&data); + + // non-owning APIs: the user has to control the lifetime of buffers + [[nodiscard]] QIOVectoredReadOperation * + readInto(qint64 offset, QSpan buffer); + [[nodiscard]] QIOVectoredWriteOperation * + writeFrom(qint64 offset, QSpan buffer); + + // vectored IO APIs, also non-owning + [[nodiscard]] QIOVectoredReadOperation * + readInto(qint64 offset, QSpan> buffers); + [[nodiscard]] QIOVectoredWriteOperation * + writeFrom(qint64 offset, QSpan> buffers); + +Q_SIGNALS: + +private: + Q_DECLARE_PRIVATE(QRandomAccessAsyncFile) + Q_DISABLE_COPY_MOVE(QRandomAccessAsyncFile) +}; + +QT_END_NAMESPACE + +#endif // QRANDOMACCESSASYNCFILE_P_H diff --git a/src/corelib/io/qrandomaccessasyncfile_p_p.h b/src/corelib/io/qrandomaccessasyncfile_p_p.h new file mode 100644 index 00000000000..129b6a8e3ea --- /dev/null +++ b/src/corelib/io/qrandomaccessasyncfile_p_p.h @@ -0,0 +1,95 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#ifndef QRANDOMACCESSASYNCFILE_P_P_H +#define QRANDOMACCESSASYNCFILE_P_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qrandomaccessasyncfile_p.h" + +#include + +#include + +#ifdef QT_RANDOMACCESSASYNCFILE_THREAD + +#include + +#include +#include +#include + +#endif // QT_RANDOMACCESSASYNCFILE_THREAD + +QT_BEGIN_NAMESPACE + +class QRandomAccessAsyncFilePrivate : public QObjectPrivate +{ + Q_DECLARE_PUBLIC(QRandomAccessAsyncFile) + Q_DISABLE_COPY_MOVE(QRandomAccessAsyncFilePrivate) +public: + QRandomAccessAsyncFilePrivate(decltype(QObjectPrivateVersion) version = QObjectPrivateVersion); + ~QRandomAccessAsyncFilePrivate() override; + + static QRandomAccessAsyncFilePrivate *get(QRandomAccessAsyncFile *file) + { return file->d_func(); } + + void init(); + void cancelAndWait(QIOOperation *op); + + bool open(const QString &path, QIODeviceBase::OpenMode mode); + void close(); + qint64 size() const; + + [[nodiscard]] QIOReadOperation *read(qint64 offset, qint64 maxSize); + [[nodiscard]] QIOWriteOperation *write(qint64 offset, const QByteArray &data); + [[nodiscard]] QIOWriteOperation *write(qint64 offset, QByteArray &&data); + + [[nodiscard]] QIOVectoredReadOperation * + readInto(qint64 offset, QSpan buffer); + [[nodiscard]] QIOVectoredWriteOperation * + writeFrom(qint64 offset, QSpan buffer); + + [[nodiscard]] QIOVectoredReadOperation * + readInto(qint64 offset, QSpan> buffers); + [[nodiscard]] QIOVectoredWriteOperation * + writeFrom(qint64 offset, QSpan> buffers); + +private: +#ifdef QT_RANDOMACCESSASYNCFILE_THREAD +public: + struct OperationResult + { + qint64 bytesProcessed; // either read or written + QIOOperation::Error error; + }; + +private: + mutable QBasicMutex m_engineMutex; + std::unique_ptr m_engine; + QFutureWatcher m_watcher; + + QQueue> m_operations; + QPointer m_currentOperation; + qsizetype numProcessedBuffers = 0; + + void executeNextOperation(); + void processBufferAt(qsizetype idx); + void operationComplete(); +#endif +}; + +QT_END_NAMESPACE + +#endif // QRANDOMACCESSASYNCFILE_P_P_H diff --git a/src/corelib/io/qrandomaccessasyncfile_threadpool.cpp b/src/corelib/io/qrandomaccessasyncfile_threadpool.cpp new file mode 100644 index 00000000000..e0027fdc37f --- /dev/null +++ b/src/corelib/io/qrandomaccessasyncfile_threadpool.cpp @@ -0,0 +1,424 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +// Qt-Security score:significant reason:default + +#include "qrandomaccessasyncfile_p_p.h" + +#include "qiooperation_p.h" +#include "qiooperation_p_p.h" + +#include +#include +#include + +QT_REQUIRE_CONFIG(thread); +QT_REQUIRE_CONFIG(future); + +QT_BEGIN_NAMESPACE + +namespace { + +// We cannot use Q_GLOBAL_STATIC(QThreadPool, foo) because the Windows +// implementation raises a qWarning in its destructor when used as a global +// static, and this warning leads to a crash on Windows CI. Cannot reproduce +// the crash locally, so cannot really fix the issue :( +// This class should act like a global thread pool, but it'll have a sort of +// ref counting, and will be created/destroyed by QRAAFP instances. +class SharedThreadPool +{ +public: + void ref() + { + QMutexLocker locker(&m_mutex); + if (m_refCount == 0) { + Q_ASSERT(!m_pool); + m_pool = new QThreadPool; + } + ++m_refCount; + } + + void deref() + { + QMutexLocker locker(&m_mutex); + Q_ASSERT(m_refCount); + if (--m_refCount == 0) { + delete m_pool; + m_pool = nullptr; + } + } + + QThreadPool *operator()() + { + QMutexLocker locker(&m_mutex); + Q_ASSERT(m_refCount > 0); + return m_pool; + } + +private: + QBasicMutex m_mutex; + QThreadPool *m_pool = nullptr; + quint64 m_refCount = 0; +}; + +static SharedThreadPool asyncFileThreadPool; + +} // anonymous namespace + +QRandomAccessAsyncFilePrivate::QRandomAccessAsyncFilePrivate(decltype(QObjectPrivateVersion) version) : + QObjectPrivate(version) +{ + asyncFileThreadPool.ref(); +} + +QRandomAccessAsyncFilePrivate::~QRandomAccessAsyncFilePrivate() +{ + asyncFileThreadPool.deref(); +} + +void QRandomAccessAsyncFilePrivate::init() +{ + QObject::connect(&m_watcher, &QFutureWatcherBase::finished, q_ptr, [this]{ + operationComplete(); + }); + QObject::connect(&m_watcher, &QFutureWatcherBase::canceled, q_ptr, [this]{ + operationComplete(); + }); +} + +void QRandomAccessAsyncFilePrivate::cancelAndWait(QIOOperation *op) +{ + if (op == m_currentOperation) { + m_currentOperation = nullptr; // to discard the result + m_watcher.cancel(); // might have no effect + m_watcher.waitForFinished(); + } else { + m_operations.removeAll(op); + } +} + +bool QRandomAccessAsyncFilePrivate::open(const QString &path, QIODeviceBase::OpenMode mode) +{ + QMutexLocker locker(&m_engineMutex); + if (m_engine) { + // already opened! + return false; + } + + m_engine = std::make_unique(path); + mode |= QIODeviceBase::Unbuffered; + return m_engine->open(mode, std::nullopt); +} + +void QRandomAccessAsyncFilePrivate::close() +{ + // all the operations should be aborted + for (const auto &op : std::as_const(m_operations)) { + if (op) { + auto *priv = QIOOperationPrivate::get(op.get()); + priv->setError(QIOOperation::Error::Aborted); + } + } + m_operations.clear(); + + // Wait until the current operation is complete + if (m_currentOperation) { + auto *priv = QIOOperationPrivate::get(m_currentOperation.get()); + priv->setError(QIOOperation::Error::Aborted); + cancelAndWait(m_currentOperation.get()); + } + + QMutexLocker locker(&m_engineMutex); + if (m_engine) { + m_engine->close(); + m_engine.reset(); + } +} + +qint64 QRandomAccessAsyncFilePrivate::size() const +{ + QMutexLocker locker(&m_engineMutex); + if (m_engine) + return m_engine->size(); + return -1; +} + +QIOReadOperation *QRandomAccessAsyncFilePrivate::read(qint64 offset, qint64 maxSize) +{ + QByteArray array; + array.resizeForOverwrite(maxSize); + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(std::move(array)); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Read; + + auto *op = new QIOReadOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +QIOWriteOperation * +QRandomAccessAsyncFilePrivate::write(qint64 offset, const QByteArray &data) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(data); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOWriteOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +QIOWriteOperation * +QRandomAccessAsyncFilePrivate::write(qint64 offset, QByteArray &&data) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(std::move(data)); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOWriteOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +QIOVectoredReadOperation * +QRandomAccessAsyncFilePrivate::readInto(qint64 offset, QSpan buffer) +{ + auto *dataStorage = + new QtPrivate::QIOOperationDataStorage(QSpan>{buffer}); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Read; + + auto *op = new QIOVectoredReadOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +QIOVectoredWriteOperation * +QRandomAccessAsyncFilePrivate::writeFrom(qint64 offset, QSpan buffer) +{ + auto *dataStorage = + new QtPrivate::QIOOperationDataStorage(QSpan>{buffer}); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOVectoredWriteOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +QIOVectoredReadOperation * +QRandomAccessAsyncFilePrivate::readInto(qint64 offset, QSpan> buffers) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(buffers); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Read; + + auto *op = new QIOVectoredReadOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +QIOVectoredWriteOperation * +QRandomAccessAsyncFilePrivate::writeFrom(qint64 offset, QSpan> buffers) +{ + auto *dataStorage = new QtPrivate::QIOOperationDataStorage(buffers); + + auto *priv = new QIOOperationPrivate(dataStorage); + priv->offset = offset; + priv->type = QIOOperation::Type::Write; + + auto *op = new QIOVectoredWriteOperation(*priv, q_ptr); + m_operations.append(op); + executeNextOperation(); + return op; +} + +static QRandomAccessAsyncFilePrivate::OperationResult +executeRead(QFSFileEngine *engine, QBasicMutex *mutex, qint64 offset, char *buffer, qint64 maxSize) +{ + QRandomAccessAsyncFilePrivate::OperationResult result{0, QIOOperation::Error::None}; + + QMutexLocker locker(mutex); + if (engine) { + if (engine->seek(offset)) { + qint64 bytesRead = engine->read(buffer, maxSize); + if (bytesRead >= 0) + result.bytesProcessed = bytesRead; + else + result.error = QIOOperation::Error::Read; + } else { + result.error = QIOOperation::Error::IncorrectOffset; + } + } else { + result.error = QIOOperation::Error::FileNotOpen; + } + return result; +} + +static QRandomAccessAsyncFilePrivate::OperationResult +executeWrite(QFSFileEngine *engine, QBasicMutex *mutex, qint64 offset, + const char *buffer, qint64 size) +{ + QRandomAccessAsyncFilePrivate::OperationResult result{0, QIOOperation::Error::None}; + + QMutexLocker locker(mutex); + if (engine) { + if (engine->seek(offset)) { + qint64 written = engine->write(buffer, size); + if (written >= 0) + result.bytesProcessed = written; + else + result.error = QIOOperation::Error::Write; + } else { + result.error = QIOOperation::Error::IncorrectOffset; + } + } else { + result.error = QIOOperation::Error::FileNotOpen; + } + return result; +} + +void QRandomAccessAsyncFilePrivate::executeNextOperation() +{ + if (m_currentOperation.isNull()) { + // start next + if (!m_operations.isEmpty()) { + m_currentOperation = m_operations.takeFirst(); + numProcessedBuffers = 0; + processBufferAt(numProcessedBuffers); + } + } +} + +void QRandomAccessAsyncFilePrivate::processBufferAt(qsizetype idx) +{ + Q_ASSERT(!m_currentOperation.isNull()); + auto *priv = QIOOperationPrivate::get(m_currentOperation.get()); + auto &dataStorage = priv->dataStorage; + // if we do not use span buffers, we have only one buffer + Q_ASSERT(dataStorage->containsReadSpans() + || dataStorage->containsWriteSpans() + || idx == 0); + if (priv->type == QIOOperation::Type::Read) { + qint64 maxSize = -1; + char *buf = nullptr; + if (dataStorage->containsReadSpans()) { + auto &readBuffers = dataStorage->getReadSpans(); + Q_ASSERT(readBuffers.size() > idx); + maxSize = readBuffers[idx].size_bytes(); + buf = reinterpret_cast(readBuffers[idx].data()); + } else { + Q_ASSERT(dataStorage->containsByteArray()); + auto &array = dataStorage->getByteArray(); + maxSize = array.size(); + buf = array.data(); + } + Q_ASSERT(maxSize >= 0); + + qint64 offset = priv->offset; + if (idx != 0) + offset += priv->processed; + QBasicMutex *mutexPtr = &m_engineMutex; + auto op = [engine = m_engine.get(), buf, maxSize, offset, mutexPtr] { + return executeRead(engine, mutexPtr, offset, buf, maxSize); + }; + + QFuture f = + QtFuture::makeReadyVoidFuture().then(asyncFileThreadPool(), op); + m_watcher.setFuture(f); + } else if (priv->type == QIOOperation::Type::Write) { + qint64 size = -1; + const char *buf = nullptr; + if (dataStorage->containsWriteSpans()) { + const auto &writeBuffers = dataStorage->getWriteSpans(); + Q_ASSERT(writeBuffers.size() > idx); + size = writeBuffers[idx].size_bytes(); + buf = reinterpret_cast(writeBuffers[idx].data()); + } else { + Q_ASSERT(dataStorage->containsByteArray()); + const auto &array = dataStorage->getByteArray(); + size = array.size(); + buf = array.constData(); + } + Q_ASSERT(size >= 0); + + qint64 offset = priv->offset; + if (idx != 0) + offset += priv->processed; + QBasicMutex *mutexPtr = &m_engineMutex; + auto op = [engine = m_engine.get(), buf, size, offset, mutexPtr] { + return executeWrite(engine, mutexPtr, offset, buf, size); + }; + + QFuture f = + QtFuture::makeReadyVoidFuture().then(asyncFileThreadPool(), op); + m_watcher.setFuture(f); + } +} + +void QRandomAccessAsyncFilePrivate::operationComplete() +{ + // TODO: if one of the buffers was read/written with an error, + // stop processing immediately + if (m_currentOperation && !m_watcher.isCanceled()) { + OperationResult res = m_watcher.future().result(); + auto *priv = QIOOperationPrivate::get(m_currentOperation.get()); + auto &dataStorage = priv->dataStorage; + qsizetype expectedBuffersCount = 1; + bool needProcessNext = false; + if (priv->type == QIOOperation::Type::Read) { + if (dataStorage->containsReadSpans()) { + auto &readBuffers = dataStorage->getReadSpans(); + expectedBuffersCount = readBuffers.size(); + Q_ASSERT(numProcessedBuffers < expectedBuffersCount); + const qsizetype unusedBytes = + readBuffers[numProcessedBuffers].size_bytes() - res.bytesProcessed; + readBuffers[numProcessedBuffers].chop(unusedBytes); + } else { + Q_ASSERT(dataStorage->containsByteArray()); + Q_ASSERT(numProcessedBuffers == 0); + auto &array = dataStorage->getByteArray(); + array.resize(res.bytesProcessed); + } + priv->appendBytesProcessed(res.bytesProcessed); + needProcessNext = (++numProcessedBuffers < expectedBuffersCount); + if (!needProcessNext) + priv->operationComplete(res.error); + } else if (priv->type == QIOOperation::Type::Write) { + if (dataStorage->containsWriteSpans()) + expectedBuffersCount = dataStorage->getWriteSpans().size(); + Q_ASSERT(numProcessedBuffers < expectedBuffersCount); + needProcessNext = (++numProcessedBuffers < expectedBuffersCount); + priv->appendBytesProcessed(res.bytesProcessed); + if (!needProcessNext) + priv->operationComplete(res.error); + } + if (needProcessNext) { + // keep executing this command + processBufferAt(numProcessedBuffers); + return; + } else { + m_currentOperation = nullptr; + } + } + executeNextOperation(); +} + +QT_END_NAMESPACE diff --git a/tests/auto/corelib/io/CMakeLists.txt b/tests/auto/corelib/io/CMakeLists.txt index 291dbfb413a..f76900162ca 100644 --- a/tests/auto/corelib/io/CMakeLists.txt +++ b/tests/auto/corelib/io/CMakeLists.txt @@ -11,6 +11,9 @@ if(QT_FEATURE_private_tests) add_subdirectory(qfileinfo) add_subdirectory(qipaddress) add_subdirectory(qloggingregistry) + if(QT_FEATURE_async_io) + add_subdirectory(qrandomaccessasyncfile) + endif() add_subdirectory(qurlinternal) endif() add_subdirectory(qbuffer) diff --git a/tests/auto/corelib/io/qrandomaccessasyncfile/CMakeLists.txt b/tests/auto/corelib/io/qrandomaccessasyncfile/CMakeLists.txt new file mode 100644 index 00000000000..5dddb513af6 --- /dev/null +++ b/tests/auto/corelib/io/qrandomaccessasyncfile/CMakeLists.txt @@ -0,0 +1,15 @@ +# Copyright (C) 2025 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +if(NOT QT_BUILD_STANDALONE_TESTS AND NOT QT_BUILDING_QT) + cmake_minimum_required(VERSION 3.16) + project(tst_qrandomaccessasyncfile LANGUAGES CXX) + find_package(Qt6BuildInternals REQUIRED COMPONENTS STANDALONE_TEST) +endif() + +qt_internal_add_test(tst_qrandomaccessasyncfile + SOURCES + tst_qrandomaccessasyncfile.cpp + LIBRARIES + Qt::CorePrivate +) diff --git a/tests/auto/corelib/io/qrandomaccessasyncfile/tst_qrandomaccessasyncfile.cpp b/tests/auto/corelib/io/qrandomaccessasyncfile/tst_qrandomaccessasyncfile.cpp new file mode 100644 index 00000000000..85c78b42995 --- /dev/null +++ b/tests/auto/corelib/io/qrandomaccessasyncfile/tst_qrandomaccessasyncfile.cpp @@ -0,0 +1,578 @@ +// Copyright (C) 2025 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only + +#include + +#include +#include + +#include +#include + +template +static bool spanIsEqualToByteArray(QSpan lhs, QByteArrayView rhs) noexcept +{ + const auto leftBytes = as_bytes(lhs); + const auto rightBytes = as_bytes(QSpan{rhs}); + return std::equal(leftBytes.begin(), leftBytes.end(), + rightBytes.begin(), rightBytes.end()); +} + +class tst_QRandomAccessAsyncFile : public QObject +{ + Q_OBJECT + +private Q_SLOTS: + void initTestCase(); + void cleanupTestCase(); + void size(); + void roundtripOwning(); + void roundtripNonOwning(); + void roundtripVectored(); + void readLessThanMax(); + void errorHandling_data(); + void errorHandling(); + void fileClosedInProgress_data(); + void fileClosedInProgress(); + void fileRemovedInProgress_data(); + void fileRemovedInProgress(); + void operationsDeletedInProgress_data(); + void operationsDeletedInProgress(); + +private: + enum class Ownership : quint8 + { + Owning, + NonOwning, + }; + void generateReadWriteOperationColumns(); + + // Write 100 Mb of random data to the file. + // We use such a large amount, because some of the backends will report + // the progress of async operations in chunks, and we want to test it. + static constexpr qint64 FileSize = 100 * 1024 * 1024; + QTemporaryFile m_file; +}; + +void tst_QRandomAccessAsyncFile::initTestCase() +{ + QVERIFY(m_file.open()); + + QByteArray data(FileSize, Qt::Uninitialized); + for (qsizetype i = 0; i < FileSize; ++i) + data[i] = char(i % 256); + + qint64 written = m_file.write(data); + QCOMPARE_EQ(written, FileSize); +} + +void tst_QRandomAccessAsyncFile::cleanupTestCase() +{ + m_file.close(); + QVERIFY(m_file.remove()); +} + +void tst_QRandomAccessAsyncFile::size() +{ + QRandomAccessAsyncFile file; + + // File not opened -> size unknown + QCOMPARE_EQ(file.size(), -1); + + QVERIFY(file.open(m_file.fileName(), QIODeviceBase::ReadOnly)); + + QCOMPARE(file.size(), FileSize); +} + +void tst_QRandomAccessAsyncFile::roundtripOwning() +{ + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODevice::ReadWrite)); + + // All operations will be deleted together with the file + + // Write some data into the file + + const qsizetype offset1 = 1024 * 1024; + const qsizetype size1 = 10 * 1024 * 1024; + + // Testing const ref overload + const QByteArray dataToWrite(size1, 'a'); + QIOWriteOperation *write1 = file.write(offset1, dataToWrite); + QSignalSpy write1Spy(write1, &QIOOperation::finished); + QSignalSpy write1ErrorSpy(write1, &QIOOperation::errorOccurred); + + const qsizetype offset2 = 20 * 1024 * 1024; + const qsizetype size2 = 5 * 1024 * 1024; + + // Testing rvalue overload + QIOWriteOperation *write2 = file.write(offset2, QByteArray(size2, 'b')); + QSignalSpy write2Spy(write2, &QIOOperation::finished); + QSignalSpy write2ErrorSpy(write2, &QIOOperation::errorOccurred); + + QTRY_COMPARE_EQ(write1Spy.size(), 1); + QCOMPARE_EQ(write1ErrorSpy.size(), 0); + QCOMPARE_EQ(write1->error(), QIOOperation::Error::None); + QCOMPARE_EQ(write1->isFinished(), true); + QCOMPARE_EQ(write1->offset(), offset1); + QCOMPARE_EQ(write1->numBytesProcessed(), size1); + + QTRY_COMPARE_EQ(write2Spy.size(), 1); + QCOMPARE_EQ(write2ErrorSpy.size(), 0); + QCOMPARE_EQ(write2->error(), QIOOperation::Error::None); + QCOMPARE_EQ(write2->isFinished(), true); + QCOMPARE_EQ(write2->offset(), offset2); + QCOMPARE_EQ(write2->numBytesProcessed(), size2); + + // Now read what we have written + + QIOReadOperation *read1 = file.read(offset1, size1); + QSignalSpy read1Spy(read1, &QIOOperation::finished); + QSignalSpy read1ErrorSpy(read1, &QIOOperation::errorOccurred); + + QIOReadOperation *read2 = file.read(offset2, size2); + QSignalSpy read2Spy(read2, &QIOOperation::finished); + QSignalSpy read2ErrorSpy(read2, &QIOOperation::errorOccurred); + + QTRY_COMPARE_EQ(read1Spy.size(), 1); + QCOMPARE_EQ(read1ErrorSpy.size(), 0); + QCOMPARE_EQ(read1->error(), QIOOperation::Error::None); + QCOMPARE_EQ(read1->isFinished(), true); + QCOMPARE_EQ(read1->offset(), offset1); + QCOMPARE_EQ(read1->data(), dataToWrite); + + QTRY_COMPARE_EQ(read2Spy.size(), 1); + QCOMPARE_EQ(read2ErrorSpy.size(), 0); + QCOMPARE_EQ(read2->error(), QIOOperation::Error::None); + QCOMPARE_EQ(read2->isFinished(), true); + QCOMPARE_EQ(read2->offset(), offset2); + QCOMPARE_EQ(read2->data(), QByteArray(size2, 'b')); +} + +void tst_QRandomAccessAsyncFile::roundtripNonOwning() +{ + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODevice::ReadWrite)); + + // All operations will be deleted together with the file + + // Write some data into the file + + const qsizetype offset1 = 1024 * 1024; + const qsizetype size1 = 10 * 1024 * 1024; + + // QSpan is an lvalue + const QByteArray dataToWrite(size1, 'a'); + const QSpan spanToWrite(as_bytes(QSpan{dataToWrite})); + QIOVectoredWriteOperation *write1 = file.writeFrom(offset1, spanToWrite); + QSignalSpy write1Spy(write1, &QIOOperation::finished); + QSignalSpy write1ErrorSpy(write1, &QIOOperation::errorOccurred); + + const qsizetype offset2 = 20 * 1024 * 1024; + const qsizetype size2 = 5 * 1024 * 1024; + + // QSpan is an rvalue + const QByteArray otherDataToWrite(size2, 'b'); + QIOVectoredWriteOperation *write2 = + file.writeFrom(offset2, as_bytes(QSpan{otherDataToWrite})); + QSignalSpy write2Spy(write2, &QIOOperation::finished); + QSignalSpy write2ErrorSpy(write2, &QIOOperation::errorOccurred); + + QTRY_COMPARE_EQ(write1Spy.size(), 1); + QCOMPARE_EQ(write1ErrorSpy.size(), 0); + QCOMPARE_EQ(write1->error(), QIOOperation::Error::None); + QCOMPARE_EQ(write1->isFinished(), true); + QCOMPARE_EQ(write1->offset(), offset1); + QCOMPARE_EQ(write1->numBytesProcessed(), size1); + + QTRY_COMPARE_EQ(write2Spy.size(), 1); + QCOMPARE_EQ(write2ErrorSpy.size(), 0); + QCOMPARE_EQ(write2->error(), QIOOperation::Error::None); + QCOMPARE_EQ(write2->isFinished(), true); + QCOMPARE_EQ(write2->offset(), offset2); + QCOMPARE_EQ(write2->numBytesProcessed(), size2); + + // Now read what we have written + + // QSpan is an lvalue + QByteArray buffer1(size1, Qt::Uninitialized); + QSpan spanToRead = as_writable_bytes(QSpan{buffer1}); + QIOVectoredReadOperation *read1 = file.readInto(offset1, spanToRead); + QSignalSpy read1Spy(read1, &QIOOperation::finished); + QSignalSpy read1ErrorSpy(read1, &QIOOperation::errorOccurred); + + // QSpan is an rvalue + QByteArray buffer2(size2, Qt::Uninitialized); + QIOVectoredReadOperation *read2 = + file.readInto(offset2, as_writable_bytes(QSpan{buffer2})); + QSignalSpy read2Spy(read2, &QIOOperation::finished); + QSignalSpy read2ErrorSpy(read2, &QIOOperation::errorOccurred); + + QTRY_COMPARE_EQ(read1Spy.size(), 1); + QCOMPARE_EQ(read1ErrorSpy.size(), 0); + QCOMPARE_EQ(read1->error(), QIOOperation::Error::None); + QCOMPARE_EQ(read1->isFinished(), true); + QCOMPARE_EQ(read1->offset(), offset1); + QVERIFY(spanIsEqualToByteArray(read1->data().front(), dataToWrite)); + + QTRY_COMPARE_EQ(read2Spy.size(), 1); + QCOMPARE_EQ(read2ErrorSpy.size(), 0); + QCOMPARE_EQ(read2->error(), QIOOperation::Error::None); + QCOMPARE_EQ(read2->isFinished(), true); + QCOMPARE_EQ(read2->offset(), offset2); + QVERIFY(spanIsEqualToByteArray(read2->data().front(), otherDataToWrite)); +} + +void tst_QRandomAccessAsyncFile::roundtripVectored() +{ + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODevice::ReadWrite)); + + // All operations will be deleted together with the file + + // Write some data into the file + + const qsizetype offset = 1024 * 1024; + const qsizetype size1 = 10 * 1024 * 1024; + const QByteArray dataToWrite(size1, 'a'); + + const qsizetype size2 = 5 * 1024 * 1024; + const QByteArray otherDataToWrite(size2, 'b'); + + // vectored write + QIOVectoredWriteOperation *write = + file.writeFrom(offset, { as_bytes(QSpan{dataToWrite}), + as_bytes(QSpan{otherDataToWrite}) }); + QSignalSpy writeSpy(write, &QIOOperation::finished); + QSignalSpy writeErrorSpy(write, &QIOOperation::errorOccurred); + + QTRY_COMPARE_EQ(writeSpy.size(), 1); + QCOMPARE_EQ(writeErrorSpy.size(), 0); + QCOMPARE_EQ(write->error(), QIOOperation::Error::None); + QCOMPARE_EQ(write->isFinished(), true); + QCOMPARE_EQ(write->offset(), offset); + QCOMPARE_EQ(write->numBytesProcessed(), size1 + size2); + + // Now read what we have written + + QByteArray buffer1(size1, Qt::Uninitialized); + QByteArray buffer2(size2, Qt::Uninitialized); + + QIOVectoredReadOperation *read = + file.readInto(offset, { as_writable_bytes(QSpan{buffer1}), + as_writable_bytes(QSpan{buffer2}) }); + QSignalSpy readSpy(read, &QIOOperation::finished); + QSignalSpy readErrorSpy(read, &QIOOperation::errorOccurred); + + QTRY_COMPARE_EQ(readSpy.size(), 1); + QCOMPARE_EQ(readErrorSpy.size(), 0); + QCOMPARE_EQ(read->error(), QIOOperation::Error::None); + QCOMPARE_EQ(read->isFinished(), true); + QCOMPARE_EQ(read->offset(), offset); + + std::array expectedResults = {&dataToWrite, &otherDataToWrite}; + + const auto buffers = read->data(); + QCOMPARE_EQ(size_t(buffers.size()), expectedResults.size()); + for (size_t i = 0; i < expectedResults.size(); ++i) + QVERIFY(spanIsEqualToByteArray(buffers[i], *expectedResults[i])); +} + +void tst_QRandomAccessAsyncFile::readLessThanMax() +{ + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODeviceBase::ReadOnly)); + + constexpr qint64 offsetFromEnd = 100; + + // owning + { + QIOReadOperation *op = file.read(FileSize - offsetFromEnd, 1024); + QSignalSpy spy(op, &QIOOperation::finished); + + QTRY_COMPARE_EQ(spy.size(), 1); + QCOMPARE_EQ(op->error(), QIOOperation::Error::None); + QCOMPARE_EQ(op->numBytesProcessed(), offsetFromEnd); + // we only read what we could + QCOMPARE_EQ(op->data().size(), offsetFromEnd); + } + + // non-owning single buffer + { + QByteArray buffer(1024, Qt::Uninitialized); + QIOVectoredReadOperation *op = + file.readInto(FileSize - offsetFromEnd, as_writable_bytes(QSpan{buffer})); + QSignalSpy spy(op, &QIOOperation::finished); + + QTRY_COMPARE_EQ(spy.size(), 1); + QCOMPARE_EQ(op->error(), QIOOperation::Error::None); + QCOMPARE_EQ(op->numBytesProcessed(), offsetFromEnd); + // we only read what we could + QCOMPARE_EQ(op->data().front().size(), offsetFromEnd); + } + + // non-owning vectored read + { + constexpr qsizetype size1 = 50; + constexpr qsizetype size2 = 150; + constexpr qsizetype size3 = size2; + + QByteArray buffer1(50, Qt::Uninitialized); + QByteArray buffer2(size2, Qt::Uninitialized); + QByteArray buffer3(size3, Qt::Uninitialized); + + std::array, 3> buffers{ as_writable_bytes(QSpan{buffer1}), + as_writable_bytes(QSpan{buffer2}), + as_writable_bytes(QSpan{buffer3}) }; + + QIOVectoredReadOperation *op = + file.readInto(FileSize - offsetFromEnd, buffers); + QSignalSpy spy(op, &QIOOperation::finished); + QTRY_COMPARE_EQ(spy.size(), 1); + QCOMPARE_EQ(op->error(), QIOOperation::Error::None); + QCOMPARE_EQ(op->numBytesProcessed(), offsetFromEnd); + + const auto results = op->data(); + QCOMPARE_EQ(size_t(results.size()), buffers.size()); + + // first buffer should be fully populated + QCOMPARE_EQ(results[0].size(), size1); + + // second buffer should only be partially populated + constexpr qsizetype expectedSize2 = offsetFromEnd - size1; + QCOMPARE_EQ(results[1].size(), expectedSize2); + + // third buffer should be empty + QCOMPARE_EQ(results[2].size(), 0); + } +} + +void tst_QRandomAccessAsyncFile::errorHandling_data() +{ + QTest::addColumn("operation"); + QTest::addColumn("openMode"); + QTest::addColumn("offset"); + QTest::addColumn("expectedError"); + + QTest::newRow("read_not_open") + << QIOOperation::Type::Read << QIODeviceBase::ReadWrite + << qint64(0) << QIOOperation::Error::FileNotOpen; + QTest::newRow("read_writeonly") + << QIOOperation::Type::Read << QIODeviceBase::WriteOnly + << qint64(0) << QIOOperation::Error::Read; + QTest::newRow("read_negative_offset") + << QIOOperation::Type::Read << QIODeviceBase::ReadOnly + << qint64(-1) << QIOOperation::Error::IncorrectOffset; + // lseek() allows it. Other backends might behave differently + // QTest::newRow("read_past_the_end") + // << QIOOperationBase::Type::Read << QIODeviceBase::ReadOnly + // << qint64(FileSize + 1) << QIOOperationBase::Error::IncorrectOffset; + + QTest::newRow("write_not_open") + << QIOOperation::Type::Write << QIODeviceBase::ReadWrite + << qint64(0) << QIOOperation::Error::FileNotOpen; + QTest::newRow("write_readonly") + << QIOOperation::Type::Write << QIODeviceBase::ReadOnly + << qint64(0) << QIOOperation::Error::Write; + QTest::newRow("write_negative_offset") + << QIOOperation::Type::Write << QIODeviceBase::WriteOnly + << qint64(-1) << QIOOperation::Error::IncorrectOffset; + // lseek() allows it. Other backends might behave differently + // QTest::newRow("write_past_the_end") + // << QIOOperationBase::Type::Write << QIODeviceBase::ReadWrite + // << qint64(FileSize + 1) << QIOOperationBase::Error::IncorrectOffset; +} + +void tst_QRandomAccessAsyncFile::errorHandling() +{ + QFETCH(const QIOOperation::Type, operation); + QFETCH(const QIODeviceBase::OpenModeFlag, openMode); + QFETCH(const qint64, offset); + QFETCH(const QIOOperation::Error, expectedError); + + QRandomAccessAsyncFile file; + if (expectedError != QIOOperation::Error::FileNotOpen) + QVERIFY(file.open(m_file.fileName(), openMode)); + + QIOOperation *op = nullptr; + if (operation == QIOOperation::Type::Read) + op = file.read(offset, 100); + else if (operation == QIOOperation::Type::Write) + op = file.write(offset, QByteArray(100, 'c')); + + QVERIFY(op); + + QSignalSpy finishedSpy(op, &QIOOperation::finished); + QSignalSpy errorSpy(op, &QIOOperation::errorOccurred); + + // error should always come before finished + QTRY_COMPARE_EQ(finishedSpy.size(), 1); + QCOMPARE_EQ(errorSpy.size(), 1); + + QCOMPARE_EQ(errorSpy.at(0).at(0).value(), expectedError); + QCOMPARE_EQ(op->error(), expectedError); +} + +void tst_QRandomAccessAsyncFile::fileClosedInProgress_data() +{ + generateReadWriteOperationColumns(); +} + +void tst_QRandomAccessAsyncFile::fileClosedInProgress() +{ + QFETCH(const Ownership, ownership); + QFETCH(const QIOOperation::Type, operation); + + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODevice::ReadWrite)); + + constexpr qint64 OneMb = 1024 * 1024; + std::array operations; + std::array buffers; + + for (size_t i = 0; i < operations.size(); ++i) { + const qint64 offset = i * OneMb; + QIOOperation *op = nullptr; + if (operation == QIOOperation::Type::Read) { + if (ownership == Ownership::Owning) { + op = file.read(offset, OneMb); + } else { + buffers[i].resizeForOverwrite(OneMb); + op = file.readInto(offset, as_writable_bytes(QSpan{buffers[i]})); + } + } else if (operation == QIOOperation::Type::Write) { + if (ownership == Ownership::Owning) { + op = file.write(offset, QByteArray(OneMb, 'd')); + } else { + buffers[i] = QByteArray(OneMb, 'd'); + op = file.writeFrom(offset, as_bytes(QSpan{buffers[i]})); + } + } + QVERIFY(op); + operations[i] = op; + } + file.close(); + + auto isAbortedOrComplete = [](QIOOperation *op) { + return op->error() == QIOOperation::Error::Aborted + || op->error() == QIOOperation::Error::None; + }; + for (auto op : operations) { + QTRY_VERIFY(op->isFinished()); + QVERIFY(isAbortedOrComplete(op)); + } +} + +void tst_QRandomAccessAsyncFile::fileRemovedInProgress_data() +{ + generateReadWriteOperationColumns(); +} + +void tst_QRandomAccessAsyncFile::fileRemovedInProgress() +{ + QFETCH(const Ownership, ownership); + QFETCH(const QIOOperation::Type, operation); + + constexpr qint64 OneMb = 1024 * 1024; + std::array operations; + std::array buffers; + + { + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODevice::ReadWrite)); + + for (size_t i = 0; i < operations.size(); ++i) { + const qint64 offset = i * OneMb; + QIOOperation *op = nullptr; + if (operation == QIOOperation::Type::Read) { + if (ownership == Ownership::Owning) { + op = file.read(offset, OneMb); + } else { + buffers[i].resizeForOverwrite(OneMb); + op = file.readInto(offset, as_writable_bytes(QSpan{buffers[i]})); + } + } else if (operation == QIOOperation::Type::Write) { + if (ownership == Ownership::Owning) { + op = file.write(offset, QByteArray(OneMb, 'd')); + } else { + buffers[i] = QByteArray(OneMb, 'd'); + op = file.writeFrom(offset, as_bytes(QSpan{buffers[i]})); + } + } + QVERIFY(op); + operations[i] = op; + } + } + // The file and all operations are removed at this point. + // We're just checking that nothing crashes. +} + +void tst_QRandomAccessAsyncFile::operationsDeletedInProgress_data() +{ + generateReadWriteOperationColumns(); +} + +void tst_QRandomAccessAsyncFile::operationsDeletedInProgress() +{ + QFETCH(const Ownership, ownership); + QFETCH(const QIOOperation::Type, operation); + + QRandomAccessAsyncFile file; + QVERIFY(file.open(m_file.fileName(), QIODevice::ReadWrite)); + + constexpr qint64 OneMb = 1024 * 1024; + std::array operations; + std::array buffers; + + for (size_t i = 0; i < operations.size(); ++i) { + const qint64 offset = i * OneMb; + QIOOperation *op = nullptr; + if (operation == QIOOperation::Type::Read) { + if (ownership == Ownership::Owning) { + op = file.read(offset, OneMb); + } else { + buffers[i].resizeForOverwrite(OneMb); + op = file.readInto(offset, as_writable_bytes(QSpan{buffers[i]})); + } + } else if (operation == QIOOperation::Type::Write) { + if (ownership == Ownership::Owning) { + op = file.write(offset, QByteArray(OneMb, 'd')); + } else { + buffers[i] = QByteArray(OneMb, 'd'); + op = file.writeFrom(offset, as_bytes(QSpan{buffers[i]})); + } + } + QVERIFY(op); + operations[i] = op; + } + + // Make sure some operation is started + QCoreApplication::processEvents(); + + // Delete all operations. We simply make sure that nothing crashes. + for (auto op : operations) + delete op; +} + +void tst_QRandomAccessAsyncFile::generateReadWriteOperationColumns() +{ + QTest::addColumn("ownership"); + QTest::addColumn("operation"); + + constexpr struct OwnershipInfo { + Ownership own; + const char name[10]; + } values[] = { + { Ownership::Owning, "owning" }, + { Ownership::NonOwning, "nonowning" } + }; + + for (const auto &v : values) { + QTest::addRow("read_%s", v.name) << v.own << QIOOperation::Type::Read; + QTest::addRow("write_%s", v.name) << v.own << QIOOperation::Type::Write; + } +} + +QTEST_MAIN(tst_QRandomAccessAsyncFile) + +#include "tst_qrandomaccessasyncfile.moc"