From ca0df4b2694fc3ca0c31cda0d5edec7e76f7dfcb Mon Sep 17 00:00:00 2001 From: Alex Trotsenko Date: Fri, 4 Aug 2017 15:33:47 +0300 Subject: [PATCH] Introduce QIODevice::skip() [ChangeLog][QtCore][QIODevice] Added skip() method to improve performance in read operations. Change-Id: I79068a3e9df108756abe37ba3d431e27e7413621 Reviewed-by: Edward Welbourne Reviewed-by: Oswald Buddenhagen Reviewed-by: Thiago Macieira --- src/corelib/io/qiodevice.cpp | 122 ++++++++++++++++++ src/corelib/io/qiodevice.h | 1 + src/corelib/io/qiodevice_p.h | 3 + .../corelib/io/qiodevice/tst_qiodevice.cpp | 91 +++++++++++++ 4 files changed, 217 insertions(+) diff --git a/src/corelib/io/qiodevice.cpp b/src/corelib/io/qiodevice.cpp index 80122eac5e..82fc34c537 100644 --- a/src/corelib/io/qiodevice.cpp +++ b/src/corelib/io/qiodevice.cpp @@ -1878,6 +1878,128 @@ QByteArray QIODevice::peek(qint64 maxSize) return d->peek(maxSize); } +/*! + \since 5.11 + + Skips up to \a maxSize bytes from the device. Returns the number of bytes + actually skipped, or -1 on error. + + This function does not wait and only discards the data that is already + available for reading. + + If the device is opened in text mode, end-of-line terminators are + translated to '\n' symbols and count as a single byte identically to the + read() and peek() behavior. + + This function works for all devices, including sequential ones that cannot + seek(). It is optimized to skip unwanted data after a peek() call. + + For random-access devices, skip() can be used to seek forward from the + current position. Negative \a maxSize values are not allowed. + + \sa peek(), seek(), read() +*/ +qint64 QIODevice::skip(qint64 maxSize) +{ + Q_D(QIODevice); + CHECK_MAXLEN(skip, qint64(-1)); + CHECK_READABLE(skip, qint64(-1)); + + const bool sequential = d->isSequential(); + +#if defined QIODEVICE_DEBUG + printf("%p QIODevice::skip(%lld), d->pos = %lld, d->buffer.size() = %lld\n", + this, maxSize, d->pos, d->buffer.size()); +#endif + + if ((sequential && d->transactionStarted) || (d->openMode & QIODevice::Text) != 0) + return d->skipByReading(maxSize); + + // First, skip over any data in the internal buffer. + qint64 skippedSoFar = 0; + if (!d->buffer.isEmpty()) { + skippedSoFar = d->buffer.skip(maxSize); +#if defined QIODEVICE_DEBUG + printf("%p \tskipping %lld bytes in buffer\n", this, skippedSoFar); +#endif + if (!sequential) + d->pos += skippedSoFar; + if (d->buffer.isEmpty()) + readData(nullptr, 0); + if (skippedSoFar == maxSize) + return skippedSoFar; + + maxSize -= skippedSoFar; + } + + // Try to seek on random-access device. At this point, + // the internal read buffer is empty. + if (!sequential) { + const qint64 bytesToSkip = qMin(size() - d->pos, maxSize); + + // If the size is unknown or file position is at the end, + // fall back to reading below. + if (bytesToSkip > 0) { + if (!seek(d->pos + bytesToSkip)) + return skippedSoFar ? skippedSoFar : Q_INT64_C(-1); + if (bytesToSkip == maxSize) + return skippedSoFar + bytesToSkip; + + skippedSoFar += bytesToSkip; + maxSize -= bytesToSkip; + } + } + + const qint64 skipResult = d->skip(maxSize); + if (skippedSoFar == 0) + return skipResult; + + if (skipResult == -1) + return skippedSoFar; + + return skippedSoFar + skipResult; +} + +/*! + \internal +*/ +qint64 QIODevicePrivate::skipByReading(qint64 maxSize) +{ + qint64 readSoFar = 0; + do { + char dummy[4096]; + const qint64 readBytes = qMin(maxSize, sizeof(dummy)); + const qint64 readResult = read(dummy, readBytes); + + // Do not try again, if we got less data. + if (readResult != readBytes) { + if (readSoFar == 0) + return readResult; + + if (readResult == -1) + return readSoFar; + + return readSoFar + readResult; + } + + readSoFar += readResult; + maxSize -= readResult; + } while (maxSize > 0); + + return readSoFar; +} + +/*! + \internal +*/ +qint64 QIODevicePrivate::skip(qint64 maxSize) +{ + // Base implementation discards the data by reading into the dummy buffer. + // It's slow, but this works for all types of devices. Subclasses can + // reimplement this function to improve on that. + return skipByReading(maxSize); +} + /*! Blocks until new data is available for reading and the readyRead() signal has been emitted, or until \a msecs milliseconds have diff --git a/src/corelib/io/qiodevice.h b/src/corelib/io/qiodevice.h index 162480d22f..e64a4d0bb1 100644 --- a/src/corelib/io/qiodevice.h +++ b/src/corelib/io/qiodevice.h @@ -136,6 +136,7 @@ public: qint64 peek(char *data, qint64 maxlen); QByteArray peek(qint64 maxlen); + qint64 skip(qint64 maxSize); virtual bool waitForReadyRead(int msecs); virtual bool waitForBytesWritten(int msecs); diff --git a/src/corelib/io/qiodevice_p.h b/src/corelib/io/qiodevice_p.h index 71a326dd53..de2aa1597e 100644 --- a/src/corelib/io/qiodevice_p.h +++ b/src/corelib/io/qiodevice_p.h @@ -174,6 +174,9 @@ public: qint64 read(char *data, qint64 maxSize, bool peeking = false); virtual qint64 peek(char *data, qint64 maxSize); virtual QByteArray peek(qint64 maxSize); + qint64 skipByReading(qint64 maxSize); + // ### Qt6: consider replacing with a protected virtual QIODevice::skipData(). + virtual qint64 skip(qint64 maxSize); #ifdef QT_NO_QOBJECT QIODevice *q_ptr; diff --git a/tests/auto/corelib/io/qiodevice/tst_qiodevice.cpp b/tests/auto/corelib/io/qiodevice/tst_qiodevice.cpp index a76fd4703e..a0188f8ba9 100644 --- a/tests/auto/corelib/io/qiodevice/tst_qiodevice.cpp +++ b/tests/auto/corelib/io/qiodevice/tst_qiodevice.cpp @@ -55,6 +55,10 @@ private slots: void readAllKeepPosition(); void writeInTextMode(); + void skip_data(); + void skip(); + void skipAfterPeek_data(); + void skipAfterPeek(); void transaction_data(); void transaction(); @@ -628,6 +632,93 @@ void tst_QIODevice::writeInTextMode() #endif } +void tst_QIODevice::skip_data() +{ + QTest::addColumn("sequential"); + QTest::addColumn("data"); + QTest::addColumn("read"); + QTest::addColumn("skip"); + QTest::addColumn("skipped"); + QTest::addColumn("expect"); + + QByteArray bigData; + bigData.fill('a', 20000); + bigData[10001] = 'x'; + + bool sequential = true; + do { + QByteArray devName(sequential ? "sequential" : "random-access"); + + QTest::newRow(qPrintable(devName + "-small_data")) << true << QByteArray("abcdefghij") + << 3 << 6 << 6 << 'j'; + QTest::newRow(qPrintable(devName + "-big_data")) << true << bigData + << 1 << 10000 << 10000 << 'x'; + QTest::newRow(qPrintable(devName + "-beyond_the_end")) << true << bigData + << 1 << 20000 << 19999 << '\0'; + + sequential = !sequential; + } while (!sequential); +} + +void tst_QIODevice::skip() +{ + QFETCH(bool, sequential); + QFETCH(QByteArray, data); + QFETCH(int, read); + QFETCH(int, skip); + QFETCH(int, skipped); + QFETCH(char, expect); + char lastChar = 0; + + QScopedPointer dev(sequential ? (QIODevice *) new SequentialReadBuffer(&data) + : (QIODevice *) new QBuffer(&data)); + dev->open(QIODevice::ReadOnly); + + for (int i = 0; i < read; ++i) + dev->getChar(nullptr); + + QCOMPARE(dev->skip(skip), skipped); + dev->getChar(&lastChar); + QCOMPARE(lastChar, expect); +} + +void tst_QIODevice::skipAfterPeek_data() +{ + QTest::addColumn("sequential"); + QTest::addColumn("data"); + + QByteArray bigData; + for (int i = 0; i < 1000; ++i) + bigData += "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + QTest::newRow("sequential") << true << bigData; + QTest::newRow("random-access") << false << bigData; +} + +void tst_QIODevice::skipAfterPeek() +{ + QFETCH(bool, sequential); + QFETCH(QByteArray, data); + + QScopedPointer dev(sequential ? (QIODevice *) new SequentialReadBuffer(&data) + : (QIODevice *) new QBuffer(&data)); + int readSoFar = 0; + qint64 bytesToSkip = 1; + + dev->open(QIODevice::ReadOnly); + forever { + QByteArray chunk = dev->peek(bytesToSkip); + if (chunk.isEmpty()) + break; + + QCOMPARE(dev->skip(bytesToSkip), qint64(chunk.size())); + QCOMPARE(chunk, data.mid(readSoFar, chunk.size())); + readSoFar += chunk.size(); + bytesToSkip <<= 1; + } + QCOMPARE(readSoFar, data.size()); +} + void tst_QIODevice::transaction_data() { QTest::addColumn("sequential");