AuroraMiddleware/qt5base-lts
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

QDecompressHelper: Add brotli support

Browse Source 
Task-number: QTBUG-83269
Change-Id: If23b098ee76a4892e4c2c6ce5c635688d8d9138d
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>
This commit is contained in:
Mårten Nordheim 2020-05-06 18:24:07 +02:00
parent af1544bda2
commit ad1a5bf63f
15 changed files with 243 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
cmake/FindWrapBrotli.cmake Normal file
View File

@ -0,0 +1,43 @@
if(TARGET WrapBrotli::WrapBrotliDec)
set(WrapBrotli_FOUND ON)
return()
endif()
# From VCPKG
find_package(unofficial-brotli CONFIG QUIET)
if (unofficial-brotli_FOUND)
add_library(WrapBrotli::WrapBrotliDec INTERFACE IMPORTED)
target_link_libraries(WrapBrotli::WrapBrotliDec INTERFACE unofficial::brotli::brotlidec)
add_library(WrapBrotli::WrapBrotliEnc INTERFACE IMPORTED)
target_link_libraries(WrapBrotli::WrapBrotliEnc INTERFACE unofficial::brotli::brotlienc)
add_library(WrapBrotli::WrapBrotliCommon INTERFACE IMPORTED)
target_link_libraries(WrapBrotli::WrapBrotliCommon INTERFACE unofficial::brotli::brotlicommon)
set(WrapBrotli_FOUND ON)
else()
find_package(PkgConfig QUIET)
if (PKG_CONFIG_FOUND)
pkg_check_modules(libbrotlidec QUIET libbrotlidec IMPORTED_TARGET)
if (libbrotlidec_FOUND)
add_library(WrapBrotli::WrapBrotliDec INTERFACE IMPORTED)
target_link_libraries(WrapBrotli::WrapBrotliDec INTERFACE PkgConfig::libbrotlidec)
set(WrapBrotli_FOUND ON)
endif()
pkg_check_modules(libbrotlienc QUIET libbrotlienc IMPORTED_TARGET)
if (libbrotlienc_FOUND)
add_library(WrapBrotli::WrapBrotliEnc INTERFACE IMPORTED)
target_link_libraries(WrapBrotli::WrapBrotliEnc INTERFACE PkgConfig::libbrotlienc)
set(WrapBrotli_FOUND ON)
endif()
pkg_check_modules(libbrotlicommon QUIET libbrotlicommon IMPORTED_TARGET)
if (libbrotlicommon_FOUND)
add_library(WrapBrotli::WrapBrotliCommon INTERFACE IMPORTED)
target_link_libraries(WrapBrotli::WrapBrotliCommon INTERFACE PkgConfig::libbrotlicommon)
set(WrapBrotli_FOUND ON)
endif()
endif()
endif()

5
src/network/.prev_CMakeLists.txt
View File

@ -123,6 +123,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_http
socket/qhttpsocketengine.cpp socket/qhttpsocketengine_p.h
)
qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
LIBRARIES
WrapBrotli::WrapBrotliDec
)
qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
LIBRARIES
ZLIB::ZLIB

5
src/network/CMakeLists.txt
View File

@ -123,6 +123,11 @@ qt_extend_target(Network CONDITION QT_FEATURE_http
socket/qhttpsocketengine.cpp socket/qhttpsocketengine_p.h
)
qt_extend_target(Network CONDITION QT_FEATURE_brotli AND QT_FEATURE_http
LIBRARIES
WrapBrotli::WrapBrotliDec
)
qt_extend_target(Network CONDITION QT_FEATURE_system_zlib
LIBRARIES
ZLIB::ZLIB

4
src/network/access/access.pri
View File

@ -116,4 +116,8 @@ qtConfig(http) {
access/qhttpthreaddelegate_p.h \
access/qnetworkreplyhttpimpl_p.h \
access/qhttp2configuration.h
qtConfig(brotli) {
QMAKE_USE_PRIVATE += brotli
}
}

115
src/network/access/qdecompresshelper.cpp
View File

@ -44,6 +44,10 @@
#include <zlib.h>
#if QT_CONFIG(brotli)
# include <brotli/decode.h>
#endif
#include <array>
QT_BEGIN_NAMESPACE
@ -57,6 +61,9 @@ struct ContentEncodingMapping
constexpr ContentEncodingMapping contentEncodingMapping[] {
{ "deflate", QDecompressHelper::Deflate },
{ "gzip", QDecompressHelper::GZip },
#if QT_CONFIG(brotli)
{ "br", QDecompressHelper::Brotli },
#endif
};
QDecompressHelper::ContentEncoding encodingFromByteArray(const QByteArray &ce) noexcept
@ -72,6 +79,13 @@ z_stream *toZlibPointer(void *ptr)
{
return static_cast<z_stream_s *>(ptr);
}
#if QT_CONFIG(brotli)
BrotliDecoderState *toBrotliPointer(void *ptr)
{
return static_cast<BrotliDecoderState *>(ptr);
}
#endif
}
bool QDecompressHelper::isSupportedEncoding(const QByteArray &encoding)
@ -134,6 +148,13 @@ bool QDecompressHelper::setEncoding(ContentEncoding ce)
decoderPointer = inflateStream;
break;
}
case Brotli:
#if QT_CONFIG(brotli)
decoderPointer = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);
#else
Q_UNREACHABLE();
#endif
break;
}
if (!decoderPointer) {
qWarning("Failed to initialize the decoder.");
@ -325,6 +346,9 @@ qsizetype QDecompressHelper::read(char *data, qsizetype maxSize)
case GZip:
bytesRead = readZLib(data, maxSize);
break;
case Brotli:
bytesRead = readBrotli(data, maxSize);
break;
}
if (bytesRead == -1)
clear();
@ -369,6 +393,14 @@ void QDecompressHelper::clear()
delete inflateStream;
break;
}
case Brotli: {
#if QT_CONFIG(brotli)
BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
if (brotliDecoderState)
BrotliDecoderDestroyInstance(brotliDecoderState);
#endif
break;
}
}
decoderPointer = nullptr;
contentEncoding = None;
@ -489,4 +521,87 @@ qsizetype QDecompressHelper::readZLib(char *data, const qsizetype maxSize)
return bytesDecoded;
}
qsizetype QDecompressHelper::readBrotli(char *data, const qsizetype maxSize)
{
#if !QT_CONFIG(brotli)
Q_UNUSED(data);
Q_UNUSED(maxSize);
Q_UNREACHABLE();
#else
qint64 bytesDecoded = 0;
BrotliDecoderState *brotliDecoderState = toBrotliPointer(decoderPointer);
while (decoderHasData && bytesDecoded < maxSize) {
Q_ASSERT(brotliUnconsumedDataPtr || BrotliDecoderHasMoreOutput(brotliDecoderState));
if (brotliUnconsumedDataPtr) {
Q_ASSERT(brotliUnconsumedAmount);
size_t toRead = std::min(size_t(maxSize - bytesDecoded), brotliUnconsumedAmount);
memcpy(data + bytesDecoded, brotliUnconsumedDataPtr, toRead);
bytesDecoded += toRead;
brotliUnconsumedAmount -= toRead;
brotliUnconsumedDataPtr += toRead;
if (brotliUnconsumedAmount == 0) {
brotliUnconsumedDataPtr = nullptr;
decoderHasData = false;
}
}
if (BrotliDecoderHasMoreOutput(brotliDecoderState) == BROTLI_TRUE) {
brotliUnconsumedDataPtr =
BrotliDecoderTakeOutput(brotliDecoderState, &brotliUnconsumedAmount);
decoderHasData = true;
}
}
if (bytesDecoded == maxSize)
return bytesDecoded;
Q_ASSERT(bytesDecoded < maxSize);
QByteArray input;
if (!compressedDataBuffer.isEmpty())
input = compressedDataBuffer.read();
const uint8_t *encodedPtr = reinterpret_cast<const uint8_t *>(input.constData());
size_t encodedBytesRemaining = input.size();
uint8_t *decodedPtr = reinterpret_cast<uint8_t *>(data + bytesDecoded);
size_t unusedDecodedSize = size_t(maxSize - bytesDecoded);
while (unusedDecodedSize > 0) {
auto previousUnusedDecodedSize = unusedDecodedSize;
BrotliDecoderResult result = BrotliDecoderDecompressStream(
brotliDecoderState, &encodedBytesRemaining, &encodedPtr, &unusedDecodedSize,
&decodedPtr, nullptr);
bytesDecoded += previousUnusedDecodedSize - unusedDecodedSize;
switch (result) {
case BROTLI_DECODER_RESULT_ERROR:
qWarning("Brotli error: %s",
BrotliDecoderErrorString(BrotliDecoderGetErrorCode(brotliDecoderState)));
return -1;
case BROTLI_DECODER_RESULT_SUCCESS:
BrotliDecoderDestroyInstance(brotliDecoderState);
decoderPointer = nullptr;
return bytesDecoded;
case BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT:
if (!compressedDataBuffer.isEmpty()) {
input = compressedDataBuffer.read();
encodedPtr = reinterpret_cast<const uint8_t *>(input.constData());
encodedBytesRemaining = input.size();
break;
}
return bytesDecoded;
case BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT:
// Some data is leftover inside the brotli decoder, remember for next time
decoderHasData = BrotliDecoderHasMoreOutput(brotliDecoderState);
Q_ASSERT(unusedDecodedSize == 0);
break;
}
}
if (encodedBytesRemaining) {
// Some input was left unused; move back to the buffer
input = input.right(QByteArray::size_type(encodedBytesRemaining));
compressedDataBuffer.prepend(input);
}
return bytesDecoded;
#endif
}
QT_END_NAMESPACE

6
src/network/access/qdecompresshelper_p.h
View File

@ -66,6 +66,7 @@ public:
None,
Deflate,
GZip,
Brotli,
};
QDecompressHelper() = default;
@ -101,6 +102,7 @@ private:
qint64 encodedBytesAvailable() const;
qsizetype readZLib(char *data, qsizetype maxSize);
qsizetype readBrotli(char *data, qsizetype maxSize);
QByteDataBuffer compressedDataBuffer;
bool decoderHasData = false;
@ -112,6 +114,10 @@ private:
ContentEncoding contentEncoding = None;
void *decoderPointer = nullptr;
#if QT_CONFIG(brotli)
const uint8_t *brotliUnconsumedDataPtr = nullptr;
size_t brotliUnconsumedAmount = 0;
#endif
};
QT_END_NAMESPACE

9
src/network/configure.cmake
View File

@ -6,6 +6,7 @@
#### Libraries
qt_find_package(WrapBrotli PROVIDED_TARGETS WrapBrotli::WrapBrotliDec)
qt_find_package(Libproxy PROVIDED_TARGETS PkgConfig::Libproxy MODULE_NAME network QMAKE_LIB libproxy)
qt_find_package(WrapOpenSSLHeaders PROVIDED_TARGETS WrapOpenSSLHeaders::WrapOpenSSLHeaders MODULE_NAME network QMAKE_LIB openssl_headers)
# openssl_headers
@ -366,6 +367,13 @@ qt_feature("networkdiskcache" PUBLIC
CONDITION QT_FEATURE_temporaryfile
)
qt_feature_definition("networkdiskcache" "QT_NO_NETWORKDISKCACHE" NEGATE VALUE "1")
qt_feature("brotli" PUBLIC
SECTION "Networking"
LABEL "Brotli Decompression Support"
PURPOSE "Support for downloading and decompressing resources compressed with Brotli through QNetworkAccessManager."
CONDITION WrapBrotli_FOUND
)
qt_feature_definition("brotli" "QT_NO_BROTLI" NEGATE VALUE "1")
qt_feature("localserver" PUBLIC
SECTION "Networking"
LABEL "QLocalServer"
@ -432,6 +440,7 @@ qt_configure_add_summary_entry(ARGS "ftp")
qt_configure_add_summary_entry(ARGS "sctp")
qt_configure_add_summary_entry(ARGS "system-proxies")
qt_configure_add_summary_entry(ARGS "gssapi")
qt_configure_add_summary_entry(ARGS "brotli")
qt_configure_end_summary_section() # end of "Qt Network" section
qt_configure_add_report_entry(
TYPE NOTE

25
src/network/configure.json
View File

@ -25,6 +25,21 @@
},
"libraries": {
"brotli": {
"label": "Brotli Decompression",
"test": {
"main": [
"BrotliDecoderState *state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr);"
]
},
"headers": [
"brotli/decode.h"
],
"sources": [
{ "type": "pkgConfig", "args": "libbrotlidec" },
"-lbrotlidec"
]
},
"corewlan": {
"label": "CoreWLan",
"export": "",
@ -387,6 +402,13 @@
"condition": "features.temporaryfile",
"output": [ "publicFeature", "feature" ]
},
"brotli": {
"label": "Brotli Decompression Support",
"purpose": "Support for downloading and decompressing resources compressed with Brotli through QNetworkAccessManager.",
"section": "Networking",
"condition": "libs.brotli",
"output": [ "publicFeature", "feature" ]
},
"localserver": {
"label": "QLocalServer",
"purpose": "Provides a local socket based server.",
@ -479,7 +501,8 @@ For example:
"ftp",
"sctp",
"system-proxies",
"gssapi"
"gssapi",
"brotli"
]
}
]

5
tests/auto/network/access/http2/tst_http2.cpp
View File

@ -796,6 +796,11 @@ void tst_Http2::contentEncoding_data()
contentEncodingData.emplace_back(
"deflate", QByteArray::fromBase64("eJzLSM3JyVcozy/KSQEAGgsEXQ=="), "hello world");
#if QT_CONFIG(brotli)
contentEncodingData.emplace_back("br", QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD"),
"hello world");
#endif
// Loop through and add the data...
for (const auto &data : contentEncodingData) {
const char *name = data.contentEncoding.data();

BIN
tests/auto/network/access/qdecompresshelper/4G.br Normal file
View File

Binary file not shown.

16
tests/auto/network/access/qdecompresshelper/tst_qdecompresshelper.cpp
View File

@ -90,6 +90,12 @@ void tst_QDecompressHelper::encodingSupported()
QVERIFY(accepted.contains("gzip"));
int expected = 2;
#if QT_CONFIG(brotli)
QVERIFY(QDecompressHelper::isSupportedEncoding("br"));
QVERIFY(accepted.contains("br"));
++expected;
#endif
QCOMPARE(expected, accepted.size());
}
@ -115,6 +121,12 @@ void tst_QDecompressHelper::sharedDecompress_data()
QTest::newRow("deflate-hello-world")
<< QByteArray("deflate") << QByteArray::fromBase64("eJzLSM3JyVcozy/KSQEAGgsEXQ==")
<< QByteArray("hello world");
#if QT_CONFIG(brotli)
QTest::newRow("brotli-hello-world")
<< QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
<< QByteArray("hello world");
#endif
}
void tst_QDecompressHelper::decompress_data()
@ -322,6 +334,10 @@ void tst_QDecompressHelper::decompressBigData_data()
QTest::newRow("gzip-4G") << QByteArray("gzip") << QString(":/4G.gz") << fourGiB;
QTest::newRow("deflate-5G") << QByteArray("deflate") << QString(":/5GiB.txt.inflate")
<< fiveGiB;
#if QT_CONFIG(brotli)
QTest::newRow("brotli-4G") << QByteArray("br") << (srcDir + "/4G.br") << fourGiB;
#endif
}
void tst_QDecompressHelper::decompressBigData()

6
tests/auto/network/access/qnetworkreply/tst_qnetworkreply.cpp
View File

@ -9264,6 +9264,12 @@ void tst_QNetworkReply::contentEncoding_data()
QTest::newRow("deflate-hello-world")
<< QByteArray("deflate") << QByteArray::fromBase64("eJzLSM3JyVcozy/KSQEAGgsEXQ==")
<< QByteArray("hello world");
#if QT_CONFIG(brotli)
QTest::newRow("brotli-hello-world")
<< QByteArray("br") << QByteArray::fromBase64("DwWAaGVsbG8gd29ybGQD")
<< QByteArray("hello world");
#endif
}
void tst_QNetworkReply::contentEncoding()

BIN
tests/benchmarks/network/access/qdecompresshelper/50mb.txt.br Normal file
View File

Binary file not shown.

5
tests/benchmarks/network/access/qdecompresshelper/main.cpp
View File

@ -34,7 +34,6 @@ class tst_QDecompressHelper : public QObject
{
Q_OBJECT
private slots:
void decompress_data();
void decompress();
};
@ -53,6 +52,10 @@ void tst_QDecompressHelper::decompress_data()
#ifndef QT_NO_COMPRESS
QTest::addRow("gzip") << QByteArray("gzip") << srcDir + QString("50mb.txt.gz");
dataAdded = true;
#endif
#if QT_CONFIG(brotli)
QTest::addRow("brotli") << QByteArray("br") << srcDir + QString("50mb.txt.br");
dataAdded = true;
#endif
if (!dataAdded)
QSKIP("There's no decompression support");

1
util/cmake/helper.py
View File

@ -372,6 +372,7 @@ _library_map = [
# 3rd party:
LibraryMapping("atspi", "ATSPI2", "PkgConfig::ATSPI2"),
LibraryMapping("bluez", "BlueZ", "PkgConfig::BlueZ"),
LibraryMapping("brotli", "WrapBrotli", "WrapBrotli::WrapBrotliDec"),
LibraryMapping("corewlan", None, None),
LibraryMapping("cups", "Cups", "Cups::Cups"),
LibraryMapping("directfb", "DirectFB", "PkgConfig::DirectFB"),