Fix overflow in SHA-3/Keccak

state->rate is always larger than or equal to state->bitsInQueue;
when bitsInQueue == rate the queue is consumed and bitsInQueue is set to
0 again.

Done-with: Marc Mutz <marc.mutz@qt.io>
Pick-to: 6.5.0 6.5 6.4.3 6.4 6.2 5.15
Change-Id: I56d268a19fb3cd542cc027edc962253f09d97a14
Reviewed-by: Marc Mutz <marc.mutz@qt.io>
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

src/3rdparty/sha3/KeccakSponge.c

@@ -170,9 +170,10 @@ static int Absorb(spongeState *state, const unsigned char *data, unsigned long l
             i += wholeBlocks*state->rate;
         }
         else {
-            partialBlock = (unsigned int)(databitlen - i);
-            if (partialBlock+state->bitsInQueue > state->rate)
+            if (databitlen-i > state->rate - state->bitsInQueue)
                 partialBlock = state->rate-state->bitsInQueue;
+            else
+                partialBlock = (unsigned int)(databitlen - i);
             partialByte = partialBlock % 8;
             partialBlock -= partialByte;
             memcpy(state->dataQueue+state->bitsInQueue/8, data+i/8, partialBlock/8);

src/3rdparty/sha3/overflow.patch

@@ -0,0 +1,31 @@
+From a60180d3f8ffac268f02d2d4b0b4fbf1bff50f11 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?M=C3=A5rten=20Nordheim?= <marten.nordheim@qt.io>
+Date: Wed, 8 Mar 2023 11:10:49 +0100
+Subject: [PATCH] Fix overflow in SHA-3/Keccak
+
+Pick-to: 6.5.0 6.5 6.4.3 6.4 6.2 5.15
+Change-Id: I56d268a19fb3cd542cc027edc962253f09d97a14
+---
+ src/3rdparty/sha3/KeccakSponge.c | 5 +++--
+ 1 file changed, 3 insertions(+), 2 deletions(-)
+
+diff --git a/src/3rdparty/sha3/KeccakSponge.c b/src/3rdparty/sha3/KeccakSponge.c
+index 6f3da95dbb..337c10ccaf 100644
+--- a/src/3rdparty/sha3/KeccakSponge.c
++++ b/src/3rdparty/sha3/KeccakSponge.c
+@@ -170,9 +170,10 @@ static int Absorb(spongeState *state, const unsigned char *data, unsigned long l
+             i += wholeBlocks*state->rate;
+         }
+         else {
+-            partialBlock = (unsigned int)(databitlen - i);
+-            if (partialBlock+state->bitsInQueue > state->rate)
++            if (databitlen-i > state->rate - state->bitsInQueue)
+                 partialBlock = state->rate-state->bitsInQueue;
++            else
++                partialBlock = (unsigned int)(databitlen - i);
+             partialByte = partialBlock % 8;
+             partialBlock -= partialByte;
+             memcpy(state->dataQueue+state->bitsInQueue/8, data+i/8, partialBlock/8);
+-- 
+2.39.2.vfs.0.0
+

src/3rdparty/sha3/qt_attribution.json

@@ -21,6 +21,7 @@
         "QtUsage": "Used in Qt Core (QCryptographicHash).",
         "Files": "https://keccak.team/obsolete/KeccakReferenceAndOptimized-3.2.zip - but it's obsolete",
         "Files": "KeccakF-1600-32-rvk.macros KeccakF-1600-32.macros KeccakF-1600-64.macros KeccakF-1600-interface.h KeccakF-1600-opt32.c KeccakF-1600-opt64.c KeccakF-1600-unrolling.macros KeccakNISTInterface.c KeccakNISTInterface.h KeccakSponge.c KeccakSponge.h",
+        "Files": "With overflow.patch applied",
 
         "Description": "SHA-3, originally known as Keccak, is a cryptographic hash function.",
         "Version": "3.2",

tests/auto/corelib/tools/qcryptographichash/tst_qcryptographichash.cpp

@@ -36,6 +36,7 @@ private slots:
     // keep last
     void moreThan4GiBOfData_data();
     void moreThan4GiBOfData();
+    void keccakBufferOverflow();
 private:
     void ensureLargeData();
     std::vector<char> large;
@@ -534,5 +535,33 @@ void tst_QCryptographicHash::moreThan4GiBOfData()
     QCOMPARE(single, chunked);
 }
 
+void tst_QCryptographicHash::keccakBufferOverflow()
+{
+#if QT_POINTER_SIZE == 4
+    QSKIP("This is a 64-bit-only test");
+#else
+
+    if (ensureLargeData(); large.empty())
+        return;
+
+    QElapsedTimer timer;
+    timer.start();
+    const auto sg = qScopeGuard([&] {
+        qDebug() << "test finished in" << timer.restart() << "ms";
+    });
+
+    constexpr qsizetype magic = INT_MAX/4;
+    QCOMPARE_GE(large.size(), size_t(magic + 1));
+
+    QCryptographicHash hash(QCryptographicHash::Algorithm::Keccak_224);
+    const auto first = QByteArrayView{large}.first(1);
+    const auto second = QByteArrayView{large}.sliced(1, magic);
+    hash.addData(first);
+    hash.addData(second);
+    (void)hash.resultView();
+    QVERIFY(true); // didn't crash
+#endif
+}
+
 QTEST_MAIN(tst_QCryptographicHash)
 #include "tst_qcryptographichash.moc"