Enable cleartext HTTP/2 and bring the auto-test back to life

HTTP/2 does not require TLS connection, it can work in a cleartext mode.
Plus at the moment only OpenSSL backend  allows HTTP/2 negotiation
via ALPN/NPN (and none of our CI configurations with OpenSSL supports
these extensions, rendering HTTP/2 auto-test useless). This patch
implements cleartext HTTP/2 ('h2c') in 'direct' mode - this is
allowed if a client has a prior knowledge that HTTP/2 is supported by
a server.

Change-Id: I4978775e9732c40bc77f549b83bb4a5d1761887e
Reviewed-by: Alex Trotsenko <alex1973tr@gmail.com>
Reviewed-by: Edward Welbourne <edward.welbourne@qt.io>

src/network/access/qhttp2protocolhandler.cpp

@@ -40,7 +40,7 @@
 #include "qhttpnetworkconnection_p.h"
 #include "qhttp2protocolhandler_p.h"
 
-#if !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#if !defined(QT_NO_HTTP)
 
 #include "http2/bitstreams_p.h"
 
@@ -1210,4 +1210,4 @@ void QHttp2ProtocolHandler::closeSession()
 
 QT_END_NAMESPACE
 
-#endif // !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#endif // !defined(QT_NO_HTTP)

src/network/access/qhttp2protocolhandler_p.h

@@ -55,7 +55,7 @@
 #include <private/qabstractprotocolhandler_p.h>
 #include <private/qhttpnetworkrequest_p.h>
 
-#if !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#if !defined(QT_NO_HTTP)
 
 #include "http2/http2protocol_p.h"
 #include "http2/http2streams_p.h"
@@ -202,6 +202,6 @@ private:
 
 QT_END_NAMESPACE
 
-#endif // !defined(QT_NO_HTTP) && !defined(QT_NO_SSL)
+#endif // !defined(QT_NO_HTTP)
 
 #endif

src/network/access/qhttpnetworkconnectionchannel.cpp

@@ -179,8 +179,11 @@ void QHttpNetworkConnectionChannel::init()
         if (!sslConfiguration.isNull())
            sslSocket->setSslConfiguration(sslConfiguration);
     } else {
-#endif // QT_NO_SSL
+#endif // !QT_NO_SSL
-        protocolHandler.reset(new QHttpProtocolHandler(this));
+        if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2)
+            protocolHandler.reset(new QHttp2ProtocolHandler(this));
+        else
+            protocolHandler.reset(new QHttpProtocolHandler(this));
 #ifndef QT_NO_SSL
     }
 #endif
@@ -835,10 +838,17 @@ void QHttpNetworkConnectionChannel::_q_connected()
 #endif
     } else {
         state = QHttpNetworkConnectionChannel::IdleState;
-        if (!reply)
+        if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
-            connection->d_func()->dequeueRequest(socket);
+            if (spdyRequestsToSend.count() > 0) {
-        if (reply)
+                // wait for data from the server first (e.g. initial window, max concurrent requests)
-            sendRequest();
+                QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+            }
+        } else {
+            if (!reply)
+                connection->d_func()->dequeueRequest(socket);
+            if (reply)
+                sendRequest();
+        }
     }
 }
 
@@ -972,9 +982,12 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
         }
     } while (!connection->d_func()->highPriorityQueue.isEmpty()
              || !connection->d_func()->lowPriorityQueue.isEmpty());
+
+    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2
 #ifndef QT_NO_SSL
-    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY ||
+        || connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY
-        connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
+#endif
+       ) {
         QList<HttpMessagePair> spdyPairs = spdyRequestsToSend.values();
         for (int a = 0; a < spdyPairs.count(); ++a) {
             // emit error for all replies
@@ -983,7 +996,6 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
             emit currentReply->finishedWithError(errorCode, errorString);
         }
     }
-#endif // QT_NO_SSL
 
     // send the next request
     QMetaObject::invokeMethod(that, "_q_startNextRequest", Qt::QueuedConnection);
@@ -1005,20 +1017,19 @@ void QHttpNetworkConnectionChannel::_q_error(QAbstractSocket::SocketError socket
 #ifndef QT_NO_NETWORKPROXY
 void QHttpNetworkConnectionChannel::_q_proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator* auth)
 {
+    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2
 #ifndef QT_NO_SSL
-    if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY ||
+        || connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY
-        connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
+#endif
+        ) {
         connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
     } else { // HTTP
-#endif // QT_NO_SSL
         // Need to dequeue the request before we can emit the error.
         if (!reply)
             connection->d_func()->dequeueRequest(socket);
         if (reply)
             connection->d_func()->emitProxyAuthenticationRequired(this, proxy, auth);
-#ifndef QT_NO_SSL
     }
-#endif // QT_NO_SSL
 }
 #endif
 
@@ -1077,9 +1088,10 @@ void QHttpNetworkConnectionChannel::_q_encrypted()
     if (connection->connectionType() == QHttpNetworkConnection::ConnectionTypeSPDY ||
         connection->connectionType() == QHttpNetworkConnection::ConnectionTypeHTTP2) {
         // we call setSpdyWasUsed(true) on the replies in the SPDY handler when the request is sent
-        if (spdyRequestsToSend.count() > 0)
+        if (spdyRequestsToSend.count() > 0) {
             // wait for data from the server first (e.g. initial window, max concurrent requests)
             QMetaObject::invokeMethod(connection, "_q_startNextRequest", Qt::QueuedConnection);
+        }
     } else { // HTTP
         if (!reply)
             connection->d_func()->dequeueRequest(socket);

src/network/access/qhttpnetworkconnectionchannel_p.h

@@ -121,11 +121,14 @@ public:
     bool authenticationCredentialsSent;
     bool proxyCredentialsSent;
     QScopedPointer<QAbstractProtocolHandler> protocolHandler;
+    // SPDY or HTTP/2 requests; SPDY is TLS-only, but
+    // HTTP/2 can be cleartext also, that's why it's
+    // outside of QT_NO_SSL section. Sorted by priority:
+    QMultiMap<int, HttpMessagePair> spdyRequestsToSend;
 #ifndef QT_NO_SSL
     bool ignoreAllSslErrors;
     QList<QSslError> ignoreSslErrorsList;
     QSslConfiguration sslConfiguration;
-    QMultiMap<int, HttpMessagePair> spdyRequestsToSend; // sorted by priority
     void ignoreSslErrors();
     void ignoreSslErrors(const QList<QSslError> &errors);
     void setSslConfiguration(const QSslConfiguration &config);

src/network/access/qhttpthreaddelegate.cpp

@@ -285,10 +285,11 @@ void QHttpThreadDelegate::startRequest()
     urlCopy.setPort(urlCopy.port(ssl ? 443 : 80));
 
     QHttpNetworkConnection::ConnectionType connectionType
-            = QHttpNetworkConnection::ConnectionTypeHTTP;
+        = httpRequest.isHTTP2Allowed() ? QHttpNetworkConnection::ConnectionTypeHTTP2
+                                       : QHttpNetworkConnection::ConnectionTypeHTTP;
+
 #ifndef QT_NO_SSL
     if (httpRequest.isHTTP2Allowed() && ssl) {
-        connectionType = QHttpNetworkConnection::ConnectionTypeHTTP2;
         QList<QByteArray> protocols;
         protocols << QSslConfiguration::ALPNProtocolHTTP2
                   << QSslConfiguration::NextProtocolHttp1_1;

tests/auto/network/access/access.pro

@@ -12,17 +12,12 @@ SUBDIRS=\
    qftp \
    qhttpnetworkreply \
    qabstractnetworkcache \
-   hpack
+   hpack \
+   http2
 
 !contains(QT_CONFIG, private_tests): SUBDIRS -= \
           qhttpnetworkconnection \
           qhttpnetworkreply \
           qftp \
-          hpack
+          hpack \
-
+          http2
-contains(QT_CONFIG, openssl) | contains(QT_CONFIG, openssl-linked) {
-    contains(QT_CONFIG, private_tests) {
-        SUBDIRS += \
-            http2
-    }
-}

tests/auto/network/access/http2/http2.pro

@@ -1,7 +1,6 @@
 QT += core core-private network network-private testlib
 
 CONFIG += testcase parallel_test c++11
-TEMPLATE = app
 TARGET = tst_http2
 HEADERS += http2srv.h
 SOURCES += tst_http2.cpp http2srv.cpp

tests/auto/network/access/http2/http2srv.cpp

@@ -33,9 +33,13 @@
 
 #include "http2srv.h"
 
+#ifndef QT_NO_SSL
 #include <QtNetwork/qsslconfiguration.h>
-#include <QtNetwork/qhostaddress.h>
 #include <QtNetwork/qsslkey.h>
+#endif
+
+#include <QtNetwork/qabstractsocket.h>
+
 #include <QtCore/qdebug.h>
 #include <QtCore/qlist.h>
 #include <QtCore/qfile.h>
@@ -60,8 +64,9 @@ inline bool is_valid_client_stream(quint32 streamID)
 
 }
 
-Http2Server::Http2Server(const Http2Settings &ss, const Http2Settings &cs)
+Http2Server::Http2Server(bool h2c, const Http2Settings &ss, const Http2Settings &cs)
-    : serverSettings(ss)
+    : serverSettings(ss),
+      clearTextHTTP2(h2c)
 {
     for (const auto &s : cs)
         expectedClientSettings[quint16(s.identifier)] = s.value;
@@ -97,6 +102,11 @@ void Http2Server::setResponseBody(const QByteArray &body)
 
 void Http2Server::startServer()
 {
+#ifdef QT_NO_SSL
+    // Let the test fail with timeout.
+    if (!clearTextHTTP2)
+        return;
+#endif
     if (listen())
         emit serverStarted(serverPort());
 }
@@ -160,19 +170,17 @@ void Http2Server::sendDATA(quint32 streamID, quint32 windowSize)
     Q_ASSERT(offset < quint32(responseBody.size()));
 
     const quint32 bytes = std::min<quint32>(windowSize, responseBody.size() - offset);
-    outboundFrame.start(FrameType::DATA, FrameFlag::EMPTY, streamID);
+    const quint32 frameSizeLimit(clientSetting(Settings::MAX_FRAME_SIZE_ID, Http2::maxFrameSize));
+    const uchar *src = reinterpret_cast<const uchar *>(responseBody.constData() + offset);
     const bool last = offset + bytes == quint32(responseBody.size());
 
-    const quint32 frameSizeLimit(clientSetting(Settings::MAX_FRAME_SIZE_ID, Http2::maxFrameSize));
+    outboundFrame.start(FrameType::DATA, FrameFlag::EMPTY, streamID);
-    outboundFrame.writeDATA(*socket, frameSizeLimit,
+    outboundFrame.writeDATA(*socket, frameSizeLimit, src, bytes);
-                            reinterpret_cast<const uchar *>(responseBody.constData() + offset),
-                            bytes);
 
     if (last) {
         outboundFrame.start(FrameType::DATA, FrameFlag::END_STREAM, streamID);
         outboundFrame.setPayloadSize(0);
         outboundFrame.write(*socket);
-
         suspendedStreams.erase(it);
         activeRequests.erase(streamID);
 
@@ -194,31 +202,45 @@ void Http2Server::sendWINDOW_UPDATE(quint32 streamID, quint32 delta)
 
 void Http2Server::incomingConnection(qintptr socketDescriptor)
 {
-    socket.reset(new QSslSocket);
+    if (clearTextHTTP2) {
-    // Add HTTP2 as supported protocol:
+        socket.reset(new QTcpSocket);
-    auto conf = QSslConfiguration::defaultConfiguration();
+        const bool set = socket->setSocketDescriptor(socketDescriptor);
-    auto protos = conf.allowedNextProtocols();
+        Q_UNUSED(set) Q_ASSERT(set);
-    protos.prepend(QSslConfiguration::ALPNProtocolHTTP2);
+        // Stop listening:
-    conf.setAllowedNextProtocols(protos);
+        close();
-    socket->setSslConfiguration(conf);
+        QMetaObject::invokeMethod(this, "connectionEstablished",
-    // SSL-related setup ...
+                                  Qt::QueuedConnection);
-    socket->setPeerVerifyMode(QSslSocket::VerifyNone);
+    } else {
-    socket->setProtocol(QSsl::TlsV1_2OrLater);
+#ifndef QT_NO_SSL
-    connect(socket.data(), SIGNAL(sslErrors(QList<QSslError>)),
+        socket.reset(new QSslSocket);
-            this, SLOT(ignoreErrorSlot()));
+        QSslSocket *sslSocket = static_cast<QSslSocket *>(socket.data());
-    QFile file(SRCDIR "certs/fluke.key");
+        // Add HTTP2 as supported protocol:
-    file.open(QIODevice::ReadOnly);
+        auto conf = QSslConfiguration::defaultConfiguration();
-    QSslKey key(file.readAll(), QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey);
+        auto protos = conf.allowedNextProtocols();
-    socket->setPrivateKey(key);
+        protos.prepend(QSslConfiguration::ALPNProtocolHTTP2);
-    auto localCert = QSslCertificate::fromPath(SRCDIR "certs/fluke.cert");
+        conf.setAllowedNextProtocols(protos);
-    socket->setLocalCertificateChain(localCert);
+        sslSocket->setSslConfiguration(conf);
-    socket->setSocketDescriptor(socketDescriptor, QAbstractSocket::ConnectedState);
+        // SSL-related setup ...
-    // Stop listening.
+        sslSocket->setPeerVerifyMode(QSslSocket::VerifyNone);
-    close();
+        sslSocket->setProtocol(QSsl::TlsV1_2OrLater);
-    // Start SSL handshake and ALPN:
+        connect(sslSocket, SIGNAL(sslErrors(QList<QSslError>)),
-    connect(socket.data(), SIGNAL(encrypted()),
+                this, SLOT(ignoreErrorSlot()));
-            this, SLOT(connectionEncrypted()));
+        QFile file(SRCDIR "certs/fluke.key");
-    socket->startServerEncryption();
+        file.open(QIODevice::ReadOnly);
+        QSslKey key(file.readAll(), QSsl::Rsa, QSsl::Pem, QSsl::PrivateKey);
+        sslSocket->setPrivateKey(key);
+        auto localCert = QSslCertificate::fromPath(SRCDIR "certs/fluke.cert");
+        sslSocket->setLocalCertificateChain(localCert);
+        sslSocket->setSocketDescriptor(socketDescriptor, QAbstractSocket::ConnectedState);
+        // Stop listening.
+        close();
+        // Start SSL handshake and ALPN:
+        connect(sslSocket, SIGNAL(encrypted()), this, SLOT(connectionEstablished()));
+        sslSocket->startServerEncryption();
+#else
+        Q_UNREACHABLE();
+#endif
+    }
 }
 
 quint32 Http2Server::clientSetting(Http2::Settings identifier, quint32 defaultValue)
@@ -229,7 +251,7 @@ quint32 Http2Server::clientSetting(Http2::Settings identifier, quint32 defaultVa
     return defaultValue;
 }
 
-void Http2Server::connectionEncrypted()
+void Http2Server::connectionEstablished()
 {
     using namespace Http2;
 
@@ -250,7 +272,9 @@ void Http2Server::connectionEncrypted()
 
 void Http2Server::ignoreErrorSlot()
 {
-    socket->ignoreSslErrors();
+#ifndef QT_NO_SSL
+    static_cast<QSslSocket *>(socket.data())->ignoreSslErrors();
+#endif
 }
 
 // Now HTTP2 "server" part:

tests/auto/network/access/http2/http2srv.h

@@ -33,9 +33,9 @@
 #include <QtNetwork/private/http2frames_p.h>
 #include <QtNetwork/private/hpack_p.h>
 
+#include <QtNetwork/qabstractsocket.h>
 #include <QtCore/qscopedpointer.h>
 #include <QtNetwork/qtcpserver.h>
-#include <QtNetwork/qsslsocket.h>
 #include <QtCore/qbytearray.h>
 #include <QtCore/qglobal.h>
 
@@ -62,7 +62,7 @@ class Http2Server : public QTcpServer
 {
     Q_OBJECT
 public:
-    Http2Server(const Http2Settings &serverSettings,
+    Http2Server(bool clearText, const Http2Settings &serverSettings,
                 const Http2Settings &clientSettings);
 
     ~Http2Server();
@@ -105,7 +105,7 @@ Q_SIGNALS:
     void windowUpdate(quint32 streamID);
 
 private slots:
-    void connectionEncrypted();
+    void connectionEstablished();
     void readReady();
 
 private:
@@ -113,7 +113,7 @@ private:
 
     quint32 clientSetting(Http2::Settings identifier, quint32 defaultValue);
 
-    QScopedPointer<QSslSocket> socket;
+    QScopedPointer<QAbstractSocket> socket;
 
     // Connection preface:
     bool waitingClientPreface = false;
@@ -155,6 +155,7 @@ private:
     quint32 streamRecvWindowSize = Http2::defaultSessionWindowSize;
 
     QByteArray responseBody;
+    bool clearTextHTTP2 = false;
 
 protected slots:
     void ignoreErrorSlot();

tests/auto/network/access/http2/tst_http2.cpp

@@ -49,7 +49,9 @@
 
 // At the moment our HTTP/2 imlpementation requires ALPN and this means OpenSSL.
 #if !defined(QT_NO_OPENSSL) && OPENSSL_VERSION_NUMBER >= 0x10002000L && !defined(OPENSSL_NO_TLSEXT)
-#define QT_ALPN
+const bool clearTextHTTP2 = false;
+#else
+const bool clearTextHTTP2 = true;
 #endif
 
 QT_BEGIN_NAMESPACE
@@ -139,9 +141,6 @@ tst_Http2::~tst_Http2()
 
 void tst_Http2::singleRequest()
 {
-#ifndef QT_ALPN
-    QSKIP("This test requires ALPN support");
-#endif
     clearHTTP2State();
 
     serverPort = 0;
@@ -154,7 +153,9 @@ void tst_Http2::singleRequest()
 
     QVERIFY(serverPort != 0);
 
-    const QUrl url(QString("https://127.0.0.1:%1/index.html").arg(serverPort));
+    const QString urlAsString(clearTextHTTP2 ? QString("http://127.0.0.1:%1/index.html")
+                                             : QString("https://127.0.0.1:%1/index.html"));
+    const QUrl url(urlAsString.arg(serverPort));
 
     QNetworkRequest request(url);
     request.setAttribute(QNetworkRequest::HTTP2AllowedAttribute, QVariant(true));
@@ -179,9 +180,6 @@ void tst_Http2::singleRequest()
 
 void tst_Http2::multipleRequests()
 {
-#ifndef QT_ALPN
-    QSKIP("This test requires ALPN support");
-#endif
     clearHTTP2State();
 
     serverPort = 0;
@@ -216,16 +214,12 @@ void tst_Http2::multipleRequests()
 
 void tst_Http2::flowControlClientSide()
 {
-#ifndef QT_ALPN
-    QSKIP("This test requires ALPN support");
-#endif
     // Create a server but impose limits:
     // 1. Small MAX frame size, so we test CONTINUATION frames.
     // 2. Small client windows so server responses cause client streams
     //    to suspend and server sends WINDOW_UPDATE frames.
     // 3. Few concurrent streams, to test protocol handler can resume
     //    suspended requests.
-
     using namespace Http2;
 
     clearHTTP2State();
@@ -238,7 +232,7 @@ void tst_Http2::flowControlClientSide()
 
     auto srv = newServer(serverSettings);
 
-    const QByteArray respond(int(Http2::defaultSessionWindowSize * 100), 'x');
+    const QByteArray respond(int(Http2::defaultSessionWindowSize * 50), 'x');
     srv->setResponseBody(respond);
 
     QMetaObject::invokeMethod(srv, "startServer", Qt::QueuedConnection);
@@ -249,7 +243,7 @@ void tst_Http2::flowControlClientSide()
     for (int i = 0; i < nRequests; ++i)
         sendRequest(i);
 
-    runEventLoop(10000);
+    runEventLoop(120000);
 
     QVERIFY(nRequests == 0);
     QVERIFY(prefaceOK);
@@ -261,9 +255,6 @@ void tst_Http2::flowControlClientSide()
 
 void tst_Http2::flowControlServerSide()
 {
-#ifndef QT_ALPN
-    QSKIP("This test requires ALPN support");
-#endif
     // Quite aggressive test:
     // low MAX_FRAME_SIZE forces a lot of small DATA frames,
     // payload size exceedes stream/session RECV window sizes
@@ -281,7 +272,7 @@ void tst_Http2::flowControlServerSide()
 
     auto srv = newServer(serverSettings);
 
-    const QByteArray payload(int(Http2::defaultSessionWindowSize * 1000), 'x');
+    const QByteArray payload(int(Http2::defaultSessionWindowSize * 500), 'x');
 
     QMetaObject::invokeMethod(srv, "startServer", Qt::QueuedConnection);
 
@@ -333,7 +324,7 @@ Http2Server *tst_Http2::newServer(const Http2Settings &serverSettings)
     // Client's settings are fixed by qhttp2protocolhandler.
     const Http2Settings clientSettings = {{Settings::MAX_FRAME_SIZE_ID, quint32(Http2::maxFrameSize)},
                                           {Settings::ENABLE_PUSH_ID, quint32(0)}};
-    auto srv = new Http2Server(serverSettings, clientSettings);
+    auto srv = new Http2Server(clearTextHTTP2, serverSettings, clientSettings);
 
     using Srv = Http2Server;
     using Cl = tst_Http2;
@@ -357,7 +348,8 @@ void tst_Http2::sendRequest(int streamNumber,
                             QNetworkRequest::Priority priority,
                             const QByteArray &payload)
 {
-    static const QString urlAsString("https://127.0.0.1:%1/stream%2.html");
+    static const QString urlAsString(clearTextHTTP2 ? "http://127.0.0.1:%1/stream%2.html"
+                                                    : "https://127.0.0.1:%1/stream%2.html");
 
     const QUrl url(urlAsString.arg(serverPort).arg(streamNumber));
     QNetworkRequest request(url);