QNetworkAccessManager: add public methods to pre-TCP/pre-SSL-connect

If an app knows it needs to connect to a host beforehand, it can "warm up" the
connection cache by making DNS lookup, TCP (and if needed SSL) handshake before
the actual HTTP request is sent. When the HTTP request is made, it will be
considerably faster when there is already a working connection.

Here are some typical results from the benchmark:

* Linux desktop with Ethernet:
  "http://www.google.com" full request: 279 ms, pre-connect request: 61 ms,
    difference: 218 ms
  "https://www.google.com" full request: 344 ms, pre-connect request: 60 ms,
    difference: 284 ms

* mobile device (BlackBerry 10) with Wifi:
  "https://www.google.com" full request: 898 ms, pre-connect request: 159 ms,
    difference: 739 ms
  "http://www.google.com" full request: 707 ms, pre-connect request: 200 ms,
    difference: 507 ms

Task-number: QTBUG-30771
Change-Id: I3566b7f08216ab93a39e2024ae7d1ceb7ae21891
Reviewed-by: Jonas Gastal <gastal@intel.com>
Reviewed-by: Richard J. Moore <rich@kde.org>

src/network/access/qhttpnetworkconnectionchannel.cpp

@@ -205,6 +205,16 @@ bool QHttpNetworkConnectionChannel::sendRequest()
             // _q_connected or _q_encrypted
             return false;
         }
+        QString scheme = request.url().scheme();
+        if (scheme == QLatin1String("preconnect-http")
+            || scheme == QLatin1String("preconnect-https")) {
+            state = QHttpNetworkConnectionChannel::IdleState;
+            reply->d_func()->state = QHttpNetworkReplyPrivate::AllDoneState;
+            allDone();
+            reply = 0; // so we can reuse this channel
+            return true; // we have a working connection and are done
+        }
+
         written = 0; // excluding the header
         bytesTotal = 0;
 

src/network/access/qhttpnetworkreply.cpp

@@ -290,6 +290,11 @@ QHttpNetworkReplyPrivate::QHttpNetworkReplyPrivate(const QUrl &newUrl)
 #endif
 
 {
+    QString scheme = newUrl.scheme();
+    if (scheme == QLatin1String("preconnect-http")
+            || scheme == QLatin1String("preconnect-https"))
+        // make sure we do not close the socket after preconnecting
+        connectionCloseEnabled = false;
 }
 
 QHttpNetworkReplyPrivate::~QHttpNetworkReplyPrivate()

src/network/access/qhttpthreaddelegate.cpp

@@ -107,8 +107,14 @@ static QByteArray makeCacheKey(QUrl &url, QNetworkProxy *proxy)
 {
     QString result;
     QUrl copy = url;
-    bool isEncrypted = copy.scheme().toLower() == QLatin1String("https");
+    QString scheme = copy.scheme().toLower();
+    bool isEncrypted = scheme == QLatin1String("https");
     copy.setPort(copy.port(isEncrypted ? 443 : 80));
+    if (scheme == QLatin1String("preconnect-http")) {
+        copy.setScheme(QLatin1String("http"));
+    } else if (scheme == QLatin1String("preconnect-https")) {
+        copy.setScheme(QLatin1String("https"));
+    }
     result = copy.toString(QUrl::RemoveUserInfo | QUrl::RemovePath |
                            QUrl::RemoveQuery | QUrl::RemoveFragment | QUrl::FullyEncoded);
 

src/network/access/qnetworkaccessmanager.cpp

@@ -96,9 +96,11 @@ bool getProxyAuth(const QString& proxyHostname, const QString &scheme, QString&
     SecProtocolType protocolType = kSecProtocolTypeAny;
     if (scheme.compare(QLatin1String("ftp"),Qt::CaseInsensitive)==0) {
         protocolType = kSecProtocolTypeFTP;
-    } else if (scheme.compare(QLatin1String("http"),Qt::CaseInsensitive)==0) {
+    } else if (scheme.compare(QLatin1String("http"),Qt::CaseInsensitive)==0
+               || scheme.compare(QLatin1String("preconnect-http"),Qt::CaseInsensitive)==0) {
         protocolType = kSecProtocolTypeHTTP;
-    } else if (scheme.compare(QLatin1String("https"),Qt::CaseInsensitive)==0) {
+    } else if (scheme.compare(QLatin1String("https"),Qt::CaseInsensitive)==0
+               || scheme.compare(QLatin1String("preconnect-https"),Qt::CaseInsensitive)==0) {
         protocolType = kSecProtocolTypeHTTPS;
     }
     QByteArray proxyHostnameUtf8(proxyHostname.toUtf8());
@@ -968,6 +970,53 @@ QNetworkAccessManager::NetworkAccessibility QNetworkAccessManager::networkAccess
     }
 }
 
+#ifndef QT_NO_SSL
+/*!
+    \since 5.2
+
+    Initiates a connection to the host given by \a hostName at port \a port, using
+    \a sslConfiguration. This function is useful to complete the TCP and SSL handshake
+    to a host before the HTTPS request is made, resulting in a lower network latency.
+
+    \note This function has no possibility to report errors.
+
+    \sa connectToHost(), get(), post(), put(), deleteResource()
+*/
+void QNetworkAccessManager::connectToHostEncrypted(const QString &hostName, quint16 port,
+                                                   const QSslConfiguration &sslConfiguration)
+{
+    QUrl url;
+    url.setHost(hostName);
+    url.setPort(port);
+    url.setScheme(QLatin1String("preconnect-https"));
+    QNetworkRequest request(url);
+    if (sslConfiguration != QSslConfiguration::defaultConfiguration())
+        request.setSslConfiguration(sslConfiguration);
+    get(request);
+}
+#endif
+
+/*!
+    \since 5.2
+
+    Initiates a connection to the host given by \a hostName at port \a port.
+    This function is useful to complete the TCP handshake
+    to a host before the HTTP request is made, resulting in a lower network latency.
+
+    \note This function has no possibility to report errors.
+
+    \sa connectToHostEncrypted(), get(), post(), put(), deleteResource()
+*/
+void QNetworkAccessManager::connectToHost(const QString &hostName, quint16 port)
+{
+    QUrl url;
+    url.setHost(hostName);
+    url.setPort(port);
+    url.setScheme(QLatin1String("preconnect-http"));
+    QNetworkRequest request(url);
+    get(request);
+}
+
 /*!
     \internal
 
@@ -1112,9 +1161,9 @@ QNetworkReply *QNetworkAccessManager::createRequest(QNetworkAccessManager::Opera
 
 #ifndef QT_NO_HTTP
     // Since Qt 5 we use the new QNetworkReplyHttpImpl
-    if (scheme == QLatin1String("http")
+    if (scheme == QLatin1String("http") || scheme == QLatin1String("preconnect-http")
 #ifndef QT_NO_SSL
-        || scheme == QLatin1String("https")
+        || scheme == QLatin1String("https") || scheme == QLatin1String("preconnect-https")
 #endif
         ) {
         QNetworkReplyHttpImpl *reply = new QNetworkReplyHttpImpl(this, request, op, outgoingData);

src/network/access/qnetworkaccessmanager.h

@@ -43,6 +43,9 @@
 #define QNETWORKACCESSMANAGER_H
 
 #include <QtCore/QObject>
+#ifndef QT_NO_SSL
+#include <QtNetwork/QSslConfiguration>
+#endif
 
 QT_BEGIN_NAMESPACE
 
@@ -135,6 +138,12 @@ public:
     NetworkAccessibility networkAccessible() const;
 #endif
 
+#ifndef QT_NO_SSL
+    void connectToHostEncrypted(const QString &hostName, quint16 port = 443,
+                                const QSslConfiguration &sslConfiguration = QSslConfiguration::defaultConfiguration());
+#endif
+    void connectToHost(const QString &hostName, quint16 port = 80);
+
 Q_SIGNALS:
 #ifndef QT_NO_NETWORKPROXY
     void proxyAuthenticationRequired(const QNetworkProxy &proxy, QAuthenticator *authenticator);

src/network/access/qnetworkreplyhttpimpl.cpp

@@ -629,7 +629,9 @@ void QNetworkReplyHttpImplPrivate::postRequest()
     QUrl url = request.url();
     httpRequest.setUrl(url);
 
-    bool ssl = url.scheme().toLower() == QLatin1String("https");
+    QString scheme = url.scheme().toLower();
+    bool ssl = (scheme == QLatin1String("https")
+                || scheme == QLatin1String("preconnect-https"));
     q->setAttribute(QNetworkRequest::ConnectionEncryptedAttribute, ssl);
     httpRequest.setSsl(ssl);
 

tests/benchmarks/network/access/qnetworkreply/qnetworkreply.pro

@@ -2,7 +2,7 @@ TEMPLATE = app
 TARGET = tst_bench_qnetworkreply
 
 QT -= gui
-QT += network testlib
+QT += core-private network network-private testlib
 
 CONFIG += release
 

tests/benchmarks/network/access/qnetworkreply/tst_qnetworkreply.cpp

@@ -50,6 +50,9 @@
 #include <QtNetwork/qtcpserver.h>
 #include "../../../../auto/network-settings.h"
 
+#ifdef QT_BUILD_INTERNAL
+#include <QtNetwork/private/qhostinfo_p.h>
+#endif
 
 Q_DECLARE_METATYPE(QSharedPointer<char>)
 
@@ -460,6 +463,7 @@ private slots:
 #ifndef QT_NO_SSL
     void echoPerformance_data();
     void echoPerformance();
+    void preConnectEncrypted();
 #endif
 
     void downloadPerformance();
@@ -472,9 +476,12 @@ private slots:
     void httpDownloadPerformanceDownloadBuffer();
     void httpsRequestChain();
     void httpsUpload();
+    void preConnect();
 
 private:
     void runHttpsUploadRequest(const QByteArray &data, const QNetworkRequest &request);
+    QPair<QNetworkReply *, qint64> runGetRequest(QNetworkAccessManager *manager,
+                                                 const QNetworkRequest &request);
 };
 
 void tst_qnetworkreply::initTestCase()
@@ -495,6 +502,19 @@ void tst_qnetworkreply::httpLatency()
     }
 }
 
+QPair<QNetworkReply *, qint64> tst_qnetworkreply::runGetRequest(
+        QNetworkAccessManager *manager, const QNetworkRequest &request)
+{
+    QElapsedTimer timer;
+    timer.start();
+    QNetworkReply *reply = manager->get(request);
+    connect(reply, SIGNAL(sslErrors(QList<QSslError>)), reply, SLOT(ignoreSslErrors()));
+    connect(reply, SIGNAL(finished()), &QTestEventLoop::instance(), SLOT(exitLoop()), Qt::QueuedConnection);
+    QTestEventLoop::instance().enterLoop(20);
+    qint64 elapsed = timer.elapsed();
+    return qMakePair(reply, elapsed);
+}
+
 #ifndef QT_NO_SSL
 void tst_qnetworkreply::echoPerformance_data()
 {
@@ -527,6 +547,51 @@ void tst_qnetworkreply::echoPerformance()
         delete reply;
     }
 }
+
+void tst_qnetworkreply::preConnectEncrypted()
+{
+    QString hostName = QLatin1String("www.google.com");
+
+    QNetworkAccessManager manager;
+    QNetworkRequest request(QUrl("https://" + hostName));
+
+    // make sure we have a full request including
+    // DNS lookup, TCP and SSL handshakes
+#ifdef QT_BUILD_INTERNAL
+    qt_qhostinfo_clear_cache();
+#else
+    qWarning("no internal build, could not clear DNS cache. Results may not be representative.");
+#endif
+
+    // first, benchmark a normal request
+    QPair<QNetworkReply *, qint64> normalResult = runGetRequest(&manager, request);
+    QNetworkReply *normalReply = normalResult.first;
+    QVERIFY(!QTestEventLoop::instance().timeout());
+    QVERIFY(normalReply->error() == QNetworkReply::NoError);
+    qint64 normalElapsed = normalResult.second;
+
+    // clear all caches again
+#ifdef QT_BUILD_INTERNAL
+    qt_qhostinfo_clear_cache();
+#else
+    qWarning("no internal build, could not clear DNS cache. Results may not be representative.");
+#endif
+    manager.clearAccessCache();
+
+    // now try to make the connection beforehand
+    manager.connectToHostEncrypted(hostName);
+    QTestEventLoop::instance().enterLoop(2);
+
+    // now make another request and hopefully use the existing connection
+    QPair<QNetworkReply *, qint64> preConnectResult = runGetRequest(&manager, request);
+    QNetworkReply *preConnectReply = normalResult.first;
+    QVERIFY(!QTestEventLoop::instance().timeout());
+    QVERIFY(preConnectReply->error() == QNetworkReply::NoError);
+    qint64 preConnectElapsed = preConnectResult.second;
+    qDebug() << request.url().toString() << "full request:" << normalElapsed
+             << "ms, pre-connect request:" << preConnectElapsed << "ms, difference:"
+             << (normalElapsed - preConnectElapsed) << "ms";
+}
 #endif
 
 void tst_qnetworkreply::downloadPerformance()
@@ -852,6 +917,50 @@ void tst_qnetworkreply::httpsUpload()
     }
 }
 
+void tst_qnetworkreply::preConnect()
+{
+    QString hostName = QLatin1String("www.google.com");
+
+    QNetworkAccessManager manager;
+    QNetworkRequest request(QUrl("http://" + hostName));
+
+    // make sure we have a full request including
+    // DNS lookup and TCP handshake
+#ifdef QT_BUILD_INTERNAL
+    qt_qhostinfo_clear_cache();
+#else
+    qWarning("no internal build, could not clear DNS cache. Results may not be representative.");
+#endif
+
+    // first, benchmark a normal request
+    QPair<QNetworkReply *, qint64> normalResult = runGetRequest(&manager, request);
+    QNetworkReply *normalReply = normalResult.first;
+    QVERIFY(!QTestEventLoop::instance().timeout());
+    QVERIFY(normalReply->error() == QNetworkReply::NoError);
+    qint64 normalElapsed = normalResult.second;
+
+    // clear all caches again
+#ifdef QT_BUILD_INTERNAL
+    qt_qhostinfo_clear_cache();
+#else
+    qWarning("no internal build, could not clear DNS cache. Results may not be representative.");
+#endif
+    manager.clearAccessCache();
+
+    // now try to make the connection beforehand
+    manager.connectToHost(hostName);
+    QTestEventLoop::instance().enterLoop(2);
+
+    // now make another request and hopefully use the existing connection
+    QPair<QNetworkReply *, qint64> preConnectResult = runGetRequest(&manager, request);
+    QNetworkReply *preConnectReply = normalResult.first;
+    QVERIFY(!QTestEventLoop::instance().timeout());
+    QVERIFY(preConnectReply->error() == QNetworkReply::NoError);
+    qint64 preConnectElapsed = preConnectResult.second;
+    qDebug() << request.url().toString() << "full request:" << normalElapsed
+             << "ms, pre-connect request:" << preConnectElapsed << "ms, difference:"
+             << (normalElapsed - preConnectElapsed) << "ms";
+}
 
 QTEST_MAIN(tst_qnetworkreply)