Add QUrl::topLevelDomain() and move TLD table from QtNetwork to QtCore

Move Qt's copy of the Mozilla public suffix list from QtNetwork
to QtCore and use it to expose a new API function QUrl::topLevelDomain().
This function returns the section of the url that is a registrar-controlled
top level domain.

QtCore now exports a couple of functions to the other Qt modules: qTopLevelDomain,
a helper function for QUrl::topLevelDomain(); and qIsEffectiveTLD(), a helper
function for QNetworkCookeieJar.

The motivation for this new API is to allow QtWebKit implement a Third-Party
Cookie blocking policy. For this QtWebKit needs to know the element of the url
that is the registry-controlled TLD. Without this knowledge it would end up
blocking third-party cookies per host rather than per registry-controlled domain.

See also https://bugs.webkit.org/show_bug.cgi?id=45455

Merge-request: 1205
Task-number: QTBUG-13601
Reviewed-by: Peter Hartmann <peter.hartmann@nokia.com>
(cherry picked from commit 154402f56dcf8303a6ce601a52215226af8d31ba)

src/corelib/io/io.pri

@@ -24,6 +24,8 @@ HEADERS +=  \
         io/qresource_p.h \
         io/qresource_iterator_p.h \
         io/qurl.h \
+        io/qurltlds_p.h \
+        io/qtldurl_p.h \
         io/qsettings.h \
         io/qsettings_p.h \
         io/qfsfileengine.h \
@@ -41,6 +43,7 @@ SOURCES += \
         io/qbuffer.cpp \
         io/qdatastream.cpp \
         io/qdataurl.cpp \
+        io/qtldurl.cpp \
         io/qdebug.cpp \
         io/qdir.cpp \
         io/qdiriterator.cpp \

src/corelib/io/qtldurl.cpp

@@ -0,0 +1,117 @@
+/****************************************************************************
+**
+** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qplatformdefs.h"
+#include "qurl.h"
+#include "private/qurltlds_p.h"
+#include "private/qtldurl_p.h"
+#include "QtCore/qstringlist.h"
+
+QT_BEGIN_NAMESPACE
+
+static bool containsTLDEntry(const QString &entry)
+{
+    int index = qHash(entry) % tldCount;
+    int currentDomainIndex = tldIndices[index];
+    while (currentDomainIndex < tldIndices[index+1]) {
+        QString currentEntry = QString::fromUtf8(tldData + currentDomainIndex);
+        if (currentEntry == entry)
+            return true;
+        currentDomainIndex += qstrlen(tldData + currentDomainIndex) + 1; // +1 for the ending \0
+    }
+    return false;
+}
+
+/*!
+    \internal
+
+    Return the top-level-domain per Qt's copy of the Mozilla public suffix list of
+    \a domain.
+*/
+
+Q_CORE_EXPORT QString qTopLevelDomain(const QString &domain)
+{
+    QStringList sections = domain.toLower().split(QLatin1Char('.'), QString::SkipEmptyParts);
+    if (sections.isEmpty())
+        return QString();
+
+    QString level, tld;
+    for (int j = sections.count() - 1; j >= 0; --j) {
+        level.prepend(QLatin1Char('.') + sections.at(j));
+        if (qIsEffectiveTLD(level.right(level.size() - 1)))
+            tld = level;
+    }
+    return tld;
+}
+
+/*!
+    \internal
+
+    Return true if \a domain is a top-level-domain per Qt's copy of the Mozilla public suffix list.
+*/
+
+Q_CORE_EXPORT bool qIsEffectiveTLD(const QString &domain)
+{
+    // for domain 'foo.bar.com':
+    // 1. return if TLD table contains 'foo.bar.com'
+    if (containsTLDEntry(domain))
+        return true;
+
+    if (domain.contains(QLatin1Char('.'))) {
+        int count = domain.size() - domain.indexOf(QLatin1Char('.'));
+        QString wildCardDomain;
+        wildCardDomain.reserve(count + 1);
+        wildCardDomain.append(QLatin1Char('*'));
+        wildCardDomain.append(domain.right(count));
+        // 2. if table contains '*.bar.com',
+        // test if table contains '!foo.bar.com'
+        if (containsTLDEntry(wildCardDomain)) {
+            QString exceptionDomain;
+            exceptionDomain.reserve(domain.size() + 1);
+            exceptionDomain.append(QLatin1Char('!'));
+            exceptionDomain.append(domain);
+            return (! containsTLDEntry(exceptionDomain));
+        }
+    }
+    return false;
+}
+
+QT_END_NAMESPACE

src/corelib/io/qtldurl_p.h

@@ -0,0 +1,66 @@
+/****************************************************************************
+**
+** Copyright (C) 2011 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the QtCore module of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef QTLDURL_P_H
+#define QTLDURL_P_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists for the convenience
+// of qDecodeDataUrl. This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+#include "QtCore/qurl.h"
+#include "QtCore/qstring.h"
+
+QT_BEGIN_NAMESPACE
+
+Q_CORE_EXPORT QString qTopLevelDomain(const QString &domain);
+Q_CORE_EXPORT bool qIsEffectiveTLD(const QString &domain);
+
+QT_END_NAMESPACE
+
+#endif // QDATAURL_P_H

src/corelib/io/qurl.cpp

@@ -192,7 +192,9 @@
 #if defined QT3_SUPPORT
 #include "qfileinfo.h"
 #endif
-
+#ifndef QT_BOOTSTRAPPED
+#include "qtldurl_p.h"
+#endif
 #if defined(Q_OS_WINCE_WM)
 #pragma optimize("g", off)
 #endif
@@ -5592,6 +5594,21 @@ bool QUrl::hasFragment() const
     return d->hasFragment;
 }
 
+/*!
+    \since 4.8
+
+    Returns the TLD (Top-Level Domain) of the URL, (e.g. .co.uk, .net).
+    Note that the return value is prefixed with a '.' unless the
+    URL does not contain a valid TLD, in which case the function returns
+    an empty string.
+*/
+#ifndef QT_BOOTSTRAPPED
+QString QUrl::topLevelDomain() const
+{
+    return qTopLevelDomain(host());
+}
+#endif
+
 /*!
     Returns the result of the merge of this URL with \a relative. This
     URL is used as a base to convert \a relative to an absolute URL.

src/corelib/io/qurl.h

@@ -181,6 +181,9 @@ public:
     void setEncodedFragment(const QByteArray &fragment);
     QByteArray encodedFragment() const;
     bool hasFragment() const;
+#ifndef QT_BOOTSTRAPPED
+    QString topLevelDomain() const;
+#endif
 
     QUrl resolved(const QUrl &relative) const;
 

src/corelib/io/qurltlds_p.h

@@ -38,15 +38,15 @@
 // the terms of any one of the MPL, the GPL or the LGPL.
 //
 
-#ifndef QNETWORKCOOKIEJARTLD_P_H
+#ifndef QURLTLD_P_H
-#define QNETWORKCOOKIEJARTLD_P_H
+#define QURLTLD_P_H
 
 //
 //  W A R N I N G
 //  -------------
 //
 // This file is not part of the Qt API.  It exists for the convenience
-// of the Network Access framework.  This header file may change from
+// of the Network Access and Core framework.  This header file may change from
 // version to version without notice, or even be removed.
 //
 // We mean it.
@@ -57,7 +57,7 @@ QT_BEGIN_NAMESPACE
 // note to maintainer:
 // this file should be updated before each release ->
 // for instructions see the program at
-// util/network/cookiejar-generateTLDs
+// util/corelib/qurl-generateTLDs
 
 static const quint16 tldCount = 3949;
 static const quint16 tldIndices[] = {
@@ -6478,4 +6478,4 @@ static const char tldData[] = {
 
 QT_END_NAMESPACE
 
-#endif // QNETWORKCOOKIEJARTLD_P_H
+#endif // QURLTLD_P_H

src/network/access/access.pri

@@ -21,7 +21,6 @@ HEADERS += \
     access/qnetworkcookie_p.h \
     access/qnetworkcookiejar.h \
     access/qnetworkcookiejar_p.h \
-    access/qnetworkcookiejartlds_p.h \
     access/qnetworkrequest.h \
     access/qnetworkrequest_p.h \
     access/qnetworkreply.h \

src/network/access/qnetworkcookiejar.cpp

@@ -40,12 +40,12 @@
 ****************************************************************************/
 
 #include "qnetworkcookiejar.h"
-#include "qnetworkcookiejartlds_p.h"
 #include "qnetworkcookiejar_p.h"
 
 #include "QtNetwork/qnetworkcookie.h"
 #include "QtCore/qurl.h"
 #include "QtCore/qdatetime.h"
+#include "private/qtldurl_p.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -216,7 +216,7 @@ bool QNetworkCookieJar::setCookiesFromUrl(const QList<QNetworkCookie> &cookieLis
             // the check for effective TLDs makes the "embedded dot" rule from RFC 2109 section 4.3.2
             // redundant; the "leading dot" rule has been relaxed anyway, see above
             // we remove the leading dot for this check
-            if (QNetworkCookieJarPrivate::isEffectiveTLD(domain.remove(0, 1)))
+            if (qIsEffectiveTLD(domain.remove(0, 1)))
                 continue; // not accepted
         }
 
@@ -304,43 +304,4 @@ QList<QNetworkCookie> QNetworkCookieJar::cookiesForUrl(const QUrl &url) const
     return result;
 }
 
-bool QNetworkCookieJarPrivate::isEffectiveTLD(const QString &domain)
-{
-    // for domain 'foo.bar.com':
-    // 1. return if TLD table contains 'foo.bar.com'
-    if (containsTLDEntry(domain))
-        return true;
-
-    if (domain.contains(QLatin1Char('.'))) {
-        int count = domain.size() - domain.indexOf(QLatin1Char('.'));
-        QString wildCardDomain;
-        wildCardDomain.reserve(count + 1);
-        wildCardDomain.append(QLatin1Char('*'));
-        wildCardDomain.append(domain.right(count));
-        // 2. if table contains '*.bar.com',
-        // test if table contains '!foo.bar.com'
-        if (containsTLDEntry(wildCardDomain)) {
-            QString exceptionDomain;
-            exceptionDomain.reserve(domain.size() + 1);
-            exceptionDomain.append(QLatin1Char('!'));
-            exceptionDomain.append(domain);
-            return (! containsTLDEntry(exceptionDomain));
-        }
-    }
-    return false;
-}
-
-bool QNetworkCookieJarPrivate::containsTLDEntry(const QString &entry)
-{
-    int index = qHash(entry) % tldCount;
-    int currentDomainIndex = tldIndices[index];
-    while (currentDomainIndex < tldIndices[index+1]) {
-        QString currentEntry = QString::fromUtf8(tldData + currentDomainIndex);
-        if (currentEntry == entry)
-            return true;
-        currentDomainIndex += qstrlen(tldData + currentDomainIndex) + 1; // +1 for the ending \0
-    }
-    return false;
-}
-
 QT_END_NAMESPACE

src/network/access/qnetworkcookiejar_p.h

@@ -63,9 +63,6 @@ class QNetworkCookieJarPrivate: public QObjectPrivate
 public:
     QList<QNetworkCookie> allCookies;
 
-    static bool Q_AUTOTEST_EXPORT isEffectiveTLD(const QString &domain);
-    static bool containsTLDEntry(const QString &entry);
-
     Q_DECLARE_PUBLIC(QNetworkCookieJar)
 };
 

tests/auto/qnetworkcookiejar/tst_qnetworkcookiejar.cpp

@@ -42,7 +42,7 @@
 
 #include <QtTest/QtTest>
 #include <QtNetwork/QNetworkCookieJar>
-#include "private/qnetworkcookiejar_p.h"
+#include "private/qtldurl_p.h"
 
 class tst_QNetworkCookieJar: public QObject
 {
@@ -438,7 +438,7 @@ void tst_QNetworkCookieJar::effectiveTLDs()
 #endif
     QFETCH(QString, domain);
     QFETCH(bool, isTLD);
-    QCOMPARE(QNetworkCookieJarPrivate::isEffectiveTLD(domain), isTLD);
+    QCOMPARE(qIsEffectiveTLD(domain), isTLD);
 }
 
 QTEST_MAIN(tst_QNetworkCookieJar)

tests/auto/qurl/tst_qurl.cpp

@@ -49,6 +49,7 @@
 #include <qurl.h>
 #include <qtextcodec.h>
 #include <qmap.h>
+#include "private/qtldurl_p.h"
 
 // For testsuites
 #define IDNA_ACE_PREFIX "xn--"
@@ -88,6 +89,8 @@ public slots:
     void init();
     void cleanup();
 private slots:
+    void effectiveTLDs_data();
+    void effectiveTLDs();
     void getSetCheck();
     void constructing();
     void assignment();
@@ -4005,5 +4008,28 @@ void tst_QUrl::taskQTBUG_8701()
     QCOMPARE(foo_uni_bar, QUrl(foo_uni_bar, QUrl::StrictMode).toString());
 }
 
+void tst_QUrl::effectiveTLDs_data()
+{
+    QTest::addColumn<QUrl>("domain");
+    QTest::addColumn<QString>("TLD");
+
+    QTest::newRow("yes0") << QUrl::fromEncoded("http://test.co.uk") << ".co.uk";
+    QTest::newRow("yes1") << QUrl::fromEncoded("http://test.com") << ".com";
+    QTest::newRow("yes2") << QUrl::fromEncoded("http://www.test.de") << ".de";
+    QTest::newRow("yes3") << QUrl::fromEncoded("http://test.ulm.museum") << ".ulm.museum";
+    QTest::newRow("yes4") << QUrl::fromEncoded("http://www.com.krodsherad.no") << ".krodsherad.no";
+    QTest::newRow("yes5") << QUrl::fromEncoded("http://www.co.uk.1.bg") << ".1.bg";
+    QTest::newRow("yes6") << QUrl::fromEncoded("http://www.com.com.cn") << ".com.cn";
+    QTest::newRow("yes7") << QUrl::fromEncoded("http://www.test.org.ws") << ".org.ws";
+    QTest::newRow("yes9") << QUrl::fromEncoded("http://www.com.co.uk.wallonie.museum") << ".wallonie.museum";
+}
+
+void tst_QUrl::effectiveTLDs()
+{
+    QFETCH(QUrl, domain);
+    QFETCH(QString, TLD);
+    QCOMPARE(domain.topLevelDomain(), TLD);
+}
+
 QTEST_MAIN(tst_QUrl)
 #include "tst_qurl.moc"

util/corelib/qurl-generateTLDs/main.cpp

@@ -77,7 +77,7 @@ int main(int argc, char **argv) {
         printf("       wget http://mxr.mozilla.org/mozilla-central/source/netwerk/dns/effective_tld_names.dat?raw=1 -O effective_tld_names.dat\n");
         printf("       grep '^[^\\/\\/]' effective_tld_names.dat > effective_tld_names.dat.trimmed\n");
         printf("       %s effective_tld_names.dat.trimmed effective_tld_names.dat.qt\n\n", argv[0]);
-        printf("Now copy the data from effective_tld_names.dat.qt to the file src/network/access/qnetworkcookiejartlds_p.h in your Qt repo\n\n");
+        printf("Now copy the data from effective_tld_names.dat.qt to the file src/corelib/io/qurltlds_p.h in your Qt repo\n\n");
         exit(1);
     }
     QFile file(argv[1]);
@@ -156,6 +156,6 @@ int main(int argc, char **argv) {
     outFile.write(outDataBufferBA);
     outFile.write("};\n");
     outFile.close();
-    printf("data generated to %s . Now copy the data from this file to src/network/access/qnetworkcookiejartlds_p.h in your Qt repo\n", argv[2]);
+    printf("data generated to %s . Now copy the data from this file to src/corelib/io/qurltlds_p.h in your Qt repo\n", argv[2]);
     exit(0);
 }