From 97e177e58d195f78ac103b3dd0b8ecedb1e07d4c Mon Sep 17 00:00:00 2001 From: Lars Knoll Date: Fri, 1 Jun 2012 17:51:06 +0200 Subject: [PATCH] Refactor the ICU code for QLocale Clean up the ICU code and make it thread-safe. Add a QIcuData structure to QLocalePrivate, that contains ICU specific data. Link against ICU directly, greatly simplifying the code. Also fix a bug in the locale specific case conversion code that would cause it to fail and fall back to the QString code if the output string was larger than the input. Change-Id: Ie67e5ea14fa204ebc5887d7aaeb1a4f3ecaf8697 Reviewed-by: Thiago Macieira --- src/corelib/tools/qlocale.cpp | 91 +++++-------- src/corelib/tools/qlocale.h | 1 + src/corelib/tools/qlocale_icu.cpp | 205 +++++------------------------- src/corelib/tools/qlocale_p.h | 25 +++- src/corelib/tools/qstring.cpp | 13 +- src/corelib/tools/tools.pri | 2 + 6 files changed, 99 insertions(+), 238 deletions(-) diff --git a/src/corelib/tools/qlocale.cpp b/src/corelib/tools/qlocale.cpp index 67e47e56bb..c809b559dd 100644 --- a/src/corelib/tools/qlocale.cpp +++ b/src/corelib/tools/qlocale.cpp @@ -82,12 +82,6 @@ static QLocaleData *system_data = 0; Q_GLOBAL_STATIC(QLocaleData, globalLocaleData) #endif -#ifdef QT_USE_ICU -extern bool qt_initIcu(const QString &localeName); -extern bool qt_u_strToUpper(const QString &str, QString *out, const QLocale &locale); -extern bool qt_u_strToLower(const QString &str, QString *out, const QLocale &locale); -#endif - /****************************************************************************** ** Helpers for accessing Qt locale database */ @@ -515,12 +509,6 @@ void QLocalePrivate::updateSystemPrivate() res = sys_locale->query(QSystemLocale::PositiveSign, QVariant()); if (!res.isNull()) system_data->m_plus = res.toString().at(0).unicode(); - -#ifdef QT_USE_ICU - if (!default_data) - qt_initIcu(sys_locale->fallbackUiLocale().bcp47Name()); -#endif - } #endif @@ -587,7 +575,7 @@ QDataStream &operator>>(QDataStream &ds, QLocale &l) static const int locale_data_size = sizeof(locale_data)/sizeof(QLocaleData) - 1; -static const QLocaleData *dataPointerHelper(quint16 index) +const QLocaleData *QLocalePrivate::dataPointerForIndex(quint16 index) { #ifndef QT_NO_SYSTEMLOCALE Q_ASSERT(index <= locale_data_size); @@ -614,6 +602,14 @@ static quint16 localeDataIndex(const QLocaleData *p) return index; } +/*! + \internal +*/ +QLocale::QLocale(QLocalePrivate &dd) + : d(&dd) +{} + + /*! Constructs a QLocale object with the specified \a name, which has the format @@ -644,11 +640,8 @@ static quint16 localeDataIndex(const QLocaleData *p) */ QLocale::QLocale(const QString &name) - : d(new QLocalePrivate()) + : d(new QLocalePrivate(localeDataIndex(findLocaleData(name)))) { - d->m_numberOptions = 0; - d->m_index = localeDataIndex(findLocaleData(name)); - d->m_data = dataPointerHelper(d->m_index); } /*! @@ -660,11 +653,8 @@ QLocale::QLocale(const QString &name) */ QLocale::QLocale() - : d(new QLocalePrivate()) + : d(new QLocalePrivate(localeDataIndex(defaultData()), default_number_options)) { - d->m_numberOptions = default_number_options; - d->m_index = localeDataIndex(defaultData()); - d->m_data = dataPointerHelper(d->m_index); } /*! @@ -687,19 +677,19 @@ QLocale::QLocale() */ QLocale::QLocale(Language language, Country country) - : d(new QLocalePrivate()) { const QLocaleData *data = QLocaleData::findLocaleData(language, QLocale::AnyScript, country); + int index; + int numberOptions = 0; // If not found, should default to system if (data->m_language_id == QLocale::C && language != QLocale::C) { - d->m_numberOptions = default_number_options; - d->m_index = localeDataIndex(defaultData()); + numberOptions = default_number_options; + index = localeDataIndex(defaultData()); } else { - d->m_numberOptions = 0; - d->m_index = localeDataIndex(data); + index = localeDataIndex(data); } - d->m_data = dataPointerHelper(d->m_index); + d = new QLocalePrivate(index, numberOptions); } \ /*! @@ -727,19 +717,19 @@ QLocale::QLocale(Language language, Country country) */ QLocale::QLocale(Language language, Script script, Country country) - : d(new QLocalePrivate()) { const QLocaleData *data = QLocaleData::findLocaleData(language, script, country); + int index; + int numberOptions = 0; // If not found, should default to system if (data->m_language_id == QLocale::C && language != QLocale::C) { - d->m_numberOptions = default_number_options; - d->m_index = localeDataIndex(defaultData()); + numberOptions = default_number_options; + index = localeDataIndex(defaultData()); } else { - d->m_numberOptions = 0; - d->m_index = localeDataIndex(data); + index = localeDataIndex(data); } - d->m_data = dataPointerHelper(d->m_index); + d = new QLocalePrivate(index, numberOptions); } /*! @@ -897,10 +887,6 @@ void QLocale::setDefault(const QLocale &locale) { default_data = locale.d->m_data; default_number_options = locale.numberOptions(); - -#ifdef QT_USE_ICU - qt_initIcu(locale.bcp47Name()); -#endif } /*! @@ -1802,10 +1788,7 @@ QString QLocale::toString(double i, char f, int prec) const QLocale QLocale::system() { - QLocale result(C); - result.d->m_index = localeDataIndex(systemData()); - result.d->m_data = dataPointerHelper(result.d->m_index); - return result; + return QLocale(*new QLocalePrivate(localeDataIndex(systemData()))); } @@ -1834,9 +1817,7 @@ QList QLocale::matchingLocales(QLocale::Language language, data += locale_index[language]; while ( (data != locale_data + locale_data_size) && (language == QLocale::AnyLanguage || data->m_language_id == uint(language))) { - QLocale locale(QLocale::C); - locale.d->m_index = localeDataIndex(data); - locale.d->m_data = dataPointerHelper(locale.d->m_index); + QLocale locale(*new QLocalePrivate(localeDataIndex(data))); result.append(locale); ++data; } @@ -2164,12 +2145,11 @@ Qt::LayoutDirection QLocale::textDirection() const QString QLocale::toUpper(const QString &str) const { #ifdef QT_USE_ICU - { - QString result; - if (qt_u_strToUpper(str, &result, *this)) - return result; - // else fall through and use Qt's toUpper - } + bool ok = true; + QString result = QIcu::toUpper(d->m_localeID, str, &ok); + if (ok) + return result; + // else fall through and use Qt's toUpper #endif return str.toUpper(); } @@ -2182,12 +2162,11 @@ QString QLocale::toUpper(const QString &str) const QString QLocale::toLower(const QString &str) const { #ifdef QT_USE_ICU - { - QString result; - if (qt_u_strToLower(str, &result, *this)) - return result; - // else fall through and use Qt's toUpper - } + bool ok = true; + QString result = QIcu::toLower(d->m_localeID, str, &ok); + if (ok) + return result; + // else fall through and use Qt's toUpper #endif return str.toLower(); } diff --git a/src/corelib/tools/qlocale.h b/src/corelib/tools/qlocale.h index a014b6b7b5..c901211af4 100644 --- a/src/corelib/tools/qlocale.h +++ b/src/corelib/tools/qlocale.h @@ -703,6 +703,7 @@ public: QString createSeparatedList(const QStringList &strl) const; private: + QLocale(QLocalePrivate &dd); friend class QLocalePrivate; QSharedDataPointer d; }; diff --git a/src/corelib/tools/qlocale_icu.cpp b/src/corelib/tools/qlocale_icu.cpp index 42651aa745..97ce82108b 100644 --- a/src/corelib/tools/qlocale_icu.cpp +++ b/src/corelib/tools/qlocale_icu.cpp @@ -42,199 +42,56 @@ #include "qglobal.h" #include "qlibrary.h" #include "qdebug.h" +#include "qlocale_p.h" +#include "qmutex.h" #include "unicode/uversion.h" #include "unicode/ucol.h" +#include "unicode/uloc.h" +#include "unicode/ustring.h" QT_BEGIN_NAMESPACE -typedef UCollator *(*Ptr_ucol_open)(const char *loc, UErrorCode *status); -typedef void (*Ptr_ucol_close)(UCollator *coll); -typedef UCollationResult (*Ptr_ucol_strcoll)(const UCollator *coll, const UChar *source, int32_t sourceLength, const UChar *target, int32_t targetLength); typedef int32_t (*Ptr_u_strToCase)(UChar *dest, int32_t destCapacity, const UChar *src, int32_t srcLength, const char *locale, UErrorCode *pErrorCode); -static Ptr_ucol_open ptr_ucol_open = 0; -static Ptr_ucol_strcoll ptr_ucol_strcoll = 0; -static Ptr_ucol_close ptr_ucol_close = 0; -static Ptr_u_strToCase ptr_u_strToUpper = 0; -static Ptr_u_strToCase ptr_u_strToLower = 0; - -enum LibLoadStatus -{ - ErrorLoading = -1, - NotLoaded = 0, - Loaded = 1 -}; - -static LibLoadStatus status = NotLoaded; - -static UCollator *icuCollator = 0; - -namespace { -struct Libraries { - QLibrary libicui18n; - QLibrary libicuuc; - ~Libraries() - { - if (icuCollator) { - ptr_ucol_close(icuCollator); - icuCollator = 0; - } - - libicui18n.unload(); - libicuuc.unload(); - } -}; -} -Q_GLOBAL_STATIC(Libraries, icuLibraries) - -static bool loadIcuLibrary(QLibrary &lib, const QString &name) -{ -#ifdef Q_OS_WIN - // QLibrary on Windows does not use the version number, the libraries - // are named "icuin.dll", though. - lib.setFileName(name); -#else - // on Unix, we can use the version number - lib.setFileNameAndVersion(name, QStringLiteral(U_ICU_VERSION_SHORT)); -#endif - - // the ICU libraries appear to allocate global statics and not free them - // set the PreventUnloadHint so that we can unload the QLibrary object and - // delete it, but the libraries themselves remain in memory - lib.setLoadHints(QLibrary::PreventUnloadHint); - return lib.load(); -} - -// this function is NOT THREAD-SAFE! -bool qt_initIcu(const QString &localeString) -{ - if (status == ErrorLoading || !icuLibraries()) - return false; - - if (status == NotLoaded) { - - // resolve libicui18n -#ifdef Q_OS_WIN - // QLibrary on Windows does not use the version number, the libraries - // are named "icuin.dll", though. - // QStringLiteral should work here and will work when MSVC fully supports C++11 - // unfortunately, current versions have do not support proper string concatenation - QString libicui18nName = QLatin1String("icuin" U_ICU_VERSION_SHORT); - QString libicuucName = QLatin1String("icuuc" U_ICU_VERSION_SHORT); -#else - QString libicui18nName = QStringLiteral("icui18n"); - QString libicuucName = QStringLiteral("icuuc"); -#endif - QLibrary &lib = icuLibraries()->libicui18n; - if (!loadIcuLibrary(lib, libicui18nName)) { - qWarning("Unable to load library '%s' version " U_ICU_VERSION_SHORT ": %s", - qPrintable(libicui18nName), - qPrintable(lib.errorString())); - status = ErrorLoading; - return false; - } - - ptr_ucol_open = (Ptr_ucol_open)lib.resolve("ucol_open"); - ptr_ucol_close = (Ptr_ucol_close)lib.resolve("ucol_close"); - ptr_ucol_strcoll = (Ptr_ucol_strcoll)lib.resolve("ucol_strcoll"); - - if (!ptr_ucol_open || !ptr_ucol_close || !ptr_ucol_strcoll) { - // try again with decorated symbol names - ptr_ucol_open = (Ptr_ucol_open)lib.resolve("ucol_open" QT_STRINGIFY(U_ICU_VERSION_SUFFIX)); - ptr_ucol_close = (Ptr_ucol_close)lib.resolve("ucol_close" QT_STRINGIFY(U_ICU_VERSION_SUFFIX)); - ptr_ucol_strcoll = (Ptr_ucol_strcoll)lib.resolve("ucol_strcoll" QT_STRINGIFY(U_ICU_VERSION_SUFFIX)); - } - - if (!ptr_ucol_open || !ptr_ucol_close || !ptr_ucol_strcoll) { - ptr_ucol_open = 0; - ptr_ucol_close = 0; - ptr_ucol_strcoll = 0; - - qWarning("Unable to find symbols in '%s'.", qPrintable(libicui18nName)); - status = ErrorLoading; - return false; - } - - // resolve libicuuc - QLibrary &ucLib = icuLibraries()->libicuuc; - if (!loadIcuLibrary(ucLib, libicuucName)) { - qWarning("Unable to load library '%s' version " U_ICU_VERSION_SHORT ": %s", - qPrintable(libicuucName), - qPrintable(ucLib.errorString())); - status = ErrorLoading; - return false; - } - - ptr_u_strToUpper = (Ptr_u_strToCase)ucLib.resolve("u_strToUpper"); - ptr_u_strToLower = (Ptr_u_strToCase)ucLib.resolve("u_strToLower"); - - if (!ptr_u_strToUpper || !ptr_u_strToLower) { - ptr_u_strToUpper = (Ptr_u_strToCase)ucLib.resolve("u_strToUpper" QT_STRINGIFY(U_ICU_VERSION_SUFFIX)); - ptr_u_strToLower = (Ptr_u_strToCase)ucLib.resolve("u_strToLower" QT_STRINGIFY(U_ICU_VERSION_SUFFIX)); - } - - if (!ptr_u_strToUpper || !ptr_u_strToLower) { - ptr_u_strToUpper = 0; - ptr_u_strToLower = 0; - - qWarning("Unable to find symbols in '%s'", qPrintable(libicuucName)); - status = ErrorLoading; - return false; - } - - // success :) - status = Loaded; - } - - if (icuCollator) { - ptr_ucol_close(icuCollator); - icuCollator = 0; - } - - UErrorCode icuStatus = U_ZERO_ERROR; - icuCollator = ptr_ucol_open(localeString.toLatin1().constData(), &icuStatus); - - if (!icuCollator) { - qWarning("Unable to open locale %s in ICU, error code %d", qPrintable(localeString), icuStatus); - return false; - } - - return true; -} - -bool qt_ucol_strcoll(const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result) +bool QIcu::strcoll(const QByteArray &localeID, + const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result) { Q_ASSERT(result); Q_ASSERT(source); Q_ASSERT(target); - if (!icuCollator) + UErrorCode icuStatus = U_ZERO_ERROR; + UCollator *collator = ucol_open(localeID, &icuStatus); + + if (U_FAILURE((icuStatus))) return false; - *result = ptr_ucol_strcoll(icuCollator, reinterpret_cast(source), int32_t(sourceLength), - reinterpret_cast(target), int32_t(targetLength)); + *result = ucol_strcoll(collator, + reinterpret_cast(source), int32_t(sourceLength), + reinterpret_cast(target), int32_t(targetLength)); + + ucol_close(collator); return true; } // caseFunc can either be u_strToUpper or u_strToLower -static bool qt_u_strToCase(const QString &str, QString *out, const QLocale &locale, Ptr_u_strToCase caseFunc) +static bool qt_u_strToCase(const QString &str, QString *out, const char *localeID, Ptr_u_strToCase caseFunc) { Q_ASSERT(out); - if (!icuCollator) - return false; - - QString result(str.size(), Qt::Uninitialized); + int32_t size = str.size(); + size += size >> 2; // add 25% for possible expansions + QString result(size, Qt::Uninitialized); UErrorCode status = U_ZERO_ERROR; - int32_t size = caseFunc(reinterpret_cast(result.data()), result.size(), + size = caseFunc(reinterpret_cast(result.data()), result.size(), reinterpret_cast(str.constData()), str.size(), - locale.bcp47Name().toLatin1().constData(), &status); + localeID, &status); - if (U_FAILURE(status)) + if (U_FAILURE(status) && status != U_BUFFER_OVERFLOW_ERROR) return false; if (size < result.size()) { @@ -246,7 +103,7 @@ static bool qt_u_strToCase(const QString &str, QString *out, const QLocale &loca status = U_ZERO_ERROR; size = caseFunc(reinterpret_cast(result.data()), result.size(), reinterpret_cast(str.constData()), str.size(), - locale.bcp47Name().toLatin1().constData(), &status); + localeID, &status); if (U_FAILURE(status)) return false; @@ -260,14 +117,22 @@ static bool qt_u_strToCase(const QString &str, QString *out, const QLocale &loca return true; } -bool qt_u_strToUpper(const QString &str, QString *out, const QLocale &locale) +QString QIcu::toUpper(const QByteArray &localeID, const QString &str, bool *ok) { - return qt_u_strToCase(str, out, locale, ptr_u_strToUpper); + QString out; + bool err = qt_u_strToCase(str, &out, localeID, u_strToUpper); + if (ok) + *ok = err; + return out; } -bool qt_u_strToLower(const QString &str, QString *out, const QLocale &locale) +QString QIcu::toLower(const QByteArray &localeID, const QString &str, bool *ok) { - return qt_u_strToCase(str, out, locale, ptr_u_strToLower); + QString out; + bool err = qt_u_strToCase(str, &out, localeID, u_strToLower); + if (ok) + *ok = err; + return out; } QT_END_NAMESPACE diff --git a/src/corelib/tools/qlocale_p.h b/src/corelib/tools/qlocale_p.h index c2c1488966..1902e4e5af 100644 --- a/src/corelib/tools/qlocale_p.h +++ b/src/corelib/tools/qlocale_p.h @@ -126,6 +126,16 @@ private: }; #endif +#ifdef QT_USE_ICU +namespace QIcu { + QString toUpper(const QByteArray &localeId, const QString &str, bool *ok); + QString toLower(const QByteArray &localeId, const QString &str, bool *ok); + + bool strcoll(const QByteArray &localeID, const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result); +} +#endif + + struct QLocaleData { public: @@ -178,7 +188,17 @@ public: class Q_CORE_EXPORT QLocalePrivate : public QSharedData { public: - QLocalePrivate() : m_index(0), m_numberOptions(0), m_data(0) {} + QLocalePrivate(int index, int numberOptions = 0) + : m_index(index), m_numberOptions(numberOptions) + { + m_data = dataPointerForIndex(index); + m_localeID = bcp47Name().toLatin1(); + m_localeID.replace('-','_'); + } + + ~QLocalePrivate() + { + } QChar decimal() const { return QChar(m_data->m_decimal); } QChar group() const { return QChar(m_data->m_group); } @@ -203,6 +223,7 @@ public: static QLocale::Country codeToCountry(const QString &code); static void getLangAndCountry(const QString &name, QLocale::Language &lang, QLocale::Script &script, QLocale::Country &cntry); + static const QLocaleData *dataPointerForIndex(quint16 index); QLocale::MeasurementSystem measurementSystem() const; @@ -285,11 +306,11 @@ public: QString dateTimeToString(const QString &format, const QDate *date, const QTime *time, const QLocale *q) const; -private: friend class QLocale; quint16 m_index; quint16 m_numberOptions; const QLocaleData *m_data; + QByteArray m_localeID; }; inline char QLocalePrivate::digitToCLocale(QChar in) const diff --git a/src/corelib/tools/qstring.cpp b/src/corelib/tools/qstring.cpp index a2d91c561c..4f8d0e5614 100644 --- a/src/corelib/tools/qstring.cpp +++ b/src/corelib/tools/qstring.cpp @@ -100,12 +100,6 @@ QT_BEGIN_NAMESPACE -#ifdef QT_USE_ICU -// qlocale_icu.cpp -extern bool qt_ucol_strcoll(const QChar *source, int sourceLength, const QChar *target, int targetLength, int *result); -#endif - - // internal int qFindString(const QChar *haystack, int haystackLen, int from, const QChar *needle, int needleLen, Qt::CaseSensitivity cs); @@ -5011,12 +5005,11 @@ int QString::localeAwareCompare_helper(const QChar *data1, int length1, return result; #elif defined(Q_OS_UNIX) # if defined(QT_USE_ICU) + QLocale locale; int res; - if (qt_ucol_strcoll(data1, length1, data2, length2, &res)) { - if (res == 0) - res = ucstrcmp(data1, length1, data2, length2); + if (QIcu::strcoll(locale.d.constData()->m_localeID, data1, length1, data2, length2, &res)) return res; - } // else fall through + // else fall through # endif // declared in int delta = strcoll(toLocal8Bit_helper(data1, length1).constData(), toLocal8Bit_helper(data2, length2).constData()); diff --git a/src/corelib/tools/tools.pri b/src/corelib/tools/tools.pri index 386db9665b..9db459ab68 100644 --- a/src/corelib/tools/tools.pri +++ b/src/corelib/tools/tools.pri @@ -112,6 +112,8 @@ else:include($$PWD/../../3rdparty/zlib_dependency.pri) contains(QT_CONFIG,icu) { SOURCES += tools/qlocale_icu.cpp DEFINES += QT_USE_ICU + win32:LIBS += -licuin -licuuc + else:LIBS += -licui18n -licuuc } pcre {