From b79e73476771c068098270ebc26fb1e015f0e149 Mon Sep 17 00:00:00 2001 From: Eskil Abrahamsen Blomfeldt Date: Thu, 30 Jan 2014 14:25:42 +0100 Subject: [PATCH] Android: Support pregenerated cache in assets file engine This provides a way for androiddeployqt to pregenerate the entry list cache for the assets file engine, greatly improving performance the first time a directory is read. If the file is not present, the cache will operate as before. Some numbers from testing on Samsung Galaxy 2, doing QDir::entryList() on a directory inside the assets folder: 10 files -------- Before: 280 ms for first read, 5 ms for subsequent reads After: 2 ms for reading pregenerated cache 5 ms for first read 5 ms for subsequent reads 2000 files ---------- Before: 1000 ms for first read, 150 ms for subsequent reads After: 5 ms for reading pregenerated cache 150 ms for first read 150 ms for subsequent reads 4000 files ---------- Before: 3000 ms for first read 300 ms for subsequent reads After: 8 ms for reading pregenerated cache 300 ms for first read 300 ms for subsequent reads [ChangeLog][Android] Speed up first time directory listing in assets by using pregenerated entry list. Task-number: QTBUG-33704 Change-Id: I3973a1d823b8b38e88a2cc7843326cbe885f8bc2 Reviewed-by: Christian Stromme --- .../qandroidassetsfileenginehandler.cpp | 128 ++++++++++++++---- .../android/qandroidassetsfileenginehandler.h | 3 + 2 files changed, 107 insertions(+), 24 deletions(-) diff --git a/src/plugins/platforms/android/qandroidassetsfileenginehandler.cpp b/src/plugins/platforms/android/qandroidassetsfileenginehandler.cpp index 5f77d1645a..b112e265a5 100644 --- a/src/plugins/platforms/android/qandroidassetsfileenginehandler.cpp +++ b/src/plugins/platforms/android/qandroidassetsfileenginehandler.cpp @@ -51,10 +51,12 @@ struct AndroidAssetDir { AndroidAssetDir(AAssetDir* ad) { - const char *fileName; - while ((fileName = AAssetDir_getNextFileName(ad))) - m_items.push_back(QString::fromUtf8(fileName)); - AAssetDir_close(ad); + if (ad) { + const char *fileName; + while ((fileName = AAssetDir_getNextFileName(ad))) + m_items.push_back(QString::fromUtf8(fileName)); + AAssetDir_close(ad); + } } FilesList m_items; }; @@ -82,7 +84,10 @@ public: { if (m_index < 0 || m_index >= m_items.size()) return QString(); - return m_items[m_index]; + QString fileName = m_items[m_index]; + if (fileName.endsWith(QLatin1Char('/'))) + fileName.chop(1); + return fileName; } virtual QString currentFilePath() const @@ -254,33 +259,106 @@ private: }; -AndroidAssetsFileEngineHandler::AndroidAssetsFileEngineHandler():m_assetsCache(std::max(5, qgetenv("QT_ANDROID_MAX_ASSETS_CACHE_SIZE").toInt())) +AndroidAssetsFileEngineHandler::AndroidAssetsFileEngineHandler() + : m_assetsCache(std::max(5, qgetenv("QT_ANDROID_MAX_ASSETS_CACHE_SIZE").toInt())) + , m_hasPrepopulatedCache(false) { m_assetManager = QtAndroid::assetManager(); + prepopulateCache(); } AndroidAssetsFileEngineHandler::~AndroidAssetsFileEngineHandler() { } +void AndroidAssetsFileEngineHandler::prepopulateCache() +{ + QMutexLocker locker(&m_assetsCacheMutext); + Q_ASSERT(m_assetsCache.isEmpty()); + + // Failsafe: Don't read cache files that are larger than 1MB + static qint64 maxPrepopulatedCacheSize = qMax(1024LL * 1024LL, + qgetenv("QT_ANDROID_MAX_PREPOPULATED_ASSETS_CACHE_SIZE").toLongLong()); + + const char *fileName = "--Added-by-androiddeployqt--/qt_cache_pregenerated_file_list"; + AAsset *asset = AAssetManager_open(m_assetManager, fileName, AASSET_MODE_BUFFER); + if (asset) { + m_hasPrepopulatedCache = true; + AndroidAbstractFileEngine fileEngine(asset, QString::fromLatin1(fileName)); + if (fileEngine.open(QIODevice::ReadOnly)) { + qint64 size = fileEngine.size(); + + if (size <= maxPrepopulatedCacheSize) { + QByteArray bytes(size, Qt::Uninitialized); + qint64 read = fileEngine.read(bytes.data(), size); + if (read != size) { + qWarning("Failed to read prepopulated cache"); + return; + } + + QDataStream stream(&bytes, QIODevice::ReadOnly); + stream.setVersion(QDataStream::Qt_5_3); + if (stream.status() != QDataStream::Ok) { + qWarning("Failed to read prepopulated cache"); + return; + } + + while (!stream.atEnd()) { + QString directoryName; + stream >> directoryName; + + int fileCount; + stream >> fileCount; + + QVector fileList; + fileList.reserve(fileCount); + while (fileCount--) { + QString fileName; + stream >> fileName; + fileList.append(fileName); + } + + QSharedPointer *aad = new QSharedPointer(new AndroidAssetDir(0)); + (*aad)->m_items = fileList; + + // Cost = 0, because we should always cache everything if there's a prepopulated cache + QByteArray key = directoryName != QLatin1String("/") + ? QByteArray("assets:/") + directoryName.toUtf8() + : QByteArray("assets:"); + + bool ok = m_assetsCache.insert(key, aad, 0); + if (!ok) + qWarning("Failed to insert in cache: %s", qPrintable(directoryName)); + } + } else { + qWarning("Prepopulated cache is too large to read.\n" + "Use environment variable QT_ANDROID_MAX_PREPOPULATED_ASSETS_CACHE_SIZE to adjust size."); + } + } + } +} + QAbstractFileEngine * AndroidAssetsFileEngineHandler::create(const QString &fileName) const { if (fileName.isEmpty()) return 0; - if (!fileName.startsWith(QLatin1String("assets:/"))) + static QLatin1String assetsPrefix("assets:"); + if (!fileName.startsWith(assetsPrefix)) return 0; - int prefixSize=8; + static int prefixSize = assetsPrefix.size() + 1; QByteArray path; if (!fileName.endsWith(QLatin1Char('/'))) { path = fileName.toUtf8(); - AAsset *asset = AAssetManager_open(m_assetManager, - path.constData() + prefixSize, - AASSET_MODE_BUFFER); - if (asset) - return new AndroidAbstractFileEngine(asset, fileName); + if (path.size() > prefixSize) { + AAsset *asset = AAssetManager_open(m_assetManager, + path.constData() + prefixSize, + AASSET_MODE_BUFFER); + if (asset) + return new AndroidAbstractFileEngine(asset, fileName); + } } if (!path.size()) @@ -290,17 +368,19 @@ QAbstractFileEngine * AndroidAssetsFileEngineHandler::create(const QString &file QSharedPointer *aad = m_assetsCache.object(path); m_assetsCacheMutext.unlock(); if (!aad) { - AAssetDir *assetDir = AAssetManager_openDir(m_assetManager, path.constData() + prefixSize); - if (assetDir) { - if (AAssetDir_getNextFileName(assetDir)) { - AAssetDir_rewind(assetDir); - aad = new QSharedPointer(new AndroidAssetDir(assetDir)); - m_assetsCacheMutext.lock(); - m_assetsCache.insert(path, aad); - m_assetsCacheMutext.unlock(); - return new AndroidAbstractFileEngine(*aad, fileName); - } else { - AAssetDir_close(assetDir); + if (!m_hasPrepopulatedCache && path.size() > prefixSize) { + AAssetDir *assetDir = AAssetManager_openDir(m_assetManager, path.constData() + prefixSize); + if (assetDir) { + if (AAssetDir_getNextFileName(assetDir)) { + AAssetDir_rewind(assetDir); + aad = new QSharedPointer(new AndroidAssetDir(assetDir)); + m_assetsCacheMutext.lock(); + m_assetsCache.insert(path, aad); + m_assetsCacheMutext.unlock(); + return new AndroidAbstractFileEngine(*aad, fileName); + } else { + AAssetDir_close(assetDir); + } } } } else { diff --git a/src/plugins/platforms/android/qandroidassetsfileenginehandler.h b/src/plugins/platforms/android/qandroidassetsfileenginehandler.h index 7bd560886c..d56367d4d8 100644 --- a/src/plugins/platforms/android/qandroidassetsfileenginehandler.h +++ b/src/plugins/platforms/android/qandroidassetsfileenginehandler.h @@ -58,9 +58,12 @@ public: QAbstractFileEngine *create(const QString &fileName) const; private: + void prepopulateCache(); + AAssetManager *m_assetManager; mutable QCache> m_assetsCache; mutable QMutex m_assetsCacheMutext; + bool m_hasPrepopulatedCache; }; #endif // QANDROIDASSETSFILEENGINEHANDLER_H