Say hello to QtVFS for SQLite3

This patch allows to open databases using QFile. This way it
can open databases from RW locations as android shared storage
or even from RO resources e.g. qrc or android assets.

[ChangeLog][QtSql][SQLite3 driver] QtVFS for SQLite3 allows to open
databases using QFile. This way it can open databases from RW
locations such as android shared storage, or even from read-only
resources e.g. qrc or android assets.

Fixes: QTBUG-107120
Change-Id: I889ad44de966c96105fe1954ee4eda175dd5a886
Reviewed-by: Christian Ehrlicher <ch.ehrlicher@gmx.de>

src/plugins/sqldrivers/sqlite/CMakeLists.txt

@@ -10,6 +10,7 @@ qt_internal_add_plugin(QSQLiteDriverPlugin
     PLUGIN_TYPE sqldrivers
     SOURCES
         qsql_sqlite.cpp qsql_sqlite_p.h
+        qsql_sqlite_vfs.cpp qsql_sqlite_vfs_p.h
         smain.cpp
     DEFINES
         QT_NO_CAST_FROM_ASCII

src/plugins/sqldrivers/sqlite/qsql_sqlite.cpp

@@ -691,6 +691,7 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c
     bool openReadOnlyOption = false;
     bool openUriOption = false;
     bool useExtendedResultCodes = true;
+    bool useQtVfs = false;
 #if QT_CONFIG(regularexpression)
     static const auto regexpConnectOption = "QSQLITE_ENABLE_REGEXP"_L1;
     bool defineRegexp = false;
@@ -708,6 +709,8 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c
                 if (ok)
                     timeOut = nt;
             }
+        } else if (option == "QSQLITE_USE_QT_VFS"_L1) {
+            useQtVfs = true;
         } else if (option == "QSQLITE_OPEN_READONLY"_L1) {
             openReadOnlyOption = true;
         } else if (option == "QSQLITE_OPEN_URI"_L1) {
@@ -742,7 +745,7 @@ bool QSQLiteDriver::open(const QString & db, const QString &, const QString &, c
 
     openMode |= SQLITE_OPEN_NOMUTEX;
 
-    const int res = sqlite3_open_v2(db.toUtf8().constData(), &d->access, openMode, nullptr);
+    const int res = sqlite3_open_v2(db.toUtf8().constData(), &d->access, openMode, useQtVfs ? "QtVFS" : nullptr);
 
     if (res == SQLITE_OK) {
         sqlite3_busy_timeout(d->access, timeOut);

src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs.cpp

@@ -0,0 +1,255 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#include "qsql_sqlite_vfs_p.h"
+
+#include <QFile>
+
+#include <limits.h> // defines PATH_MAX on unix
+#include <sqlite3.h>
+#include <stdio.h> // defines FILENAME_MAX everywhere
+
+#ifndef PATH_MAX
+#  define PATH_MAX FILENAME_MAX
+#endif
+
+namespace {
+struct Vfs : sqlite3_vfs {
+    sqlite3_vfs *pVfs;
+    sqlite3_io_methods ioMethods;
+};
+
+struct File : sqlite3_file {
+    class QtFile : public QFile {
+    public:
+        QtFile(const QString &name, bool removeOnClose)
+            : QFile(name)
+            , removeOnClose(removeOnClose)
+        {}
+
+        ~QtFile() override
+        {
+            if (removeOnClose)
+                remove();
+        }
+    private:
+        bool removeOnClose;
+    };
+    QtFile *pFile;
+};
+
+
+int xClose(sqlite3_file *sfile)
+{
+    auto file = static_cast<File *>(sfile);
+    delete file->pFile;
+    file->pFile = nullptr;
+    return SQLITE_OK;
+}
+
+int xRead(sqlite3_file *sfile, void *ptr, int iAmt, sqlite3_int64 iOfst)
+{
+    auto file = static_cast<File *>(sfile);
+    if (!file->pFile->seek(iOfst))
+        return SQLITE_IOERR_READ;
+
+    auto sz = file->pFile->read(static_cast<char *>(ptr), iAmt);
+    if (sz < iAmt) {
+        memset(static_cast<char *>(ptr) + sz, 0, size_t(iAmt - sz));
+        return SQLITE_IOERR_SHORT_READ;
+    }
+    return SQLITE_OK;
+}
+
+int xWrite(sqlite3_file *sfile, const void *data, int iAmt, sqlite3_int64 iOfst)
+{
+    auto file = static_cast<File *>(sfile);
+    if (!file->pFile->seek(iOfst))
+        return SQLITE_IOERR_SEEK;
+    return file->pFile->write(reinterpret_cast<const char*>(data), iAmt) == iAmt ? SQLITE_OK : SQLITE_IOERR_WRITE;
+}
+
+int xTruncate(sqlite3_file *sfile, sqlite3_int64 size)
+{
+    auto file = static_cast<File *>(sfile);
+    return file->pFile->resize(size) ? SQLITE_OK : SQLITE_IOERR_TRUNCATE;
+}
+
+int xSync(sqlite3_file *sfile, int /*flags*/)
+{
+    static_cast<File *>(sfile)->pFile->flush();
+    return SQLITE_OK;
+}
+
+int xFileSize(sqlite3_file *sfile, sqlite3_int64 *pSize)
+{
+    auto file = static_cast<File *>(sfile);
+    *pSize = file->pFile->size();
+    return SQLITE_OK;
+}
+
+// No lock/unlock for QFile, QLockFile doesn't work for me
+
+int xLock(sqlite3_file *, int) { return SQLITE_OK; }
+
+int xUnlock(sqlite3_file *, int) { return SQLITE_OK; }
+
+int xCheckReservedLock(sqlite3_file *, int *pResOut)
+{
+    *pResOut = 0;
+    return SQLITE_OK;
+}
+
+int xFileControl(sqlite3_file *, int, void *) { return SQLITE_NOTFOUND; }
+
+int xSectorSize(sqlite3_file *)
+{
+    return 4096;
+}
+
+int xDeviceCharacteristics(sqlite3_file *)
+{
+    return 0; // no SQLITE_IOCAP_XXX
+}
+
+int xOpen(sqlite3_vfs *svfs, sqlite3_filename zName, sqlite3_file *sfile,
+          int flags, int *pOutFlags)
+{
+    auto vfs = static_cast<Vfs *>(svfs);
+    auto file = static_cast<File *>(sfile);
+    memset(file, 0, sizeof(File));
+    QIODeviceBase::OpenMode mode = QIODeviceBase::NotOpen;
+    if (!zName || (flags & SQLITE_OPEN_MEMORY))
+        return SQLITE_PERM;
+    if ((flags & SQLITE_OPEN_READONLY) &&
+        !(flags & SQLITE_OPEN_READWRITE) &&
+        !(flags & SQLITE_OPEN_CREATE) &&
+        !(flags & SQLITE_OPEN_DELETEONCLOSE)) {
+        mode |= QIODeviceBase::OpenModeFlag::ReadOnly;
+    } else {
+        /*
+            ** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction
+            ** with the [SQLITE_OPEN_CREATE] flag, which are both directly
+            ** analogous to the O_EXCL and O_CREAT flags of the POSIX open()
+            ** API.  The SQLITE_OPEN_EXCLUSIVE flag, when paired with the
+            ** SQLITE_OPEN_CREATE, is used to indicate that file should always
+            ** be created, and that it is an error if it already exists.
+            ** It is <i>not</i> used to indicate the file should be opened
+            ** for exclusive access.
+         */
+        if ((flags & SQLITE_OPEN_CREATE) && (flags & SQLITE_OPEN_EXCLUSIVE))
+            mode |= QIODeviceBase::OpenModeFlag::NewOnly;
+
+        if (flags & SQLITE_OPEN_READWRITE)
+            mode |= QIODeviceBase::OpenModeFlag::ReadWrite;
+    }
+
+    file->pMethods = &vfs->ioMethods;
+    file->pFile = new File::QtFile{QString::fromUtf8(zName), bool(flags & SQLITE_OPEN_DELETEONCLOSE)};
+    if (!file->pFile->open(mode))
+        return SQLITE_CANTOPEN;
+    if (pOutFlags)
+        *pOutFlags = flags;
+
+    return SQLITE_OK;
+}
+
+int xDelete(sqlite3_vfs *, const char *zName, int)
+{
+    return QFile::remove(QString::fromUtf8(zName)) ? SQLITE_OK : SQLITE_ERROR;
+}
+
+int xAccess(sqlite3_vfs */*svfs*/, const char *zName, int flags, int *pResOut)
+{
+    *pResOut = 0;
+    switch (flags) {
+    case SQLITE_ACCESS_EXISTS:
+    case SQLITE_ACCESS_READ:
+        *pResOut = QFile::exists(QString::fromUtf8(zName));
+        break;
+    default:
+        break;
+    }
+    return SQLITE_OK;
+}
+
+int xFullPathname(sqlite3_vfs *, const char *zName, int nOut, char *zOut)
+{
+    if (!zName)
+        return SQLITE_ERROR;
+
+    int i = 0;
+    for (;zName[i] && i < nOut; ++i)
+        zOut[i] = zName[i];
+
+    if (i >= nOut)
+        return SQLITE_ERROR;
+
+    zOut[i] = '\0';
+    return SQLITE_OK;
+}
+
+int xRandomness(sqlite3_vfs *svfs, int nByte, char *zOut)
+{
+    auto vfs = static_cast<Vfs *>(svfs)->pVfs;
+    return vfs->xRandomness(vfs, nByte, zOut);
+}
+
+int xSleep(sqlite3_vfs *svfs, int microseconds)
+{
+    auto vfs = static_cast<Vfs *>(svfs)->pVfs;
+    return vfs->xSleep(vfs, microseconds);
+}
+
+int xCurrentTime(sqlite3_vfs *svfs, double *zOut)
+{
+    auto vfs = static_cast<Vfs *>(svfs)->pVfs;
+    return vfs->xCurrentTime(vfs, zOut);
+}
+
+int xGetLastError(sqlite3_vfs *, int, char *)
+{
+   return 0;
+}
+
+int xCurrentTimeInt64(sqlite3_vfs *svfs, sqlite3_int64 *zOut)
+{
+   auto vfs = static_cast<Vfs *>(svfs)->pVfs;
+   return vfs->xCurrentTimeInt64(vfs, zOut);
+}
+} // namespace {
+
+void register_qt_vfs()
+{
+    static Vfs vfs;
+    memset(&vfs, 0, sizeof(Vfs));
+    vfs.iVersion = 1;
+    vfs.szOsFile = sizeof(File);
+    vfs.mxPathname = PATH_MAX;
+    vfs.zName = "QtVFS";
+    vfs.xOpen = &xOpen;
+    vfs.xDelete = &xDelete;
+    vfs.xAccess = &xAccess;
+    vfs.xFullPathname = &xFullPathname;
+    vfs.xRandomness = &xRandomness;
+    vfs.xSleep = &xSleep;
+    vfs.xCurrentTime = &xCurrentTime;
+    vfs.xGetLastError = &xGetLastError;
+    vfs.xCurrentTimeInt64 = &xCurrentTimeInt64;
+    vfs.pVfs = sqlite3_vfs_find(nullptr);
+    vfs.ioMethods.iVersion = 1;
+    vfs.ioMethods.xClose = &xClose;
+    vfs.ioMethods.xRead = &xRead;
+    vfs.ioMethods.xWrite = &xWrite;
+    vfs.ioMethods.xTruncate = &xTruncate;
+    vfs.ioMethods.xSync = &xSync;
+    vfs.ioMethods.xFileSize = &xFileSize;
+    vfs.ioMethods.xLock = &xLock;
+    vfs.ioMethods.xUnlock = &xUnlock;
+    vfs.ioMethods.xCheckReservedLock = &xCheckReservedLock;
+    vfs.ioMethods.xFileControl = &xFileControl;
+    vfs.ioMethods.xSectorSize = &xSectorSize;
+    vfs.ioMethods.xDeviceCharacteristics = &xDeviceCharacteristics;
+
+    sqlite3_vfs_register(&vfs, 0);
+}

src/plugins/sqldrivers/sqlite/qsql_sqlite_vfs_p.h

@@ -0,0 +1,21 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only
+
+#ifndef QSQL_SQLITE_VFS_H
+#define QSQL_SQLITE_VFS_H
+
+//
+//  W A R N I N G
+//  -------------
+//
+// This file is not part of the Qt API.  It exists purely as an
+// implementation detail.  This header file may change from version to
+// version without notice, or even be removed.
+//
+// We mean it.
+//
+
+void register_qt_vfs();
+
+
+#endif // QSQL_SQLITE_VFS_H

src/plugins/sqldrivers/sqlite/smain.cpp

@@ -4,6 +4,7 @@
 #include <qsqldriverplugin.h>
 #include <qstringlist.h>
 #include "qsql_sqlite_p.h"
+#include "qsql_sqlite_vfs_p.h"
 
 QT_BEGIN_NAMESPACE
 
@@ -23,6 +24,7 @@ public:
 QSQLiteDriverPlugin::QSQLiteDriverPlugin()
     : QSqlDriverPlugin()
 {
+    register_qt_vfs();
 }
 
 QSqlDriver* QSQLiteDriverPlugin::create(const QString &name)

src/sql/doc/src/sql-driver.qdoc

@@ -723,6 +723,17 @@
       \li Busy handler timeout in milliseconds (val <= 0: disabled),
           see \l {https://www.sqlite.org/c3ref/busy_timeout.html}
           {SQLite documentation} for more information
+
+    \row
+      \li QSQLITE_USE_QT_VFS
+      \li If set, the database is opened using Qt's VFS which allows to
+          open databases using QFile. This way it can open databases from
+          any read-write locations (e.g.android shared storage) but also
+          from read-only resources (e.g. qrc or android assets). Be aware
+          that when opening databases from read-only resources make sure
+          you add QSQLITE_OPEN_READONLY attribute as well.
+          Otherwise it will fail to open it.
+
     \row
       \li QSQLITE_OPEN_READONLY
       \li If set, the database is open in read-only mode which will fail

tests/auto/sql/kernel/CMakeLists.txt

@@ -11,3 +11,4 @@ add_subdirectory(qsqlrecord)
 add_subdirectory(qsqlthread)
 add_subdirectory(qsql)
 add_subdirectory(qsqlresult)
+add_subdirectory(qvfssql)

tests/auto/sql/kernel/qvfssql/CMakeLists.txt

@@ -0,0 +1,24 @@
+# Copyright (C) 2023 The Qt Company Ltd.
+# SPDX-License-Identifier: BSD-3-Clause
+
+#####################################################################
+## tst_qsqlfield Test:
+#####################################################################
+
+qt_internal_add_test(tst_qvfssql
+    SOURCES
+        tst_qvfssql.cpp
+    LIBRARIES
+        Qt::SqlPrivate
+)
+
+set(qvfssql_resource_files
+    "sample.db"
+)
+
+qt_internal_add_resource(tst_qvfssql "tst_qvfssql"
+    PREFIX
+        "/ro/"
+    FILES
+        ${qvfssql_resource_files}
+)

tests/auto/sql/kernel/qvfssql/tst_qvfssql.cpp

@@ -0,0 +1,94 @@
+// Copyright (C) 2023 The Qt Company Ltd.
+// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0
+
+#include <QTest>
+
+#include <qsqldatabase.h>
+#include <qstandardpaths.h>
+
+#include "../qsqldatabase/tst_databases.h"
+
+using namespace Qt::StringLiterals;
+
+class tst_QVfsSql : public QObject
+{
+    Q_OBJECT
+private slots:
+    void testRoDb();
+    void testRwDb();
+};
+
+void tst_QVfsSql::testRoDb()
+{
+    QVERIFY(QSqlDatabase::drivers().contains("QSQLITE"_L1));
+    QSqlDatabase::addDatabase("QSQLITE"_L1, "ro_db"_L1);
+    QSqlDatabase db = QSqlDatabase::database("ro_db"_L1, false);
+    QVERIFY_SQL(db, isValid());
+    db.setDatabaseName(":/ro/sample.db"_L1);
+
+    db.setConnectOptions("QSQLITE_USE_QT_VFS"_L1);
+    QVERIFY(!db.open()); // can not open as the QSQLITE_OPEN_READONLY attribute is missing
+
+    db.setConnectOptions("QSQLITE_USE_QT_VFS;QSQLITE_OPEN_READONLY"_L1);
+    QVERIFY_SQL(db, open());
+
+    QStringList tables = db.tables();
+    QSqlQuery q{db};
+    for (auto table : {"reltest1"_L1, "reltest2"_L1, "reltest3"_L1, "reltest4"_L1, "reltest5"_L1}) {
+        QVERIFY(tables.contains(table));
+        QVERIFY_SQL(q, exec("select * from " + table));
+        QVERIFY(q.next());
+    }
+    QVERIFY_SQL(q, exec("select * from reltest1 where id = 4"_L1));
+    QVERIFY_SQL(q, first());
+    QVERIFY(q.value(0).toInt() == 4);
+    QVERIFY(q.value(1).toString() == "boris"_L1);
+    QVERIFY(q.value(2).toInt() == 2);
+    QVERIFY(q.value(3).toInt() == 2);
+}
+
+void tst_QVfsSql::testRwDb()
+{
+    QSqlDatabase::addDatabase("QSQLITE"_L1, "rw_db"_L1);
+    QSqlDatabase db = QSqlDatabase::database("rw_db"_L1, false);
+    QVERIFY_SQL(db, isValid());
+    const auto dbPath = QStandardPaths::writableLocation(QStandardPaths::TempLocation) + "/test_qt_vfs.db"_L1;
+    db.setDatabaseName(dbPath);
+    QFile::remove(dbPath);
+
+    db.setConnectOptions("QSQLITE_USE_QT_VFS;QSQLITE_OPEN_READONLY"_L1);
+    QVERIFY(!db.open()); // can not open as the QSQLITE_OPEN_READONLY attribute is set and the file is missing
+
+    db.setConnectOptions("QSQLITE_USE_QT_VFS"_L1);
+    QVERIFY_SQL(db, open());
+
+    QVERIFY(db.tables().isEmpty());
+    QSqlQuery q{db};
+    QVERIFY_SQL(q, exec("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)"_L1));
+    QVERIFY_SQL(q, exec("BEGIN"_L1));
+    for (int i = 0; i < 1000; ++i) {
+        q.prepare("INSERT INTO test (val) VALUES (:val)"_L1);
+        q.bindValue(":val"_L1, i);
+        QVERIFY_SQL(q, exec());
+    }
+    QVERIFY_SQL(q, exec("COMMIT"_L1));
+    QVERIFY_SQL(q, exec("SELECT val FROM test ORDER BY val"_L1));
+    for (int i = 0; i < 1000; ++i) {
+        QVERIFY_SQL(q, next());
+        QCOMPARE(q.value(0).toInt() , i);
+    }
+    QVERIFY_SQL(q, exec("DELETE FROM test WHERE val < 500"_L1));
+    auto fileSize = QFileInfo{dbPath}.size();
+    QVERIFY_SQL(q, exec("VACUUM"_L1));
+    QVERIFY(QFileInfo{dbPath}.size() < fileSize); // TEST xTruncate VFS
+    QVERIFY_SQL(q, exec("SELECT val FROM test ORDER BY val"_L1));
+    for (int i = 500; i < 1000; ++i) {
+        QVERIFY_SQL(q, next());
+        QCOMPARE(q.value(0).toInt() , i);
+    }
+    db.close();
+    QFile::remove(dbPath);
+}
+
+QTEST_APPLESS_MAIN(tst_QVfsSql)
+#include "tst_qvfssql.moc"