Calculate velocity in QMutEventPoint::setTimestamp() with Kalman filter

This functionality was only in Qt Quick in Qt 5.  Now we move it up to QtGui
so that every QEventPoint will have a valid velocity() before being delivered
anywhere.

[ChangeLog][QtGui][QPointerEvent] Every QEventPoint should now carry a valid
velocity(): if the operating system doesn't provide it, Qt will calculate it,
using a simple Kalman filter to provide a weighted average over time.

Fixes: QTBUG-33891
Change-Id: I40352f717f0ad6edd87cf71ef55e955a591eeea1
Reviewed-by: Volker Hilsheimer <volker.hilsheimer@qt.io>

src/gui/kernel/qevent.cpp

@@ -62,6 +62,7 @@
 QT_BEGIN_NAMESPACE
 
 Q_LOGGING_CATEGORY(lcPointerGrab, "qt.pointer.grab")
+Q_LOGGING_CATEGORY(lcPointerVel, "qt.pointer.velocity")
 Q_LOGGING_CATEGORY(lcEPDetach, "qt.pointer.eventpoint.detach")
 
 /*!
@@ -335,6 +336,14 @@ QSizeF QEventPoint::ellipseDiameters() const
 bool QEventPoint::isAccepted() const
 { return d->accept; }
 
+/*!
+    Returns the time from the previous QPointerEvent that contained this point.
+
+    \sa globalLastPosition()
+*/
+ulong QEventPoint::lastTimestamp() const
+{ return d->lastTimestamp; }
+
 /*!
     Sets the accepted state of the point.
 
@@ -487,14 +496,30 @@ void QMutableEventPoint::updateFrom(const QEventPoint &other)
 }
 
 /*! \internal
-    Set the timestamp from the event that updated this point's positions.
+    Set the timestamp from the event that updated this point's positions,
+    and calculate a new value for velocity().
+
+    The velocity calculation is done here because none of the QPointerEvent
+    subclass constructors take the timestamp directly, and because
+    QGuiApplication traditionally constructs an event first and then sets its
+    timestamp (see for example QGuiApplicationPrivate::processMouseEvent()).
+
+    This function looks up the corresponding instance in QPointingDevicePrivate::activePoints,
+    and assumes that its timestamp() still holds the previous time when this point
+    was updated, its velocity() holds this point's last-known velocity, and
+    its globalPosition() and globalLastPosition() hold this point's current
+    and previous positions, respectively.  We assume timestamps are in milliseconds.
+
+    The velocity calculation is skipped if the platform has promised to
+    provide velocities already by setting the QInputDevice::Velocity capability.
 */
 void QMutableEventPoint::setTimestamp(const ulong t)
 {
     // On mouse press, if the mouse has moved from its last-known location,
     // QGuiApplicationPrivate::processMouseEvent() sends first a mouse move and
     // then a press. Both events will get the same timestamp. So we need to set
-    // the press timestamp and position even when the timestamp isn't advancing.
+    // the press timestamp and position even when the timestamp isn't advancing,
+    // but skip setting lastTimestamp and velocity because those need a time delta.
     if (state() == QEventPoint::State::Pressed) {
         d->pressTimestamp = t;
         d->globalPressPos = d->globalPos;
@@ -502,6 +527,33 @@ void QMutableEventPoint::setTimestamp(const ulong t)
     if (d->timestamp == t)
         return;
     detach();
+    if (device()) {
+        // get the persistent instance out of QPointingDevicePrivate::activePoints
+        // (which sometimes might be the same as this instance)
+        QEventPointPrivate *pd = QPointingDevicePrivate::get(
+                    const_cast<QPointingDevice *>(d->device))->pointById(id())->eventPoint.d;
+        if (t > pd->timestamp) {
+            pd->lastTimestamp = pd->timestamp;
+            pd->timestamp = t;
+            if (state() == QEventPoint::State::Pressed)
+                pd->pressTimestamp = t;
+            if (pd->lastTimestamp > 0 && !device()->capabilities().testFlag(QInputDevice::Capability::Velocity)) {
+                // calculate instantaneous velocity according to time and distance moved since the previous point
+                QVector2D newVelocity = QVector2D(pd->globalPos - pd->globalLastPos) / (t - pd->lastTimestamp) * 1000;
+                // VERY simple kalman filter: does a weighted average
+                // where the older velocities get less and less significant
+                static const float KalmanGain = 0.7f;
+                pd->velocity = newVelocity * KalmanGain + pd->velocity * (1.0f - KalmanGain);
+                qCDebug(lcPointerVel) << "velocity" << newVelocity << "filtered" << pd->velocity <<
+                                         "based on movement" << pd->globalLastPos << "->" << pd->globalPos <<
+                                         "over time" << pd->lastTimestamp << "->" << pd->timestamp;
+            }
+            if (d != pd) {
+                d->lastTimestamp = pd->lastTimestamp;
+                d->velocity = pd->velocity;
+            }
+        }
+    }
     d->timestamp = t;
 }
 
@@ -4820,10 +4872,17 @@ bool QTouchEvent::isReleaseEvent() const
 
 /*!
     \fn QVector2D QEventPoint::velocity() const
-    Returns a velocity vector for this point.
+    Returns a velocity vector, in units of pixels per second, in the coordinate
-    The vector is in the screen's coordinate system, using pixels per seconds for the magnitude.
+    system of the screen or desktop.
 
-    \note The returned vector is only valid if the device's capabilities include QInputDevice::Velocity.
+    \note If the device's capabilities include QInputDevice::Velocity, it means
+    velocity comes from the operating system (perhaps the touch hardware or
+    driver provides it). But usually the \c Velocity capability is not set,
+    indicating that the velocity is calculated by Qt, using a simple Kalman
+    filter to provide a smoothed average velocity rather than an instantaneous
+    value. Effectively it tells how fast and in what direction the user has
+    been dragging this point over the last few events, with the most recent
+    event having the strongest influence.
 
     \sa QInputDevice::capabilities(), QInputEvent::device()
 */

src/gui/kernel/qevent.h

@@ -157,6 +157,7 @@ public:
     int id() const;
     QPointingDeviceUniqueId uniqueId() const;
     ulong timestamp() const;
+    ulong lastTimestamp() const;
     ulong pressTimestamp() const;
     qreal timeHeld() const;
     qreal pressure() const;

src/gui/kernel/qevent_p.h

@@ -81,6 +81,7 @@ struct QEventPointPrivate {
     QSizeF ellipseDiameters = QSizeF(0, 0);
     QVector2D velocity;
     ulong timestamp = 0;
+    ulong lastTimestamp = 0;
     ulong pressTimestamp = 0;
     QPointingDeviceUniqueId uniqueId;
     int refCount = 1;

tests/auto/gui/kernel/qmouseevent/tst_qmouseevent.cpp

@@ -1,6 +1,6 @@
 /****************************************************************************
 **
-** Copyright (C) 2016 The Qt Company Ltd.
+** Copyright (C) 2020 The Qt Company Ltd.
 ** Contact: https://www.qt.io/licensing/
 **
 ** This file is part of the test suite of the Qt Toolkit.
@@ -51,11 +51,17 @@ public:
     int mouseReleaseButton;
     int mouseReleaseButtons;
     int mouseReleaseModifiers;
+    ulong timestamp;
     ulong pressTimestamp;
+    ulong lastTimestamp;
+    QVector2D velocity;
 protected:
     void mousePressEvent(QMouseEvent *e) override
     {
         const auto &firstPoint = e->point(0);
+        qCDebug(lcTests) << e << firstPoint;
+        timestamp = firstPoint.timestamp();
+        lastTimestamp = firstPoint.lastTimestamp();
         if (e->type() == QEvent::MouseButtonPress) {
             auto firstPoint = e->points().first();
             QCOMPARE(e->exclusiveGrabber(firstPoint), nullptr);
@@ -77,12 +83,23 @@ protected:
         if (grabPassive)
             e->addPassiveGrabber(firstPoint, this);
     }
+    void mouseMoveEvent(QMouseEvent *e) override
+    {
+        qCDebug(lcTests) << e << e->points().first();
+        timestamp = e->points().first().timestamp();
+        lastTimestamp = e->points().first().lastTimestamp();
+        velocity = e->points().first().velocity();
+    }
     void mouseReleaseEvent(QMouseEvent *e) override
     {
+        qCDebug(lcTests) << e << e->points().first();
         QWindow::mouseReleaseEvent(e);
         mouseReleaseButton = e->button();
         mouseReleaseButtons = e->buttons();
         mouseReleaseModifiers = e->modifiers();
+        timestamp = e->points().first().timestamp();
+        lastTimestamp = e->points().first().lastTimestamp();
+        velocity = e->points().first().velocity();
         mouseReleaseEventRecieved = true;
         e->accept();
     }
@@ -104,6 +121,7 @@ private slots:
     void checkMouseReleaseEvent();
     void grabbers_data();
     void grabbers();
+    void velocity();
 
 private:
     MouseEventWidget* testMouseWidget;
@@ -279,5 +297,41 @@ void tst_QMouseEvent::grabbers()
     QCOMPARE(firstEPD->passiveGrabbers.count(), 0);
 }
 
+void tst_QMouseEvent::velocity()
+{
+    testMouseWidget->grabExclusive = true;
+    auto devPriv = QPointingDevicePrivate::get(const_cast<QPointingDevice *>(QPointingDevice::primaryPointingDevice()));
+    devPriv->activePoints.clear();
+
+    qCDebug(lcTests) << "sending mouse press event";
+    QPoint pos(10, 10);
+    QTest::mousePress(testMouseWidget, Qt::LeftButton, Qt::KeyboardModifiers(), pos);
+    QCOMPARE(devPriv->activePoints.count(), 1);
+    QVERIFY(devPriv->activePoints.count() <= 2);
+    const auto &firstPoint = devPriv->pointById(0)->eventPoint;
+    QVERIFY(firstPoint.timestamp() > 0);
+    QCOMPARE(firstPoint.state(), QEventPoint::State::Pressed);
+
+    ulong timestamp = firstPoint.timestamp();
+    for (int i = 1; i < 4; ++i) {
+        qCDebug(lcTests) << "sending mouse move event" << i;
+        pos += {10, 10};
+        QTest::mouseMove(testMouseWidget, pos, 1);
+        qApp->processEvents();
+        qCDebug(lcTests) << firstPoint;
+        // currently we expect it to be updated in-place in devPriv->activePoints
+        QVERIFY(firstPoint.timestamp() > timestamp);
+        QVERIFY(testMouseWidget->timestamp > testMouseWidget->lastTimestamp);
+        QCOMPARE(testMouseWidget->timestamp, firstPoint.timestamp());
+        timestamp = firstPoint.timestamp();
+        QVERIFY(testMouseWidget->velocity.x() > 0);
+        QVERIFY(testMouseWidget->velocity.y() > 0);
+    }
+    QTest::mouseRelease(testMouseWidget, Qt::LeftButton, Qt::KeyboardModifiers(), pos, 1);
+    qCDebug(lcTests) << firstPoint;
+    QVERIFY(testMouseWidget->velocity.x() > 0);
+    QVERIFY(testMouseWidget->velocity.y() > 0);
+}
+
 QTEST_MAIN(tst_QMouseEvent)
 #include "tst_qmouseevent.moc"

tests/auto/gui/kernel/qtouchevent/tst_qtouchevent.cpp

@@ -215,10 +215,16 @@ struct GrabberWindow : public QWindow
 {
     bool grabExclusive = false;
     bool grabPassive = false;
+    QVector2D velocity;
+    ulong timestamp;
+    ulong lastTimestamp;
 
     void touchEvent(QTouchEvent *ev) override {
         qCDebug(lcTests) << ev;
         const auto &firstPoint = ev->point(0);
+        velocity = firstPoint.velocity();
+        timestamp = firstPoint.timestamp();
+        lastTimestamp = firstPoint.lastTimestamp();
         switch (ev->type()) {
         case QEvent::TouchBegin: {
             QCOMPARE(ev->exclusiveGrabber(firstPoint), nullptr);
@@ -264,6 +270,7 @@ private slots:
     void testMultiDevice();
     void grabbers_data();
     void grabbers();
+    void velocity();
 
 private:
     QPointingDevice *touchScreenDevice;
@@ -1942,6 +1949,45 @@ void tst_QTouchEvent::grabbers()
     QTRY_COMPARE(devPriv->activePoints.count(), 0);
 }
 
+void tst_QTouchEvent::velocity()
+{
+    GrabberWindow w;
+    w.grabExclusive = true;
+    w.setGeometry(100, 100, 100, 100);
+    w.show();
+    QVERIFY(QTest::qWaitForWindowExposed(&w));
+
+    auto devPriv = QPointingDevicePrivate::get(touchScreenDevice);
+    devPriv->activePoints.clear();
+    QPoint pos(10, 10);
+    QTest::touchEvent(&w, touchScreenDevice).press(0, pos, &w);
+    QCOMPARE(devPriv->activePoints.count(), 1);
+    const auto &firstPoint = devPriv->pointById(0)->eventPoint;
+    qCDebug(lcTests) << "persistent active point after press" << firstPoint;
+    QCOMPARE(firstPoint.velocity(), QVector2D());
+
+    QCOMPARE(firstPoint.pressTimestamp(), firstPoint.timestamp());
+    QVERIFY(firstPoint.timestamp() > 0);
+    QCOMPARE(firstPoint.state(), QEventPoint::State::Pressed);
+
+    ulong timestamp = firstPoint.timestamp();
+    for (int i = 1; i < 4; ++i) {
+        qCDebug(lcTests) << "sending touch move event" << i;
+        pos += {10, 10};
+        QTest::touchEvent(&w, touchScreenDevice).move(0, pos, &w);
+        qCDebug(lcTests) << firstPoint;
+        QVERIFY(firstPoint.timestamp() > timestamp);
+        QVERIFY(w.timestamp > w.lastTimestamp);
+        QCOMPARE(w.timestamp, firstPoint.timestamp());
+        timestamp = firstPoint.timestamp();
+        QVERIFY(w.velocity.x() > 0);
+        QVERIFY(w.velocity.y() > 0);
+    }
+    QTest::touchEvent(&w, touchScreenDevice).release(0, pos, &w);
+    QVERIFY(w.velocity.x() > 0);
+    QVERIFY(w.velocity.y() > 0);
+}
+
 QTEST_MAIN(tst_QTouchEvent)
 
 #include "tst_qtouchevent.moc"