diff --git a/src/plugins/platforms/cocoa/qcocoamessagedialog.mm b/src/plugins/platforms/cocoa/qcocoamessagedialog.mm
index 13be407848..d5d9469832 100644
--- a/src/plugins/platforms/cocoa/qcocoamessagedialog.mm
+++ b/src/plugins/platforms/cocoa/qcocoamessagedialog.mm
@@ -307,6 +307,8 @@ void QCocoaMessageDialog::hide()
         }
     } else {
         qCDebug(lcQpaDialogs) << "No need to hide already hidden" << m_alert;
+        auto alert = std::exchange(m_alert, nil);
+        [alert autorelease];
     }
 }
 
diff --git a/tests/auto/widgets/dialogs/qerrormessage/CMakeLists.txt b/tests/auto/widgets/dialogs/qerrormessage/CMakeLists.txt
index ba4ea75b79..1cb63beea9 100644
--- a/tests/auto/widgets/dialogs/qerrormessage/CMakeLists.txt
+++ b/tests/auto/widgets/dialogs/qerrormessage/CMakeLists.txt
@@ -12,6 +12,7 @@ qt_internal_add_test(tst_qerrormessage
         tst_qerrormessage.cpp
     LIBRARIES
         Qt::Gui
+        Qt::GuiPrivate
         Qt::Widgets
 )
 
diff --git a/tests/auto/widgets/dialogs/qerrormessage/tst_qerrormessage.cpp b/tests/auto/widgets/dialogs/qerrormessage/tst_qerrormessage.cpp
index 97dad396e5..5cae8263dc 100644
--- a/tests/auto/widgets/dialogs/qerrormessage/tst_qerrormessage.cpp
+++ b/tests/auto/widgets/dialogs/qerrormessage/tst_qerrormessage.cpp
@@ -5,16 +5,38 @@
 #include <QDebug>
 #include <QCheckBox>
 
+#include <qpa/qplatformtheme.h>
+#include <private/qguiapplication_p.h>
+
 class tst_QErrorMessage : public QObject
 {
     Q_OBJECT
 
 private slots:
+    void initTestCase_data();
+    void init();
+
     void dontShowAgain();
     void dontShowCategoryAgain();
 
 };
 
+void tst_QErrorMessage::initTestCase_data()
+{
+    QTest::addColumn<bool>("useNativeDialog");
+    QTest::newRow("widget") << false;
+    if (const QPlatformTheme *theme = QGuiApplicationPrivate::platformTheme()) {
+        if (theme->usePlatformNativeDialog(QPlatformTheme::MessageDialog))
+            QTest::newRow("native") << true;
+    }
+}
+
+void tst_QErrorMessage::init()
+{
+    QFETCH_GLOBAL(bool, useNativeDialog);
+    qApp->setAttribute(Qt::AA_DontUseNativeDialogs, !useNativeDialog);
+}
+
 void tst_QErrorMessage::dontShowAgain()
 {
     QString plainString = QLatin1String("foo");