macOS: Don't try to close already-closed/closing file dialog

Doing so results in a warning about "modalSession has been exited
prematurely - check for a reentrant call to endModalSession:", and
on Big Sur will also result in the file failing to save because the
return code from runModal will no longer be NSModalResponseOK.

This would happen when the completion handler for beginSheetModalForWindow
would call QNSOpenSavePanelDelegate_panelClosed, resulting in calls to
QDialog::done(), which in turn tries to hide the dialog, via
QCocoaFileDialogHelper::hideCocoaFilePanel().

Pick-to: 6.0 5.15
Fixes: QTBUG-89959
Change-Id: I048afe3dcc7fe62e0d0273f12b4b2c0237abb052
Reviewed-by: Timur Pocheptsov <timur.pocheptsov@qt.io>

src/plugins/platforms/cocoa/qcocoafiledialoghelper.mm

@@ -74,6 +74,8 @@ QT_USE_NAMESPACE
 
 typedef QSharedPointer<QFileDialogOptions> SharedPointerFileDialogOptions;
 
+static const int kReturnCodeNotSet = -1;
+
 @implementation QNSOpenSavePanelDelegate {
     @public
     NSOpenPanel *mOpenPanel;
@@ -110,7 +112,7 @@ typedef QSharedPointer<QFileDialogOptions> SharedPointerFileDialogOptions;
     if ([mSavePanel respondsToSelector:@selector(setLevel:)])
         [mSavePanel setLevel:NSModalPanelWindowLevel];
 
-    mReturnCode = -1;
+    mReturnCode = kReturnCodeNotSet;
     mHelper = helper;
     mNameFilterDropDownList = new QStringList(mOptions->nameFilters());
     QString selectedVisualNameFilter = mOptions->initiallySelectedNameFilter();
@@ -177,6 +179,10 @@ static QString strippedText(QString s)
 
 - (void)closePanel
 {
+    // An already closed/closing panel has its return code set
+    if (mReturnCode != kReturnCodeNotSet)
+        return;
+
     *mCurrentSelection = QString::fromNSString([[mSavePanel URL] path]).normalized(QString::NormalizationForm_C);
     if ([mSavePanel respondsToSelector:@selector(close)])
         [mSavePanel close];