Make toplevel transparent windows work on Mac OS X

This patch includes a few different fixes to make transparent toplevels
work on cocoa.
 - When setting alpha on the toplevel, it also needs setOpaque:NO
 - The OpenGL context needs a separate flag for this to work.
 - Make sure setOpaque fighting between setMask, setFormat and
   setOpacity ends up correctly

Task-number: QTBUG-28214
Change-Id: Ic3a2d71193bb653e181c98787b4ebda002424092
Reviewed-by: Shawn Rutledge <shawn.rutledge@digia.com>
Reviewed-by: Morten Johan Sørvig <morten.sorvig@digia.com>

src/plugins/platforms/cocoa/qcocoaglcontext.mm

@@ -78,6 +78,10 @@ QCocoaGLContext::QCocoaGLContext(const QSurfaceFormat &format, QPlatformOpenGLCo
     const GLint interval = 1;
     [m_context setValues:&interval forParameter:NSOpenGLCPSwapInterval];
 
+    if (format.alphaBufferSize() > 0) {
+        int zeroOpacity = 0;
+        [m_context setValues:&zeroOpacity forParameter:NSOpenGLCPSurfaceOpacity];
+    }
 }
 
 QCocoaGLContext::~QCocoaGLContext()

src/plugins/platforms/cocoa/qcocoawindow.h

@@ -157,6 +157,8 @@ protected:
     QCocoaWindow *parentCocoaWindow() const;
     void syncWindowState(Qt::WindowState newState);
 
+    void updateOpaque();
+
 // private:
 public: // for QNSView
     friend class QCocoaBackingStore;

src/plugins/platforms/cocoa/qcocoawindow.mm

@@ -516,20 +516,30 @@ void QCocoaWindow::propagateSizeHints()
     }
 }
 
+void QCocoaWindow::updateOpaque()
+{
+    bool translucent = window()->format().alphaBufferSize() > 0
+            || window()->opacity() < 1
+            || (m_contentView && [m_contentView hasMask]);
+    [m_nsWindow setOpaque:!translucent];
+}
+
+
 void QCocoaWindow::setOpacity(qreal level)
 {
-    if (m_nsWindow)
+    if (m_nsWindow) {
         [m_nsWindow setAlphaValue:level];
+        updateOpaque();
+    }
 }
 
 void QCocoaWindow::setMask(const QRegion &region)
 {
-    if (m_nsWindow) {
-        [m_nsWindow setOpaque:NO];
+    if (m_nsWindow)
         [m_nsWindow setBackgroundColor:[NSColor clearColor]];
-    }
 
     [m_contentView setMaskRegion:&region];
+    updateOpaque();
 }
 
 bool QCocoaWindow::setKeyboardGrabEnabled(bool grab)
@@ -726,6 +736,12 @@ NSWindow * QCocoaWindow::createNSWindow()
 
     NSInteger level = windowLevel(flags);
     [createdWindow setLevel:level];
+
+    if (window()->format().alphaBufferSize() > 0) {
+        [createdWindow setBackgroundColor:[NSColor clearColor]];
+        [createdWindow setOpaque:NO];
+    }
+
     m_windowModality = window()->modality();
     return createdWindow;
 }

src/plugins/platforms/cocoa/qnsview.h

@@ -83,6 +83,7 @@ QT_END_NAMESPACE
 - (BOOL)isFlipped;
 - (BOOL)acceptsFirstResponder;
 - (BOOL)becomeFirstResponder;
+- (BOOL)hasMask;
 
 - (void)resetMouseButtons;
 

src/plugins/platforms/cocoa/qnsview.mm

@@ -254,6 +254,11 @@ static QTouchDevice *touchDevice = 0;
     [self setNeedsDisplayInRect:NSMakeRect(br.x(), br.y(), br.width(), br.height())];
 }
 
+- (BOOL) hasMask
+{
+    return m_maskData != 0;
+}
+
 - (void) setMaskRegion:(const QRegion *)region
 {
     if (m_maskImage)