From 6056c686b8264d00022825005262618e0f0ade00 Mon Sep 17 00:00:00 2001
From: attila <attila@juce.com>
Date: Tue, 30 Jan 2024 11:19:56 +0100
Subject: [PATCH] MacOS: Fix invisible PopupMenu in Logic

We avoid modifying window controls in modal situations if possible.
Doing as much as [window setStyleMask: [window styleMask]]; for a
PopupMenu window will make it invisible on MacOS when the plugin is
hosted in an external subprocess.

This fixes a regression caused by
fb14118771eeb34f51146e23def0a1f94a49fbad.
---
 .../native/juce_NSViewComponentPeer_mac.mm    | 78 +++++++++++++++----
 1 file changed, 64 insertions(+), 14 deletions(-)

diff --git a/modules/juce_gui_basics/native/juce_NSViewComponentPeer_mac.mm b/modules/juce_gui_basics/native/juce_NSViewComponentPeer_mac.mm
index da1a462bcf..cdf7c363ee 100644
--- a/modules/juce_gui_basics/native/juce_NSViewComponentPeer_mac.mm
+++ b/modules/juce_gui_basics/native/juce_NSViewComponentPeer_mac.mm
@@ -1998,33 +1998,83 @@ private:
        #endif
     }
 
+    /*  Used to store and restore the values of the NSWindowStyleMaskClosable and
+        NSWindowStyleMaskMiniaturizable flags.
+    */
+    struct StoredStyleFlags
+    {
+        StoredStyleFlags (NSWindowStyleMask m)
+            : stored { m }
+        {}
+
+        static auto getStoredFlags()
+        {
+            return std::array<NSWindowStyleMask, 2> { NSWindowStyleMaskClosable,
+                                                      NSWindowStyleMaskMiniaturizable };
+        }
+
+        auto withFlagsRestored (NSWindowStyleMask m) const
+        {
+            for (const auto& f : getStoredFlags())
+                m = withFlagFromStored (m, f);
+
+            return m;
+        }
+
+    private:
+        NSWindowStyleMask withFlagFromStored (NSWindowStyleMask m, NSWindowStyleMask flag) const
+        {
+            return (m & ~flag) | (stored & flag);
+        }
+
+        NSWindowStyleMask stored;
+    };
+
     void modalComponentManagerChanged()
     {
-        if (isSharedWindow)
+        // We are only changing the style flags if we absolutely have to. Plugin windows generally
+        // don't like to be modified. Windows created under plugin hosts running in an external
+        // subprocess are particularly touchy, and may make the window invisible even if we call
+        // [window setStyleMask [window setStyleMask]].
+        if (isSharedWindow || ! hasNativeTitleBar())
             return;
 
-        auto style = [window styleMask];
-
-        if (ModalComponentManager::getInstance()->getNumModalComponents() > 0)
+        const auto newStyleMask = [&]() -> std::optional<NSWindowStyleMask>
         {
-            style &= ~NSWindowStyleMaskMiniaturizable;
-            style &= ~NSWindowStyleMaskClosable;
-        }
-        else
-        {
-            const auto flags = getStyleFlags();
+            const auto currentStyleMask = [window styleMask];
 
-            if ((flags & windowHasMinimiseButton) != 0)  style |= NSWindowStyleMaskMiniaturizable;
-            if ((flags & windowHasCloseButton) != 0)     style |= NSWindowStyleMaskClosable;
-        }
+            if (ModalComponentManager::getInstance()->getNumModalComponents() > 0)
+            {
+                if (! storedFlags)
+                    storedFlags.emplace (currentStyleMask);
 
-        [window setStyleMask: style];
+                auto updatedMask = (storedFlags->withFlagsRestored (currentStyleMask)) & ~NSWindowStyleMaskMiniaturizable;
+
+                if (component.isCurrentlyBlockedByAnotherModalComponent())
+                    updatedMask &= ~NSWindowStyleMaskClosable;
+
+                return updatedMask;
+            }
+
+            if (storedFlags)
+            {
+                const auto flagsToApply = storedFlags->withFlagsRestored (currentStyleMask);
+                storedFlags.reset();
+                return flagsToApply;
+            }
+
+            return {};
+        }();
+
+        if (newStyleMask && *newStyleMask != [window styleMask])
+            [window setStyleMask: *newStyleMask];
     }
 
     //==============================================================================
     std::vector<ScopedNotificationCenterObserver> scopedObservers;
     std::vector<ScopedNotificationCenterObserver> windowObservers;
 
+    std::optional<StoredStyleFlags> storedFlags;
     ErasedScopeGuard modalChangeListenerScope =
         detail::ComponentHelpers::ModalComponentManagerChangeNotifier::getInstance().addListener ([this]
                                                                                                   {