From 6c09aa69d9d70ffc72f110d27bbecfed0c79852b Mon Sep 17 00:00:00 2001
From: attila <attila@juce.com>
Date: Tue, 28 Jun 2022 17:35:05 +0200
Subject: [PATCH] VST3 Client:  Fix setComponentState() threading on Linux

Before this commit it was possible for the plugin to transfer control
to its internal MessageThread and call
IComponentHandler::restartComponent() from it.
---
 .../VST3/juce_VST3_Wrapper.cpp                | 84 ++++++++++++++-----
 1 file changed, 63 insertions(+), 21 deletions(-)

diff --git a/modules/juce_audio_plugin_client/VST3/juce_VST3_Wrapper.cpp b/modules/juce_audio_plugin_client/VST3/juce_VST3_Wrapper.cpp
index 1e77dee858..b80ae5a5bb 100644
--- a/modules/juce_audio_plugin_client/VST3/juce_VST3_Wrapper.cpp
+++ b/modules/juce_audio_plugin_client/VST3/juce_VST3_Wrapper.cpp
@@ -125,6 +125,34 @@ using namespace Steinberg;
 //==============================================================================
 #if JUCE_LINUX || JUCE_BSD
 
+enum class HostMessageThreadAttached { no, yes };
+
+class HostMessageThreadState
+{
+public:
+    template <typename Callback>
+    void setStateWithAction (HostMessageThreadAttached stateIn, Callback&& action)
+    {
+        const std::lock_guard<std::mutex> lock { m };
+        state = stateIn;
+        action();
+    }
+
+    void assertHostMessageThread()
+    {
+        const std::lock_guard<std::mutex> lock { m };
+
+        if (state == HostMessageThreadAttached::no)
+            return;
+
+        JUCE_ASSERT_MESSAGE_THREAD
+    }
+
+private:
+    HostMessageThreadAttached state = HostMessageThreadAttached::no;
+    std::mutex m;
+};
+
 class EventHandler final  : public Steinberg::Linux::IEventHandler,
                             private LinuxEventLoopInternal::Listener
 {
@@ -141,7 +169,8 @@ public:
         LinuxEventLoopInternal::deregisterLinuxEventLoopListener (*this);
 
         if (! messageThread->isRunning())
-            messageThread->start();
+            hostMessageThreadState.setStateWithAction (HostMessageThreadAttached::no,
+                                                       [this]() { messageThread->start(); });
     }
 
     JUCE_DECLARE_VST3_COM_REF_METHODS
@@ -173,6 +202,17 @@ public:
             refreshAttachedEventLoop ([this, runLoop] { hostRunLoops.erase (runLoop); });
     }
 
+    /* Asserts if it can be established that the calling thread is different from the host's message
+       thread.
+
+       On Linux this can only be determined if the host has already registered its run loop. Until
+       then JUCE messages are serviced by a background thread internal to the plugin.
+    */
+    static void assertHostMessageThread()
+    {
+        hostMessageThreadState.assertHostMessageThread();
+    }
+
 private:
     //==============================================================================
     /*  Connects all known FDs to a single host event loop instance. */
@@ -240,7 +280,8 @@ private:
             if (messageThread->isRunning())
                 messageThread->stop();
 
-            MessageManager::getInstance()->setCurrentThreadAsMessageThread();
+            hostMessageThreadState.setStateWithAction (HostMessageThreadAttached::yes,
+                                                       [] { MessageManager::getInstance()->setCurrentThreadAsMessageThread(); });
         }
     }
 
@@ -283,13 +324,26 @@ private:
     std::multiset<Steinberg::Linux::IRunLoop*> hostRunLoops;
     AttachedEventLoop attachedEventLoop;
 
+    static HostMessageThreadState hostMessageThreadState;
+
     //==============================================================================
     JUCE_DECLARE_NON_MOVEABLE (EventHandler)
     JUCE_DECLARE_NON_COPYABLE_WITH_LEAK_DETECTOR (EventHandler)
 };
 
+HostMessageThreadState EventHandler::hostMessageThreadState;
+
 #endif
 
+static void assertHostMessageThread()
+{
+   #if JUCE_LINUX || JUCE_BSD
+    EventHandler::assertHostMessageThread();
+   #else
+    JUCE_ASSERT_MESSAGE_THREAD
+   #endif
+}
+
 //==============================================================================
 class InParameterChangedCallbackSetter
 {
@@ -968,25 +1022,8 @@ public:
     //==============================================================================
     tresult PLUGIN_API setComponentState (IBStream* stream) override
     {
-        if (! MessageManager::existsAndIsCurrentThread())
-       #if JUCE_LINUX || JUCE_BSD
-        {
-            tresult result = kResultOk;
-            WaitableEvent finishedEvent;
-
-            MessageManager::callAsync ([&]
-            {
-                result = setComponentState (stream);
-                finishedEvent.signal();
-            });
-
-            finishedEvent.wait();
-            return result;
-        }
-       #else
         // As an IEditController member, the host should only call this from the message thread.
-        jassertfalse;
-       #endif
+        assertHostMessageThread();
 
         if (auto* pluginInstance = getPluginInstance())
         {
@@ -1965,7 +2002,12 @@ private:
                     owner->lastScaleFactorReceived = editorScaleFactor;
 
                 if (component != nullptr)
+                {
+                   #if JUCE_LINUX || JUCE_BSD
+                    const MessageManagerLock mmLock;
+                   #endif
                     component->setEditorScaleFactor (editorScaleFactor);
+                }
             }
 
             return kResultTrue;
@@ -2776,7 +2818,7 @@ public:
     {
         // The VST3 spec requires that this function is called from the UI thread.
         // If this assertion fires, your host is misbehaving!
-        JUCE_ASSERT_MESSAGE_THREAD
+        assertHostMessageThread();
 
         if (state == nullptr)
             return kInvalidArgument;