diff --git a/modules/juce_gui_basics/layout/juce_Viewport.cpp b/modules/juce_gui_basics/layout/juce_Viewport.cpp
index e37dd362f3..dfccd68ac9 100644
--- a/modules/juce_gui_basics/layout/juce_Viewport.cpp
+++ b/modules/juce_gui_basics/layout/juce_Viewport.cpp
@@ -222,7 +222,7 @@ struct Viewport::DragToScrollListener   : private MouseListener,
                                                                 (int) offsetY.getPosition()));
     }
 
-    void mouseDown (const MouseEvent&) override
+    void mouseDown (const MouseEvent& e) override
     {
         if (! isGlobalMouseListener)
         {
@@ -235,12 +235,15 @@ struct Viewport::DragToScrollListener   : private MouseListener,
             Desktop::getInstance().addGlobalMouseListener (this);
 
             isGlobalMouseListener = true;
+
+            scrollSource = e.source;
         }
     }
 
     void mouseDrag (const MouseEvent& e) override
     {
-        if (Desktop::getInstance().getNumDraggingMouseSources() == 1 && ! doesMouseEventComponentBlockViewportDrag (e.eventComponent))
+        if (e.source == scrollSource
+            && ! doesMouseEventComponentBlockViewportDrag (e.eventComponent))
         {
             auto totalOffset = e.getOffsetFromDragStart().toFloat();
 
@@ -263,9 +266,9 @@ struct Viewport::DragToScrollListener   : private MouseListener,
         }
     }
 
-    void mouseUp (const MouseEvent&) override
+    void mouseUp (const MouseEvent& e) override
     {
-        if (isGlobalMouseListener && Desktop::getInstance().getNumDraggingMouseSources() == 0)
+        if (isGlobalMouseListener && e.source == scrollSource)
             endDragAndClearGlobalMouseListener();
     }
 
@@ -293,6 +296,7 @@ struct Viewport::DragToScrollListener   : private MouseListener,
     Viewport& viewport;
     ViewportDragPosition offsetX, offsetY;
     Point<int> originalViewPos;
+    MouseInputSource scrollSource = Desktop::getInstance().getMainMouseSource();
     bool isDragging = false;
     bool isGlobalMouseListener = false;