diff --git a/modules/juce_audio_basics/buffers/juce_AudioSampleBuffer.cpp b/modules/juce_audio_basics/buffers/juce_AudioSampleBuffer.cpp
index 15b59dafce..8fa04655a1 100644
--- a/modules/juce_audio_basics/buffers/juce_AudioSampleBuffer.cpp
+++ b/modules/juce_audio_basics/buffers/juce_AudioSampleBuffer.cpp
@@ -116,8 +116,11 @@ void AudioSampleBuffer::setDataToReferTo (float** dataToReferTo,
     jassert (dataToReferTo != nullptr);
     jassert (newNumChannels >= 0 && newNumSamples >= 0);
 
-    allocatedBytes = 0;
-    allocatedData.free();
+    if (allocatedBytes != 0)
+    {
+        allocatedBytes = 0;
+        allocatedData.free();
+    }
 
     numChannels = newNumChannels;
     size = newNumSamples;