diff --git a/muxers/mplex/src/lib.rs b/muxers/mplex/src/lib.rs
index d83234b8..708f85f6 100644
--- a/muxers/mplex/src/lib.rs
+++ b/muxers/mplex/src/lib.rs
@@ -506,7 +506,9 @@ where C: AsyncRead + AsyncWrite
 
     fn destroy_substream(&self, mut substream: Self::Substream) {
         let _ = self.shutdown_substream(&mut substream);        // TODO: this doesn't necessarily send the close message
-        self.inner.lock().buffer.retain(|elem| elem.substream_id() != substream.num);
+        self.inner.lock().buffer.retain(|elem| {
+            elem.substream_id() != substream.num || elem.endpoint() == Some(substream.endpoint)
+        })
     }
 }