diff --git a/swarm/src/behaviour/external_addresses.rs b/swarm/src/behaviour/external_addresses.rs
index 0ce07962..d914dd30 100644
--- a/swarm/src/behaviour/external_addresses.rs
+++ b/swarm/src/behaviour/external_addresses.rs
@@ -42,7 +42,7 @@ impl ExternalAddresses {
                 }
             }
             FromSwarm::ExpiredExternalAddr(ExpiredExternalAddr { addr, .. }) => {
-                self.addresses.insert((*addr).clone());
+                self.addresses.remove(addr);
             }
             _ => {}
         }
diff --git a/swarm/src/behaviour/listen_addresses.rs b/swarm/src/behaviour/listen_addresses.rs
index 37a7d05d..0259ebd0 100644
--- a/swarm/src/behaviour/listen_addresses.rs
+++ b/swarm/src/behaviour/listen_addresses.rs
@@ -25,7 +25,7 @@ impl ListenAddresses {
                 self.addresses.insert((*addr).clone());
             }
             FromSwarm::ExpiredListenAddr(ExpiredListenAddr { addr, .. }) => {
-                self.addresses.insert((*addr).clone());
+                self.addresses.remove(addr);
             }
             _ => {}
         }