diff --git a/src/shared/bap.c b/src/shared/bap.c
index db7def7..4ba65cb 100644
--- a/src/shared/bap.c
+++ b/src/shared/bap.c
@@ -2478,8 +2478,10 @@ static void remove_streams(void *data, void *user_data)
 	struct bt_bap_stream *stream;
 
 	stream = queue_remove_if(bap->streams, match_stream_lpac, pac);
-	if (stream)
+	if (stream) {
 		bt_bap_stream_release(stream, NULL, NULL);
+		stream_set_state(stream, BT_BAP_STREAM_STATE_IDLE);
+	}
 }
 
 bool bt_bap_remove_pac(struct bt_bap_pac *pac)