diff --git a/plugins/dbusoob.c b/plugins/dbusoob.c
index e58b353..7d9a858 100644
--- a/plugins/dbusoob.c
+++ b/plugins/dbusoob.c
@@ -346,8 +346,6 @@ static void dbusoob_exit(void)
 {
 	DBG("Cleanup dbusoob plugin");
 
-	manager_foreach_adapter((adapter_cb) oob_remove, NULL);
-
 	btd_unregister_adapter_driver(&oob_driver);
 }
 
diff --git a/src/adapter.c b/src/adapter.c
index 0d1dfea..163360f 100644
--- a/src/adapter.c
+++ b/src/adapter.c
@@ -3291,6 +3291,11 @@ int btd_register_adapter_driver(struct btd_adapter_driver *driver)
 
 static void unload_driver(struct btd_adapter *adapter, gpointer data)
 {
+	struct btd_adapter_driver *driver = data;
+
+	if (driver->remove)
+		driver->remove(adapter);
+
 	adapter->drivers = g_slist_remove(adapter->drivers, data);
 }