diff --git a/audio/manager.c b/audio/manager.c
index 170ed23..aa2547c 100644
--- a/audio/manager.c
+++ b/audio/manager.c
@@ -916,14 +916,14 @@ static int headset_server_probe(struct btd_adapter *adapter)
 	if (!adp)
 		return -EINVAL;
 
-	btd_adapter_register_powered_callback(adapter, state_changed);
-
 	err = headset_server_init(adp);
 	if (err < 0) {
 		audio_adapter_unref(adp);
 		return err;
 	}
 
+	btd_adapter_register_powered_callback(adapter, state_changed);
+
 	return 0;
 }
 
@@ -934,6 +934,8 @@ static void headset_server_remove(struct btd_adapter *adapter)
 
 	DBG("path %s", path);
 
+	btd_adapter_unregister_powered_callback(adapter, state_changed);
+
 	adp = find_adapter(adapters, adapter);
 	if (!adp)
 		return;
@@ -960,8 +962,6 @@ static void headset_server_remove(struct btd_adapter *adapter)
 		adp->hfp_ag_server = NULL;
 	}
 
-	btd_adapter_unregister_powered_callback(adapter, state_changed);
-
 	audio_adapter_unref(adp);
 }