diff --git a/profiles/audio/bap.c b/profiles/audio/bap.c
index ec3502b..b07d65e 100644
--- a/profiles/audio/bap.c
+++ b/profiles/audio/bap.c
@@ -1922,7 +1922,10 @@ static bool pac_select(struct bt_bap_pac *lpac, struct bt_bap_pac *rpac,
 	bt_bap_select(data->bap, lpac, rpac, 0, &select->remaining,
 								select_cb, ep);
 
-	return true;
+	/* For initial configuration consider only one endpoint (for each
+	 * direction).
+	 */
+	return select->reconfigure;
 }
 
 static int bap_select_all(struct bap_data *data, bool reconfigure,