diff --git a/test/l2test.c b/test/l2test.c
index 438d21c..5034775 100644
--- a/test/l2test.c
+++ b/test/l2test.c
@@ -469,13 +469,6 @@ static void do_listen(void (*handler)(int sk))
 	}
 
 
-	if (priority > 0 && setsockopt(sk, SOL_SOCKET, SO_PRIORITY, &priority,
-						sizeof(priority)) < 0) {
-		syslog(LOG_ERR, "Can't set socket priority: %s (%d)",
-							strerror(errno), errno);
-		goto error;
-	}
-
 	/* Listen for connections */
 	if (listen(sk, 10)) {
 		syslog(LOG_ERR, "Can not listen on the socket: %s (%d)",
@@ -542,6 +535,14 @@ static void do_listen(void (*handler)(int sk))
 			}
 		}
 
+		if (priority > 0 && setsockopt(sk, SOL_SOCKET, SO_PRIORITY,
+					&priority, sizeof(priority)) < 0) {
+			syslog(LOG_ERR, "Can't set socket priority: %s (%d)",
+						strerror(errno), errno);
+			close(nsk);
+			goto error;
+		}
+
 		optlen = sizeof(priority);
 		if (getsockopt(nsk, SOL_SOCKET, SO_PRIORITY, &opt, &optlen) < 0) {
 			syslog(LOG_ERR, "Can't get socket priority: %s (%d)",