diff --git a/src/newconf.c b/src/newconf.c
index 62bd8d79..53c3067b 100644
--- a/src/newconf.c
+++ b/src/newconf.c
@@ -865,9 +865,9 @@ conf_set_listen_port_both(void *data, int ssl)
 		}
                 if(listener_address == NULL)
                 {
-			add_listener(args->v.number, listener_address, AF_INET, ssl, yy_defer_accept);
+			add_listener(args->v.number, listener_address, AF_INET, ssl, ssl || yy_defer_accept);
 #ifdef RB_IPV6
-			add_listener(args->v.number, listener_address, AF_INET6, ssl, yy_defer_accept);
+			add_listener(args->v.number, listener_address, AF_INET6, ssl, ssl || yy_defer_accept);
 #endif
                 }
 		else
@@ -880,7 +880,7 @@ conf_set_listen_port_both(void *data, int ssl)
 #endif
 				family = AF_INET;
 
-			add_listener(args->v.number, listener_address, family, ssl, yy_defer_accept);
+			add_listener(args->v.number, listener_address, family, ssl, ssl || yy_defer_accept);
 
                 }