Support of address reuse in server reload (#6)

* Support of address reuse in server reload

Update server reload logic to support multiple servers having the same address

Signed-off-by: Gleb Kogtev <[email protected]>

* Fix issue with multiple servers with different addresses

Signed-off-by: Gleb Kogtev <[email protected]>

---------

Signed-off-by: Gleb Kogtev <[email protected]>

Author: glebkin

caddy.go

@@ -1,5 +1,7 @@
 // Copyright 2015 Light Code Labs, LLC
 //
+// Copyright 2024 MWS
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
@@ -218,22 +220,22 @@ func (i *Instance) Restart(newCaddyfile Input) (*Instance, error) {
 	}
 
 	// Add file descriptors of all the sockets that are capable of it
-	restartFds := make(map[string]restartTriple)
+	restartFds := make(map[string][]restartTriple)
 	for _, s := range i.servers {
 		gs, srvOk := s.server.(GracefulServer)
 		ln, lnOk := s.listener.(Listener)
 		pc, pcOk := s.packet.(PacketConn)
 		if srvOk {
 			if lnOk && pcOk {
-				restartFds[gs.Address()] = restartTriple{server: gs, listener: ln, packet: pc}
+				restartFds[gs.Address()] = append(restartFds[gs.Address()], restartTriple{server: gs, listener: ln, packet: pc})
 				continue
 			}
 			if lnOk {
-				restartFds[gs.Address()] = restartTriple{server: gs, listener: ln}
+				restartFds[gs.Address()] = append(restartFds[gs.Address()], restartTriple{server: gs, listener: ln})
 				continue
 			}
 			if pcOk {
-				restartFds[gs.Address()] = restartTriple{server: gs, packet: pc}
+				restartFds[gs.Address()] = append(restartFds[gs.Address()], restartTriple{server: gs, packet: pc})
 				continue
 			}
 		}
@@ -484,7 +486,7 @@ func Start(cdyfile Input) (*Instance, error) {
 	return inst, nil
 }
 
-func startWithListenerFds(cdyfile Input, inst *Instance, restartFds map[string]restartTriple) error {
+func startWithListenerFds(cdyfile Input, inst *Instance, restartFds map[string][]restartTriple) error {
 	// save this instance in the list now so that
 	// plugins can access it if need be, for example
 	// the caddytls package, so it can perform cert
@@ -684,7 +686,7 @@ func executeDirectives(inst *Instance, filename string,
 	return nil
 }
 
-func startServers(serverList []Server, inst *Instance, restartFds map[string]restartTriple) error {
+func startServers(serverList []Server, inst *Instance, restartFds map[string][]restartTriple) error {
 	errChan := make(chan error, len(serverList))
 
 	// used for signaling to error logging goroutine to terminate
@@ -734,7 +736,16 @@ func startServers(serverList []Server, inst *Instance, restartFds map[string]res
 		// reuse the listener for a graceful restart.
 		if gs, ok := s.(GracefulServer); ok && restartFds != nil {
 			addr := gs.Address()
-			if old, ok := restartFds[addr]; ok {
+			// Multiple servers may use the same addr (SO_REUSEPORT option set), so it's important to ensure
+			// that we don't reuse the same listener/packetconn.
+			// We'll create new listeners in case there are no more available triples for the same address.
+			if triples, ok := restartFds[addr]; ok && len(triples) > 0 {
+				// Take first available triple
+				old := triples[0]
+				// Remove reused triple from restartFds
+				triples[0] = triples[len(triples)-1]
+				restartFds[addr] = triples[:len(triples)-1]
+
 				// listener
 				if old.listener != nil {
 					file, err := old.listener.File()