fix closing listening socket blocked

with go v1.11, closing a listening socket is blocked and unittests
fail due to timeout.

Signed-off-by: FUJITA Tomonori <fujita.tomonori@lab.ntt.co.jp>

7 files changed, 146 insertions, 163 deletions

diff --git a/pkg/server/server.go b/pkg/server/server.go
index 8f471d11..20f5d609 100644
--- a/pkg/server/server.go
+++ b/pkg/server/server.go
@@ -45,11 +45,8 @@ func (l *TCPListener) Close() error {
 	if err := l.l.Close(); err != nil {
 		return err
 	}
-	t := time.NewTicker(time.Second)
 	select {
 	case <-l.ch:
-	case <-t.C:
-		return fmt.Errorf("close timeout")
 	}
 	return nil
 }
diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go
index e56c6302..f1809e26 100644
--- a/pkg/server/server_test.go
+++ b/pkg/server/server_test.go
@@ -115,6 +115,7 @@ func TestListPolicyAssignment(t *testing.T) {
 		},
 	})
 	assert.Nil(err)
+	defer s.StopBgp(context.Background(), &api.StopBgpRequest{})
 
 	for i := 1; i < 4; i++ {
 		addr := fmt.Sprintf("127.0.0.%d", i)
@@ -211,6 +212,11 @@ func TestMonitor(test *testing.T) {
 				RemotePort: 10179,
 			},
 		},
+		Timers: config.Timers{
+			Config: config.TimersConfig{
+				ConnectRetry: 10,
+			},
+		},
 	}
 	err = t.AddPeer(context.Background(), &api.AddPeerRequest{Peer: NewPeerFromConfigStruct(m)})
 	assert.Nil(err)
@@ -344,6 +350,11 @@ func TestNumGoroutineWithAddDeleteNeighbor(t *testing.T) {
 	// wait goroutines to finish (e.g. internal goroutine for
 	// InfiniteChannel)
 	time.Sleep(time.Second * 5)
+	for i := 0; i < 5; i++ {
+		if num == runtime.NumGoroutine() {
+			return
+		}
+	}
 	assert.Equal(num, runtime.NumGoroutine())
 }
 
@@ -580,6 +591,11 @@ func TestPeerGroup(test *testing.T) {
 				RemotePort: 10179,
 			},
 		},
+		Timers: config.Timers{
+			Config: config.TimersConfig{
+				ConnectRetry: 10,
+			},
+		},
 	}
 	err = t.AddPeer(context.Background(), &api.AddPeerRequest{Peer: NewPeerFromConfigStruct(m)})
 	assert.Nil(err)
@@ -647,6 +663,11 @@ func TestDynamicNeighbor(t *testing.T) {
 				RemotePort: 10179,
 			},
 		},
+		Timers: config.Timers{
+			Config: config.TimersConfig{
+				ConnectRetry: 10,
+			},
+		},
 	}
 	err = s2.AddPeer(context.Background(), &api.AddPeerRequest{Peer: NewPeerFromConfigStruct(m)})
 
@@ -722,6 +743,11 @@ func TestGracefulRestartTimerExpired(t *testing.T) {
 				RestartTime: 1,
 			},
 		},
+		Timers: config.Timers{
+			Config: config.TimersConfig{
+				ConnectRetry: 10,
+			},
+		},
 	}
 	err = s2.addNeighbor(m)
 	assert.Nil(err)
diff --git a/pkg/server/sockopt_bsd.go b/pkg/server/sockopt_bsd.go
index 651e4e58..488a203e 100644
--- a/pkg/server/sockopt_bsd.go
+++ b/pkg/server/sockopt_bsd.go
@@ -18,7 +18,6 @@ package server
 
 import (
 	"net"
-	"os"
 	"syscall"
 )
 
@@ -27,63 +26,44 @@ const (
 	IPV6_MINHOPCOUNT = 73   // Generalized TTL Security Mechanism (RFC5082)
 )
 
-func setsockoptTcpMD5Sig(fd int, address string, key string) error {
-	// always enable and assumes that the configuration is done by setkey()
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, TCP_MD5SIG, 1))
-}
-
 func setTcpMD5SigSockopt(l *net.TCPListener, address string, key string) error {
-	fi, _, err := extractFileAndFamilyFromTCPListener(l)
-	defer fi.Close()
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-	return setsockoptTcpMD5Sig(int(fi.Fd()), address, key)
-}
-
-func setsockoptIpTtl(fd int, family int, value int) error {
-	level := syscall.IPPROTO_IP
-	name := syscall.IP_TTL
-	if family == syscall.AF_INET6 {
-		level = syscall.IPPROTO_IPV6
-		name = syscall.IPV6_UNICAST_HOPS
-	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
+	// always enable and assumes that the configuration is done by setkey()
+	return setsockOptInt(sc, syscall.IPPROTO_TCP, TCP_MD5SIG, 1)
 }
 
 func setListenTcpTTLSockopt(l *net.TCPListener, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPListener(l)
-	defer fi.Close()
+	family := extractFamilyFromTCPListener(l)
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
 func setTcpTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
-	defer fi.Close()
+	family := extractFamilyFromTCPConn(conn)
+	sc, err := conn.SyscallConn()
 	if err != nil {
 		return err
 	}
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
-func setsockoptIpMinTtl(fd int, family int, value int) error {
+func setTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
+	family := extractFamilyFromTCPConn(conn)
+	sc, err := conn.SyscallConn()
+	if err != nil {
+		return err
+	}
 	level := syscall.IPPROTO_IP
 	name := syscall.IP_MINTTL
 	if family == syscall.AF_INET6 {
 		level = syscall.IPPROTO_IPV6
 		name = IPV6_MINHOPCOUNT
 	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
-}
-
-func setTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
-	defer fi.Close()
-	if err != nil {
-		return err
-	}
-	return setsockoptIpMinTtl(int(fi.Fd()), family, ttl)
+	return setsockOptInt(sc, level, name, ttl)
 }
diff --git a/pkg/server/sockopt_darwin.go b/pkg/server/sockopt_darwin.go
index 4bad54ff..abb69d35 100644
--- a/pkg/server/sockopt_darwin.go
+++ b/pkg/server/sockopt_darwin.go
@@ -19,7 +19,7 @@ package server
 import (
 	"fmt"
 	"net"
-	"os"
+	"strings"
 	"syscall"
 )
 
@@ -27,36 +27,25 @@ func setTcpMD5SigSockopt(l *net.TCPListener, address string, key string) error {
 	return fmt.Errorf("setting md5 is not supported")
 }
 
-func setsockoptIpTtl(fd int, family int, value int) error {
-	level := syscall.IPPROTO_IP
-	name := syscall.IP_TTL
-	if family == syscall.AF_INET6 {
-		level = syscall.IPPROTO_IPV6
-		name = syscall.IPV6_UNICAST_HOPS
-	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
-}
-
 func setListenTcpTTLSockopt(l *net.TCPListener, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPListener(l)
+	family := extractFamilyFromTCPListener(l)
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-
-	defer fi.Close()
-
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
 func setTcpTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
+	family := syscall.AF_INET
+	if strings.Contains(conn.RemoteAddr().String(), "[") {
+		family = syscall.AF_INET6
+	}
+	sc, err := conn.SyscallConn()
 	if err != nil {
 		return err
 	}
-
-	defer fi.Close()
-
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
 func setTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
diff --git a/pkg/server/sockopt_linux.go b/pkg/server/sockopt_linux.go
index 9fe02ba5..152e97de 100644
--- a/pkg/server/sockopt_linux.go
+++ b/pkg/server/sockopt_linux.go
@@ -57,73 +57,80 @@ func buildTcpMD5Sig(address string, key string) (tcpmd5sig, error) {
 	return t, nil
 }
 
-func setsockoptTcpMD5Sig(fd int, address string, key string) error {
+func SetTcpMD5SigSockopt(l *net.TCPListener, address string, key string) error {
 	t, err := buildTcpMD5Sig(address, key)
 	if err != nil {
 		return err
 	}
 	b := *(*[unsafe.Sizeof(t)]byte)(unsafe.Pointer(&t))
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptString(fd, syscall.IPPROTO_TCP, TCP_MD5SIG, string(b[:])))
-}
 
-func SetTcpMD5SigSockopt(l *net.TCPListener, address string, key string) error {
-	fi, _, err := extractFileAndFamilyFromTCPListener(l)
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-	defer fi.Close()
-
-	return setsockoptTcpMD5Sig(int(fi.Fd()), address, key)
-}
-
-func setsockoptIpTtl(fd int, family int, value int) error {
-	level := syscall.IPPROTO_IP
-	name := syscall.IP_TTL
-	if family == syscall.AF_INET6 {
-		level = syscall.IPPROTO_IPV6
-		name = syscall.IPV6_UNICAST_HOPS
-	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
+	return setsockOptString(sc, syscall.IPPROTO_TCP, TCP_MD5SIG, string(b[:]))
 }
 
 func SetListenTcpTTLSockopt(l *net.TCPListener, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPListener(l)
+	family := extractFamilyFromTCPListener(l)
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-	defer fi.Close()
-
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
 func SetTcpTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
+	family := extractFamilyFromTCPConn(conn)
+	sc, err := conn.SyscallConn()
 	if err != nil {
 		return err
 	}
-	defer fi.Close()
-
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
-func setsockoptIpMinTtl(fd int, family int, value int) error {
+func SetTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
+	family := extractFamilyFromTCPConn(conn)
+	sc, err := conn.SyscallConn()
+	if err != nil {
+		return err
+	}
 	level := syscall.IPPROTO_IP
 	name := syscall.IP_MINTTL
 	if family == syscall.AF_INET6 {
 		level = syscall.IPPROTO_IPV6
 		name = IPV6_MINHOPCOUNT
 	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
+	return setsockOptInt(sc, level, name, ttl)
 }
 
-func SetTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
+func setsockoptTcpMD5Sig(fd int, address string, key string) error {
+	t, err := buildTcpMD5Sig(address, key)
 	if err != nil {
 		return err
 	}
-	defer fi.Close()
+	b := *(*[unsafe.Sizeof(t)]byte)(unsafe.Pointer(&t))
+	return os.NewSyscallError("setsockopt", syscall.SetsockoptString(fd, syscall.IPPROTO_TCP, TCP_MD5SIG, string(b[:])))
+}
 
-	return setsockoptIpMinTtl(int(fi.Fd()), family, ttl)
+func setsockoptIpTtl2(fd int, family int, value int) error {
+	level := syscall.IPPROTO_IP
+	name := syscall.IP_TTL
+	if family == syscall.AF_INET6 {
+		level = syscall.IPPROTO_IPV6
+		name = syscall.IPV6_UNICAST_HOPS
+	}
+	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
+}
+
+func setsockoptIpMinTtl(fd int, family int, value int) error {
+	level := syscall.IPPROTO_IP
+	name := syscall.IP_MINTTL
+	if family == syscall.AF_INET6 {
+		level = syscall.IPPROTO_IPV6
+		name = IPV6_MINHOPCOUNT
+	}
+	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
 }
 
 type TCPDialer struct {
@@ -208,7 +215,7 @@ func (d *TCPDialer) DialTCP(addr string, port int) (*net.TCPConn, error) {
 	}
 
 	if d.Ttl != 0 {
-		if err = setsockoptIpTtl(fd, family, int(d.Ttl)); err != nil {
+		if err = setsockoptIpTtl2(fd, family, int(d.Ttl)); err != nil {
 			return nil, err
 		}
 	}
diff --git a/pkg/server/sockopt_openbsd.go b/pkg/server/sockopt_openbsd.go
index f52c1447..4b7dc816 100644
--- a/pkg/server/sockopt_openbsd.go
+++ b/pkg/server/sockopt_openbsd.go
@@ -352,9 +352,9 @@ const (
 	IPV6_MINHOPCOUNT = 73  // Generalized TTL Security Mechanism (RFC5082)
 )
 
-func setsockoptTcpMD5Sig(fd int, address string, key string) error {
-	if err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, TCP_MD5SIG, 1); err != nil {
-		return os.NewSyscallError("setsockopt", err)
+func setsockoptTcpMD5Sig(sc syscall.RawConn, address string, key string) error {
+	if err := setsockOptInt(sc, syscall.IPPROTO_TCP, TCP_MD5SIG, 1); err != nil {
+		return err
 	}
 	if len(key) > 0 {
 		return saAdd(address, key)
@@ -363,59 +363,44 @@ func setsockoptTcpMD5Sig(fd int, address string, key string) error {
 }
 
 func SetTcpMD5SigSockopt(l *net.TCPListener, address string, key string) error {
-	fi, _, err := extractFileAndFamilyFromTCPListener(l)
-	defer fi.Close()
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-	return setsockoptTcpMD5Sig(int(fi.Fd()), address, key)
-}
-
-func setsockoptIpTtl(fd int, family int, value int) error {
-	level := syscall.IPPROTO_IP
-	name := syscall.IP_TTL
-	if family == syscall.AF_INET6 {
-		level = syscall.IPPROTO_IPV6
-		name = syscall.IPV6_UNICAST_HOPS
-	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
+	return setsockoptTcpMD5Sig(sc, address, key)
 }
 
 func SetListenTcpTTLSockopt(l *net.TCPListener, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPListener(l)
-	defer fi.Close()
+	family := extractFamilyFromTCPListener(l)
+	sc, err := l.SyscallConn()
 	if err != nil {
 		return err
 	}
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
 func SetTcpTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
-	defer fi.Close()
+	family := extractFamilyFromTCPConn(conn)
+	sc, err := conn.SyscallConn()
 	if err != nil {
 		return err
 	}
-	return setsockoptIpTtl(int(fi.Fd()), family, ttl)
+	return setsockoptIpTtl(sc, family, ttl)
 }
 
-func setsockoptIpMinTtl(fd int, family int, value int) error {
+func SetTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
+	family := extractFamilyFromTCPConn(conn)
+	sc, err := conn.SyscallConn()
+	if err != nil {
+		return err
+	}
 	level := syscall.IPPROTO_IP
 	name := syscall.IP_MINTTL
 	if family == syscall.AF_INET6 {
 		level = syscall.IPPROTO_IPV6
 		name = IPV6_MINHOPCOUNT
 	}
-	return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(fd, level, name, value))
-}
-
-func SetTcpMinTTLSockopt(conn *net.TCPConn, ttl int) error {
-	fi, family, err := extractFileAndFamilyFromTCPConn(conn)
-	defer fi.Close()
-	if err != nil {
-		return err
-	}
-	return setsockoptIpMinTtl(int(fi.Fd()), family, ttl)
+	return setsockOptInt(sc, level, name, ttl)
 }
 
 type TCPDialer struct {
diff --git a/pkg/server/util.go b/pkg/server/util.go
index ba6fe70a..c774c6dd 100644
--- a/pkg/server/util.go
+++ b/pkg/server/util.go
@@ -17,7 +17,6 @@ package server
 
 import (
 	"net"
-	"os"
 	"strings"
 	"syscall"
 
@@ -66,52 +65,52 @@ func decodeAdministrativeCommunication(data []byte) (string, []byte) {
 	return string(data[1 : communicationLen+1]), data[communicationLen+1:]
 }
 
-func extractFileAndFamilyFromTCPListener(l *net.TCPListener) (*os.File, int, error) {
-	// Note #1: TCPListener.File() has the unexpected side-effect of putting
-	// the original socket into blocking mode. See Note #2.
-	fi, err := l.File()
-	if err != nil {
-		return nil, 0, err
-	}
-
-	// Note #2: Call net.FileListener() to put the original socket back into
-	// non-blocking mode.
-	fl, err := net.FileListener(fi)
-	if err != nil {
-		fi.Close()
-		return nil, 0, err
-	}
-	fl.Close()
-
+func extractFamilyFromTCPListener(l *net.TCPListener) int {
 	family := syscall.AF_INET
 	if strings.Contains(l.Addr().String(), "[") {
 		family = syscall.AF_INET6
 	}
-
-	return fi, family, nil
+	return family
 }
 
-func extractFileAndFamilyFromTCPConn(conn *net.TCPConn) (*os.File, int, error) {
-	// Note #1: TCPConn.File() has the unexpected side-effect of putting
-	// the original socket into blocking mode. See Note #2.
-	fi, err := conn.File()
-	if err != nil {
-		return nil, 0, err
+func extractFamilyFromTCPConn(conn *net.TCPConn) int {
+	family := syscall.AF_INET
+	if strings.Contains(conn.RemoteAddr().String(), "[") {
+		family = syscall.AF_INET6
 	}
+	return family
+}
 
-	// Note #2: Call net.FileConn() to put the original socket back into
-	// non-blocking mode.
-	fc, err := net.FileConn(fi)
-	if err != nil {
-		fi.Close()
-		return nil, 0, err
+func setsockOptString(sc syscall.RawConn, level int, opt int, str string) error {
+	var opterr error
+	fn := func(s uintptr) {
+		opterr = syscall.SetsockoptString(int(s), level, opt, str)
 	}
-	fc.Close()
+	err := sc.Control(fn)
+	if opterr == nil {
+		return err
+	}
+	return opterr
+}
 
-	family := syscall.AF_INET
-	if strings.Contains(conn.RemoteAddr().String(), "[") {
-		family = syscall.AF_INET6
+func setsockOptInt(sc syscall.RawConn, level, name, value int) error {
+	var opterr error
+	fn := func(s uintptr) {
+		opterr = syscall.SetsockoptInt(int(s), level, name, value)
+	}
+	err := sc.Control(fn)
+	if opterr == nil {
+		return err
 	}
+	return opterr
+}
 
-	return fi, family, nil
+func setsockoptIpTtl(sc syscall.RawConn, family int, value int) error {
+	level := syscall.IPPROTO_IP
+	name := syscall.IP_TTL
+	if family == syscall.AF_INET6 {
+		level = syscall.IPPROTO_IPV6
+		name = syscall.IPV6_UNICAST_HOPS
+	}
+	return setsockOptInt(sc, level, name, value)
 }