Merge release-20201208.0-89-g3ff7324df (automated)

6 files changed, 211 insertions, 166 deletions

diff --git a/pkg/tcpip/transport/tcp/accept.go b/pkg/tcpip/transport/tcp/accept.go
index 3e1041cbe..2d96a65bd 100644
--- a/pkg/tcpip/transport/tcp/accept.go
+++ b/pkg/tcpip/transport/tcp/accept.go
@@ -778,7 +778,7 @@ func (e *endpoint) protocolListenLoop(rcvWnd seqnum.Size) {
 		e.waiterQueue.Notify(waiter.EventIn | waiter.EventOut | waiter.EventHUp | waiter.EventErr)
 	}()
 
-	s := sleep.Sleeper{}
+	var s sleep.Sleeper
 	s.AddWaker(&e.notificationWaker, wakerForNotification)
 	s.AddWaker(&e.newSegmentWaker, wakerForNewSegment)
 	for {
diff --git a/pkg/tcpip/transport/tcp/connect.go b/pkg/tcpip/transport/tcp/connect.go
index c944dccc0..0dc710276 100644
--- a/pkg/tcpip/transport/tcp/connect.go
+++ b/pkg/tcpip/transport/tcp/connect.go
@@ -462,7 +462,7 @@ func (h *handshake) processSegments() *tcpip.Error {
 
 func (h *handshake) resolveRoute() *tcpip.Error {
 	// Set up the wakers.
-	s := sleep.Sleeper{}
+	var s sleep.Sleeper
 	resolutionWaker := &sleep.Waker{}
 	s.AddWaker(resolutionWaker, wakerForResolution)
 	s.AddWaker(&h.ep.notificationWaker, wakerForNotification)
@@ -470,24 +470,27 @@ func (h *handshake) resolveRoute() *tcpip.Error {
 
 	// Initial action is to resolve route.
 	index := wakerForResolution
+	attemptedResolution := false
 	for {
 		switch index {
 		case wakerForResolution:
-			if _, err := h.ep.route.Resolve(resolutionWaker); err != tcpip.ErrWouldBlock {
-				if err == tcpip.ErrNoLinkAddress {
-					h.ep.stats.SendErrors.NoLinkAddr.Increment()
-				} else if err != nil {
+			if _, err := h.ep.route.Resolve(resolutionWaker.Assert); err != tcpip.ErrWouldBlock {
+				if err != nil {
 					h.ep.stats.SendErrors.NoRoute.Increment()
 				}
 				// Either success (err == nil) or failure.
 				return err
 			}
+			if attemptedResolution {
+				h.ep.stats.SendErrors.NoLinkAddr.Increment()
+				return tcpip.ErrNoLinkAddress
+			}
+			attemptedResolution = true
 			// Resolution not completed. Keep trying...
 
 		case wakerForNotification:
 			n := h.ep.fetchNotifications()
 			if n&notifyClose != 0 {
-				h.ep.route.RemoveWaker(resolutionWaker)
 				return tcpip.ErrAborted
 			}
 			if n&notifyDrain != 0 {
@@ -563,7 +566,7 @@ func (h *handshake) start() *tcpip.Error {
 // complete completes the TCP 3-way handshake initiated by h.start().
 func (h *handshake) complete() *tcpip.Error {
 	// Set up the wakers.
-	s := sleep.Sleeper{}
+	var s sleep.Sleeper
 	resendWaker := sleep.Waker{}
 	s.AddWaker(&resendWaker, wakerForResend)
 	s.AddWaker(&h.ep.notificationWaker, wakerForNotification)
@@ -1512,7 +1515,7 @@ func (e *endpoint) protocolMainLoop(handshake bool, wakerInitDone chan<- struct{
 	}
 
 	// Initialize the sleeper based on the wakers in funcs.
-	s := sleep.Sleeper{}
+	var s sleep.Sleeper
 	for i := range funcs {
 		s.AddWaker(funcs[i].w, i)
 	}
@@ -1699,7 +1702,7 @@ func (e *endpoint) doTimeWait() (twReuse func()) {
 	const notification = 2
 	const timeWaitDone = 3
 
-	s := sleep.Sleeper{}
+	var s sleep.Sleeper
 	defer s.Done()
 	s.AddWaker(&e.newSegmentWaker, newSegment)
 	s.AddWaker(&e.notificationWaker, notification)
diff --git a/pkg/tcpip/transport/tcp/endpoint.go b/pkg/tcpip/transport/tcp/endpoint.go
index 7a37c10bb..6e3c8860e 100644
--- a/pkg/tcpip/transport/tcp/endpoint.go
+++ b/pkg/tcpip/transport/tcp/endpoint.go
@@ -502,9 +502,6 @@ type endpoint struct {
 	// sack holds TCP SACK related information for this endpoint.
 	sack SACKInfo
 
-	// bindToDevice is set to the NIC on which to bind or disabled if 0.
-	bindToDevice tcpip.NICID
-
 	// delay enables Nagle's algorithm.
 	//
 	// delay is a boolean (0 is false) and must be accessed atomically.
@@ -1303,6 +1300,15 @@ func (e *endpoint) LastError() *tcpip.Error {
 	return e.lastErrorLocked()
 }
 
+// UpdateLastError implements tcpip.SocketOptionsHandler.UpdateLastError.
+func (e *endpoint) UpdateLastError(err *tcpip.Error) {
+	e.LockUser()
+	e.lastErrorMu.Lock()
+	e.lastError = err
+	e.lastErrorMu.Unlock()
+	e.UnlockUser()
+}
+
 // Read reads data from the endpoint.
 func (e *endpoint) Read(*tcpip.FullAddress) (buffer.View, tcpip.ControlMessages, *tcpip.Error) {
 	e.LockUser()
@@ -1812,18 +1818,13 @@ func (e *endpoint) SetSockOptInt(opt tcpip.SockOptInt, v int) *tcpip.Error {
 	return nil
 }
 
+func (e *endpoint) HasNIC(id int32) bool {
+	return id == 0 || e.stack.HasNIC(tcpip.NICID(id))
+}
+
 // SetSockOpt sets a socket option.
 func (e *endpoint) SetSockOpt(opt tcpip.SettableSocketOption) *tcpip.Error {
 	switch v := opt.(type) {
-	case *tcpip.BindToDeviceOption:
-		id := tcpip.NICID(*v)
-		if id != 0 && !e.stack.HasNIC(id) {
-			return tcpip.ErrUnknownDevice
-		}
-		e.LockUser()
-		e.bindToDevice = id
-		e.UnlockUser()
-
 	case *tcpip.KeepaliveIdleOption:
 		e.keepalive.Lock()
 		e.keepalive.idle = time.Duration(*v)
@@ -2004,11 +2005,6 @@ func (e *endpoint) GetSockOptInt(opt tcpip.SockOptInt) (int, *tcpip.Error) {
 // GetSockOpt implements tcpip.Endpoint.GetSockOpt.
 func (e *endpoint) GetSockOpt(opt tcpip.GettableSocketOption) *tcpip.Error {
 	switch o := opt.(type) {
-	case *tcpip.BindToDeviceOption:
-		e.LockUser()
-		*o = tcpip.BindToDeviceOption(e.bindToDevice)
-		e.UnlockUser()
-
 	case *tcpip.TCPInfoOption:
 		*o = tcpip.TCPInfoOption{}
 		e.LockUser()
@@ -2211,11 +2207,12 @@ func (e *endpoint) connect(addr tcpip.FullAddress, handshake bool, run bool) *tc
 			}
 		}
 
+		bindToDevice := tcpip.NICID(e.ops.GetBindToDevice())
 		if _, err := e.stack.PickEphemeralPortStable(portOffset, func(p uint16) (bool, *tcpip.Error) {
 			if sameAddr && p == e.ID.RemotePort {
 				return false, nil
 			}
-			if _, err := e.stack.ReservePort(netProtos, ProtocolNumber, e.ID.LocalAddress, p, e.portFlags, e.bindToDevice, addr, nil /* testPort */); err != nil {
+			if _, err := e.stack.ReservePort(netProtos, ProtocolNumber, e.ID.LocalAddress, p, e.portFlags, bindToDevice, addr, nil /* testPort */); err != nil {
 				if err != tcpip.ErrPortInUse || !reuse {
 					return false, nil
 				}
@@ -2253,15 +2250,15 @@ func (e *endpoint) connect(addr tcpip.FullAddress, handshake bool, run bool) *tc
 				tcpEP.notifyProtocolGoroutine(notifyAbort)
 				tcpEP.UnlockUser()
 				// Now try and Reserve again if it fails then we skip.
-				if _, err := e.stack.ReservePort(netProtos, ProtocolNumber, e.ID.LocalAddress, p, e.portFlags, e.bindToDevice, addr, nil /* testPort */); err != nil {
+				if _, err := e.stack.ReservePort(netProtos, ProtocolNumber, e.ID.LocalAddress, p, e.portFlags, bindToDevice, addr, nil /* testPort */); err != nil {
 					return false, nil
 				}
 			}
 
 			id := e.ID
 			id.LocalPort = p
-			if err := e.stack.RegisterTransportEndpoint(nicID, netProtos, ProtocolNumber, id, e, e.portFlags, e.bindToDevice); err != nil {
-				e.stack.ReleasePort(netProtos, ProtocolNumber, e.ID.LocalAddress, p, e.portFlags, e.bindToDevice, addr)
+			if err := e.stack.RegisterTransportEndpoint(nicID, netProtos, ProtocolNumber, id, e, e.portFlags, bindToDevice); err != nil {
+				e.stack.ReleasePort(netProtos, ProtocolNumber, e.ID.LocalAddress, p, e.portFlags, bindToDevice, addr)
 				if err == tcpip.ErrPortInUse {
 					return false, nil
 				}
@@ -2272,7 +2269,7 @@ func (e *endpoint) connect(addr tcpip.FullAddress, handshake bool, run bool) *tc
 			// the selected port.
 			e.ID = id
 			e.isPortReserved = true
-			e.boundBindToDevice = e.bindToDevice
+			e.boundBindToDevice = bindToDevice
 			e.boundPortFlags = e.portFlags
 			e.boundDest = addr
 			return true, nil
@@ -2283,7 +2280,8 @@ func (e *endpoint) connect(addr tcpip.FullAddress, handshake bool, run bool) *tc
 
 	e.isRegistered = true
 	e.setEndpointState(StateConnecting)
-	e.route = r.Clone()
+	r.Acquire()
+	e.route = r
 	e.boundNICID = nicID
 	e.effectiveNetProtos = netProtos
 	e.connectingAddress = connectingAddr
@@ -2624,7 +2622,8 @@ func (e *endpoint) bindLocked(addr tcpip.FullAddress) (err *tcpip.Error) {
 		e.ID.LocalAddress = addr.Addr
 	}
 
-	port, err := e.stack.ReservePort(netProtos, ProtocolNumber, addr.Addr, addr.Port, e.portFlags, e.bindToDevice, tcpip.FullAddress{}, func(p uint16) bool {
+	bindToDevice := tcpip.NICID(e.ops.GetBindToDevice())
+	port, err := e.stack.ReservePort(netProtos, ProtocolNumber, addr.Addr, addr.Port, e.portFlags, bindToDevice, tcpip.FullAddress{}, func(p uint16) bool {
 		id := e.ID
 		id.LocalPort = p
 		// CheckRegisterTransportEndpoint should only return an error if there is a
@@ -2635,7 +2634,7 @@ func (e *endpoint) bindLocked(addr tcpip.FullAddress) (err *tcpip.Error) {
 		// demuxer. Further connected endpoints always have a remote
 		// address/port. Hence this will only return an error if there is a matching
 		// listening endpoint.
-		if err := e.stack.CheckRegisterTransportEndpoint(nic, netProtos, ProtocolNumber, id, e.portFlags, e.bindToDevice); err != nil {
+		if err := e.stack.CheckRegisterTransportEndpoint(nic, netProtos, ProtocolNumber, id, e.portFlags, bindToDevice); err != nil {
 			return false
 		}
 		return true
@@ -2644,7 +2643,7 @@ func (e *endpoint) bindLocked(addr tcpip.FullAddress) (err *tcpip.Error) {
 		return err
 	}
 
-	e.boundBindToDevice = e.bindToDevice
+	e.boundBindToDevice = bindToDevice
 	e.boundPortFlags = e.portFlags
 	// TODO(gvisor.dev/issue/3691): Add test to verify boundNICID is correct.
 	e.boundNICID = nic
@@ -2708,6 +2707,41 @@ func (e *endpoint) enqueueSegment(s *segment) bool {
 	return true
 }
 
+func (e *endpoint) onICMPError(err *tcpip.Error, id stack.TransportEndpointID, errType byte, errCode byte, extra uint32, pkt *stack.PacketBuffer) {
+	// Update last error first.
+	e.lastErrorMu.Lock()
+	e.lastError = err
+	e.lastErrorMu.Unlock()
+
+	// Update the error queue if IP_RECVERR is enabled.
+	if e.SocketOptions().GetRecvError() {
+		e.SocketOptions().QueueErr(&tcpip.SockError{
+			Err:       err,
+			ErrOrigin: header.ICMPOriginFromNetProto(pkt.NetworkProtocolNumber),
+			ErrType:   errType,
+			ErrCode:   errCode,
+			ErrInfo:   extra,
+			// Linux passes the payload with the TCP header. We don't know if the TCP
+			// header even exists, it may not for fragmented packets.
+			Payload: pkt.Data.ToView(),
+			Dst: tcpip.FullAddress{
+				NIC:  pkt.NICID,
+				Addr: id.RemoteAddress,
+				Port: id.RemotePort,
+			},
+			Offender: tcpip.FullAddress{
+				NIC:  pkt.NICID,
+				Addr: id.LocalAddress,
+				Port: id.LocalPort,
+			},
+			NetProto: pkt.NetworkProtocolNumber,
+		})
+	}
+
+	// Notify of the error.
+	e.notifyProtocolGoroutine(notifyError)
+}
+
 // HandleControlPacket implements stack.TransportEndpoint.HandleControlPacket.
 func (e *endpoint) HandleControlPacket(id stack.TransportEndpointID, typ stack.ControlType, extra uint32, pkt *stack.PacketBuffer) {
 	switch typ {
@@ -2722,16 +2756,10 @@ func (e *endpoint) HandleControlPacket(id stack.TransportEndpointID, typ stack.C
 		e.notifyProtocolGoroutine(notifyMTUChanged)
 
 	case stack.ControlNoRoute:
-		e.lastErrorMu.Lock()
-		e.lastError = tcpip.ErrNoRoute
-		e.lastErrorMu.Unlock()
-		e.notifyProtocolGoroutine(notifyError)
+		e.onICMPError(tcpip.ErrNoRoute, id, byte(header.ICMPv4DstUnreachable), byte(header.ICMPv4HostUnreachable), extra, pkt)
 
 	case stack.ControlNetworkUnreachable:
-		e.lastErrorMu.Lock()
-		e.lastError = tcpip.ErrNetworkUnreachable
-		e.lastErrorMu.Unlock()
-		e.notifyProtocolGoroutine(notifyError)
+		e.onICMPError(tcpip.ErrNetworkUnreachable, id, byte(header.ICMPv6DstUnreachable), byte(header.ICMPv6NetworkUnreachable), extra, pkt)
 	}
 }
 
@@ -2989,6 +3017,7 @@ func (e *endpoint) completeState() stack.TCPEndpointState {
 		Ssthresh:         e.snd.sndSsthresh,
 		SndCAAckCount:    e.snd.sndCAAckCount,
 		Outstanding:      e.snd.outstanding,
+		SackedOut:        e.snd.sackedOut,
 		SndWnd:           e.snd.sndWnd,
 		SndUna:           e.snd.sndUna,
 		SndNxt:           e.snd.sndNxt,
diff --git a/pkg/tcpip/transport/tcp/rcv.go b/pkg/tcpip/transport/tcp/rcv.go
index f2b1b68da..405a6dce7 100644
--- a/pkg/tcpip/transport/tcp/rcv.go
+++ b/pkg/tcpip/transport/tcp/rcv.go
@@ -172,14 +172,12 @@ func (r *receiver) getSendParams() (rcvNxt seqnum.Value, rcvWnd seqnum.Size) {
 
 	// If we started off with a window larger than what can he held in
 	// the 16bit window field, we ceil the value to the max value.
-	// While ceiling, we still do not want to grow the right edge when
-	// not applicable.
 	if scaledWnd > math.MaxUint16 {
-		if toGrow {
-			scaledWnd = seqnum.Size(math.MaxUint16)
-		} else {
-			scaledWnd = seqnum.Size(uint16(scaledWnd))
-		}
+		scaledWnd = seqnum.Size(math.MaxUint16)
+
+		// Ensure that the stashed receive window always reflects what
+		// is being advertised.
+		r.rcvWnd = scaledWnd << r.rcvWndScale
 	}
 	return r.rcvNxt, scaledWnd
 }
diff --git a/pkg/tcpip/transport/tcp/snd.go b/pkg/tcpip/transport/tcp/snd.go
index baec762e1..cc991aba6 100644
--- a/pkg/tcpip/transport/tcp/snd.go
+++ b/pkg/tcpip/transport/tcp/snd.go
@@ -137,6 +137,9 @@ type sender struct {
 	// that have been sent but not yet acknowledged.
 	outstanding int
 
+	// sackedOut is the number of packets which are selectively acked.
+	sackedOut int
+
 	// sndWnd is the send window size.
 	sndWnd seqnum.Size
 
@@ -372,6 +375,7 @@ func (s *sender) updateMaxPayloadSize(mtu, count int) {
 		m = 1
 	}
 
+	oldMSS := s.maxPayloadSize
 	s.maxPayloadSize = m
 	if s.gso {
 		s.ep.gso.MSS = uint16(m)
@@ -394,6 +398,7 @@ func (s *sender) updateMaxPayloadSize(mtu, count int) {
 
 	// Rewind writeNext to the first segment exceeding the MTU. Do nothing
 	// if it is already before such a packet.
+	nextSeg := s.writeNext
 	for seg := s.writeList.Front(); seg != nil; seg = seg.Next() {
 		if seg == s.writeNext {
 			// We got to writeNext before we could find a segment
@@ -401,16 +406,22 @@ func (s *sender) updateMaxPayloadSize(mtu, count int) {
 			break
 		}
 
-		if seg.data.Size() > m {
+		if nextSeg == s.writeNext && seg.data.Size() > m {
 			// We found a segment exceeding the MTU. Rewind
 			// writeNext and try to retransmit it.
-			s.writeNext = seg
-			break
+			nextSeg = seg
+		}
+
+		if s.ep.sackPermitted && s.ep.scoreboard.IsSACKED(seg.sackBlock()) {
+			// Update sackedOut for new maximum payload size.
+			s.sackedOut -= s.pCount(seg, oldMSS)
+			s.sackedOut += s.pCount(seg, s.maxPayloadSize)
 		}
 	}
 
 	// Since we likely reduced the number of outstanding packets, we may be
 	// ready to send some more.
+	s.writeNext = nextSeg
 	s.sendData()
 }
 
@@ -629,13 +640,13 @@ func (s *sender) retransmitTimerExpired() bool {
 
 // pCount returns the number of packets in the segment. Due to GSO, a segment
 // can be composed of multiple packets.
-func (s *sender) pCount(seg *segment) int {
+func (s *sender) pCount(seg *segment, maxPayloadSize int) int {
 	size := seg.data.Size()
 	if size == 0 {
 		return 1
 	}
 
-	return (size-1)/s.maxPayloadSize + 1
+	return (size-1)/maxPayloadSize + 1
 }
 
 // splitSeg splits a given segment at the size specified and inserts the
@@ -1023,7 +1034,7 @@ func (s *sender) sendData() {
 			break
 		}
 		dataSent = true
-		s.outstanding += s.pCount(seg)
+		s.outstanding += s.pCount(seg, s.maxPayloadSize)
 		s.writeNext = seg.Next()
 	}
 
@@ -1038,6 +1049,7 @@ func (s *sender) enterRecovery() {
 	// We inflate the cwnd by 3 to account for the 3 packets which triggered
 	// the 3 duplicate ACKs and are now not in flight.
 	s.sndCwnd = s.sndSsthresh + 3
+	s.sackedOut = 0
 	s.fr.first = s.sndUna
 	s.fr.last = s.sndNxt - 1
 	s.fr.maxCwnd = s.sndCwnd + s.outstanding
@@ -1207,6 +1219,7 @@ func (s *sender) walkSACK(rcvdSeg *segment) {
 				s.rc.update(seg, rcvdSeg, s.ep.tsOffset)
 				s.rc.detectReorder(seg)
 				seg.acked = true
+				s.sackedOut += s.pCount(seg, s.maxPayloadSize)
 			}
 			seg = seg.Next()
 		}
@@ -1380,10 +1393,10 @@ func (s *sender) handleRcvdSegment(rcvdSeg *segment) {
 			datalen := seg.logicalLen()
 
 			if datalen > ackLeft {
-				prevCount := s.pCount(seg)
+				prevCount := s.pCount(seg, s.maxPayloadSize)
 				seg.data.TrimFront(int(ackLeft))
 				seg.sequenceNumber.UpdateForward(ackLeft)
-				s.outstanding -= prevCount - s.pCount(seg)
+				s.outstanding -= prevCount - s.pCount(seg, s.maxPayloadSize)
 				break
 			}
 
@@ -1399,11 +1412,13 @@ func (s *sender) handleRcvdSegment(rcvdSeg *segment) {
 
 			s.writeList.Remove(seg)
 
-			// If SACK is enabled then Only reduce outstanding if
+			// If SACK is enabled then only reduce outstanding if
 			// the segment was not previously SACKED as these have
 			// already been accounted for in SetPipe().
 			if !s.ep.sackPermitted || !s.ep.scoreboard.IsSACKED(seg.sackBlock()) {
-				s.outstanding -= s.pCount(seg)
+				s.outstanding -= s.pCount(seg, s.maxPayloadSize)
+			} else {
+				s.sackedOut -= s.pCount(seg, s.maxPayloadSize)
 			}
 			seg.decRef()
 			ackLeft -= datalen
diff --git a/pkg/tcpip/transport/tcp/tcp_state_autogen.go b/pkg/tcpip/transport/tcp/tcp_state_autogen.go
index 8eba0efeb..5922083a9 100644
--- a/pkg/tcpip/transport/tcp/tcp_state_autogen.go
+++ b/pkg/tcpip/transport/tcp/tcp_state_autogen.go
@@ -187,7 +187,6 @@ func (e *endpoint) StateFields() []string {
 		"shutdownFlags",
 		"sackPermitted",
 		"sack",
-		"bindToDevice",
 		"delay",
 		"scoreboard",
 		"segmentQueue",
@@ -232,7 +231,7 @@ func (e *endpoint) StateSave(stateSinkObject state.Sink) {
 	var recentTSTimeValue unixTime = e.saveRecentTSTime()
 	stateSinkObject.SaveValue(26, recentTSTimeValue)
 	var acceptedChanValue []*endpoint = e.saveAcceptedChan()
-	stateSinkObject.SaveValue(50, acceptedChanValue)
+	stateSinkObject.SaveValue(49, acceptedChanValue)
 	stateSinkObject.Save(0, &e.EndpointInfo)
 	stateSinkObject.Save(1, &e.DefaultSocketOptionsHandler)
 	stateSinkObject.Save(2, &e.waiterQueue)
@@ -260,36 +259,35 @@ func (e *endpoint) StateSave(stateSinkObject state.Sink) {
 	stateSinkObject.Save(28, &e.shutdownFlags)
 	stateSinkObject.Save(29, &e.sackPermitted)
 	stateSinkObject.Save(30, &e.sack)
-	stateSinkObject.Save(31, &e.bindToDevice)
-	stateSinkObject.Save(32, &e.delay)
-	stateSinkObject.Save(33, &e.scoreboard)
-	stateSinkObject.Save(34, &e.segmentQueue)
-	stateSinkObject.Save(35, &e.synRcvdCount)
-	stateSinkObject.Save(36, &e.userMSS)
-	stateSinkObject.Save(37, &e.maxSynRetries)
-	stateSinkObject.Save(38, &e.windowClamp)
-	stateSinkObject.Save(39, &e.sndBufSize)
-	stateSinkObject.Save(40, &e.sndBufUsed)
-	stateSinkObject.Save(41, &e.sndClosed)
-	stateSinkObject.Save(42, &e.sndBufInQueue)
-	stateSinkObject.Save(43, &e.sndQueue)
-	stateSinkObject.Save(44, &e.cc)
-	stateSinkObject.Save(45, &e.packetTooBigCount)
-	stateSinkObject.Save(46, &e.sndMTU)
-	stateSinkObject.Save(47, &e.keepalive)
-	stateSinkObject.Save(48, &e.userTimeout)
-	stateSinkObject.Save(49, &e.deferAccept)
-	stateSinkObject.Save(51, &e.rcv)
-	stateSinkObject.Save(52, &e.snd)
-	stateSinkObject.Save(53, &e.connectingAddress)
-	stateSinkObject.Save(54, &e.amss)
-	stateSinkObject.Save(55, &e.sendTOS)
-	stateSinkObject.Save(56, &e.gso)
-	stateSinkObject.Save(57, &e.tcpLingerTimeout)
-	stateSinkObject.Save(58, &e.closed)
-	stateSinkObject.Save(59, &e.txHash)
-	stateSinkObject.Save(60, &e.owner)
-	stateSinkObject.Save(61, &e.ops)
+	stateSinkObject.Save(31, &e.delay)
+	stateSinkObject.Save(32, &e.scoreboard)
+	stateSinkObject.Save(33, &e.segmentQueue)
+	stateSinkObject.Save(34, &e.synRcvdCount)
+	stateSinkObject.Save(35, &e.userMSS)
+	stateSinkObject.Save(36, &e.maxSynRetries)
+	stateSinkObject.Save(37, &e.windowClamp)
+	stateSinkObject.Save(38, &e.sndBufSize)
+	stateSinkObject.Save(39, &e.sndBufUsed)
+	stateSinkObject.Save(40, &e.sndClosed)
+	stateSinkObject.Save(41, &e.sndBufInQueue)
+	stateSinkObject.Save(42, &e.sndQueue)
+	stateSinkObject.Save(43, &e.cc)
+	stateSinkObject.Save(44, &e.packetTooBigCount)
+	stateSinkObject.Save(45, &e.sndMTU)
+	stateSinkObject.Save(46, &e.keepalive)
+	stateSinkObject.Save(47, &e.userTimeout)
+	stateSinkObject.Save(48, &e.deferAccept)
+	stateSinkObject.Save(50, &e.rcv)
+	stateSinkObject.Save(51, &e.snd)
+	stateSinkObject.Save(52, &e.connectingAddress)
+	stateSinkObject.Save(53, &e.amss)
+	stateSinkObject.Save(54, &e.sendTOS)
+	stateSinkObject.Save(55, &e.gso)
+	stateSinkObject.Save(56, &e.tcpLingerTimeout)
+	stateSinkObject.Save(57, &e.closed)
+	stateSinkObject.Save(58, &e.txHash)
+	stateSinkObject.Save(59, &e.owner)
+	stateSinkObject.Save(60, &e.ops)
 }
 
 func (e *endpoint) StateLoad(stateSourceObject state.Source) {
@@ -320,41 +318,40 @@ func (e *endpoint) StateLoad(stateSourceObject state.Source) {
 	stateSourceObject.Load(28, &e.shutdownFlags)
 	stateSourceObject.Load(29, &e.sackPermitted)
 	stateSourceObject.Load(30, &e.sack)
-	stateSourceObject.Load(31, &e.bindToDevice)
-	stateSourceObject.Load(32, &e.delay)
-	stateSourceObject.Load(33, &e.scoreboard)
-	stateSourceObject.LoadWait(34, &e.segmentQueue)
-	stateSourceObject.Load(35, &e.synRcvdCount)
-	stateSourceObject.Load(36, &e.userMSS)
-	stateSourceObject.Load(37, &e.maxSynRetries)
-	stateSourceObject.Load(38, &e.windowClamp)
-	stateSourceObject.Load(39, &e.sndBufSize)
-	stateSourceObject.Load(40, &e.sndBufUsed)
-	stateSourceObject.Load(41, &e.sndClosed)
-	stateSourceObject.Load(42, &e.sndBufInQueue)
-	stateSourceObject.LoadWait(43, &e.sndQueue)
-	stateSourceObject.Load(44, &e.cc)
-	stateSourceObject.Load(45, &e.packetTooBigCount)
-	stateSourceObject.Load(46, &e.sndMTU)
-	stateSourceObject.Load(47, &e.keepalive)
-	stateSourceObject.Load(48, &e.userTimeout)
-	stateSourceObject.Load(49, &e.deferAccept)
-	stateSourceObject.LoadWait(51, &e.rcv)
-	stateSourceObject.LoadWait(52, &e.snd)
-	stateSourceObject.Load(53, &e.connectingAddress)
-	stateSourceObject.Load(54, &e.amss)
-	stateSourceObject.Load(55, &e.sendTOS)
-	stateSourceObject.Load(56, &e.gso)
-	stateSourceObject.Load(57, &e.tcpLingerTimeout)
-	stateSourceObject.Load(58, &e.closed)
-	stateSourceObject.Load(59, &e.txHash)
-	stateSourceObject.Load(60, &e.owner)
-	stateSourceObject.Load(61, &e.ops)
+	stateSourceObject.Load(31, &e.delay)
+	stateSourceObject.Load(32, &e.scoreboard)
+	stateSourceObject.LoadWait(33, &e.segmentQueue)
+	stateSourceObject.Load(34, &e.synRcvdCount)
+	stateSourceObject.Load(35, &e.userMSS)
+	stateSourceObject.Load(36, &e.maxSynRetries)
+	stateSourceObject.Load(37, &e.windowClamp)
+	stateSourceObject.Load(38, &e.sndBufSize)
+	stateSourceObject.Load(39, &e.sndBufUsed)
+	stateSourceObject.Load(40, &e.sndClosed)
+	stateSourceObject.Load(41, &e.sndBufInQueue)
+	stateSourceObject.LoadWait(42, &e.sndQueue)
+	stateSourceObject.Load(43, &e.cc)
+	stateSourceObject.Load(44, &e.packetTooBigCount)
+	stateSourceObject.Load(45, &e.sndMTU)
+	stateSourceObject.Load(46, &e.keepalive)
+	stateSourceObject.Load(47, &e.userTimeout)
+	stateSourceObject.Load(48, &e.deferAccept)
+	stateSourceObject.LoadWait(50, &e.rcv)
+	stateSourceObject.LoadWait(51, &e.snd)
+	stateSourceObject.Load(52, &e.connectingAddress)
+	stateSourceObject.Load(53, &e.amss)
+	stateSourceObject.Load(54, &e.sendTOS)
+	stateSourceObject.Load(55, &e.gso)
+	stateSourceObject.Load(56, &e.tcpLingerTimeout)
+	stateSourceObject.Load(57, &e.closed)
+	stateSourceObject.Load(58, &e.txHash)
+	stateSourceObject.Load(59, &e.owner)
+	stateSourceObject.Load(60, &e.ops)
 	stateSourceObject.LoadValue(4, new(string), func(y interface{}) { e.loadHardError(y.(string)) })
 	stateSourceObject.LoadValue(5, new(string), func(y interface{}) { e.loadLastError(y.(string)) })
 	stateSourceObject.LoadValue(13, new(EndpointState), func(y interface{}) { e.loadState(y.(EndpointState)) })
 	stateSourceObject.LoadValue(26, new(unixTime), func(y interface{}) { e.loadRecentTSTime(y.(unixTime)) })
-	stateSourceObject.LoadValue(50, new([]*endpoint), func(y interface{}) { e.loadAcceptedChan(y.([]*endpoint)) })
+	stateSourceObject.LoadValue(49, new([]*endpoint), func(y interface{}) { e.loadAcceptedChan(y.([]*endpoint)) })
 	stateSourceObject.AfterLoad(e.afterLoad)
 }
 
@@ -724,6 +721,7 @@ func (s *sender) StateFields() []string {
 		"sndSsthresh",
 		"sndCAAckCount",
 		"outstanding",
+		"sackedOut",
 		"sndWnd",
 		"sndUna",
 		"sndNxt",
@@ -755,9 +753,9 @@ func (s *sender) StateSave(stateSinkObject state.Sink) {
 	var lastSendTimeValue unixTime = s.saveLastSendTime()
 	stateSinkObject.SaveValue(1, lastSendTimeValue)
 	var rttMeasureTimeValue unixTime = s.saveRttMeasureTime()
-	stateSinkObject.SaveValue(13, rttMeasureTimeValue)
+	stateSinkObject.SaveValue(14, rttMeasureTimeValue)
 	var firstRetransmittedSegXmitTimeValue unixTime = s.saveFirstRetransmittedSegXmitTime()
-	stateSinkObject.SaveValue(14, firstRetransmittedSegXmitTimeValue)
+	stateSinkObject.SaveValue(15, firstRetransmittedSegXmitTimeValue)
 	stateSinkObject.Save(0, &s.ep)
 	stateSinkObject.Save(2, &s.dupAckCount)
 	stateSinkObject.Save(3, &s.fr)
@@ -766,25 +764,26 @@ func (s *sender) StateSave(stateSinkObject state.Sink) {
 	stateSinkObject.Save(6, &s.sndSsthresh)
 	stateSinkObject.Save(7, &s.sndCAAckCount)
 	stateSinkObject.Save(8, &s.outstanding)
-	stateSinkObject.Save(9, &s.sndWnd)
-	stateSinkObject.Save(10, &s.sndUna)
-	stateSinkObject.Save(11, &s.sndNxt)
-	stateSinkObject.Save(12, &s.rttMeasureSeqNum)
-	stateSinkObject.Save(15, &s.closed)
-	stateSinkObject.Save(16, &s.writeNext)
-	stateSinkObject.Save(17, &s.writeList)
-	stateSinkObject.Save(18, &s.rtt)
-	stateSinkObject.Save(19, &s.rto)
-	stateSinkObject.Save(20, &s.minRTO)
-	stateSinkObject.Save(21, &s.maxRTO)
-	stateSinkObject.Save(22, &s.maxRetries)
-	stateSinkObject.Save(23, &s.maxPayloadSize)
-	stateSinkObject.Save(24, &s.gso)
-	stateSinkObject.Save(25, &s.sndWndScale)
-	stateSinkObject.Save(26, &s.maxSentAck)
-	stateSinkObject.Save(27, &s.state)
-	stateSinkObject.Save(28, &s.cc)
-	stateSinkObject.Save(29, &s.rc)
+	stateSinkObject.Save(9, &s.sackedOut)
+	stateSinkObject.Save(10, &s.sndWnd)
+	stateSinkObject.Save(11, &s.sndUna)
+	stateSinkObject.Save(12, &s.sndNxt)
+	stateSinkObject.Save(13, &s.rttMeasureSeqNum)
+	stateSinkObject.Save(16, &s.closed)
+	stateSinkObject.Save(17, &s.writeNext)
+	stateSinkObject.Save(18, &s.writeList)
+	stateSinkObject.Save(19, &s.rtt)
+	stateSinkObject.Save(20, &s.rto)
+	stateSinkObject.Save(21, &s.minRTO)
+	stateSinkObject.Save(22, &s.maxRTO)
+	stateSinkObject.Save(23, &s.maxRetries)
+	stateSinkObject.Save(24, &s.maxPayloadSize)
+	stateSinkObject.Save(25, &s.gso)
+	stateSinkObject.Save(26, &s.sndWndScale)
+	stateSinkObject.Save(27, &s.maxSentAck)
+	stateSinkObject.Save(28, &s.state)
+	stateSinkObject.Save(29, &s.cc)
+	stateSinkObject.Save(30, &s.rc)
 }
 
 func (s *sender) StateLoad(stateSourceObject state.Source) {
@@ -796,28 +795,29 @@ func (s *sender) StateLoad(stateSourceObject state.Source) {
 	stateSourceObject.Load(6, &s.sndSsthresh)
 	stateSourceObject.Load(7, &s.sndCAAckCount)
 	stateSourceObject.Load(8, &s.outstanding)
-	stateSourceObject.Load(9, &s.sndWnd)
-	stateSourceObject.Load(10, &s.sndUna)
-	stateSourceObject.Load(11, &s.sndNxt)
-	stateSourceObject.Load(12, &s.rttMeasureSeqNum)
-	stateSourceObject.Load(15, &s.closed)
-	stateSourceObject.Load(16, &s.writeNext)
-	stateSourceObject.Load(17, &s.writeList)
-	stateSourceObject.Load(18, &s.rtt)
-	stateSourceObject.Load(19, &s.rto)
-	stateSourceObject.Load(20, &s.minRTO)
-	stateSourceObject.Load(21, &s.maxRTO)
-	stateSourceObject.Load(22, &s.maxRetries)
-	stateSourceObject.Load(23, &s.maxPayloadSize)
-	stateSourceObject.Load(24, &s.gso)
-	stateSourceObject.Load(25, &s.sndWndScale)
-	stateSourceObject.Load(26, &s.maxSentAck)
-	stateSourceObject.Load(27, &s.state)
-	stateSourceObject.Load(28, &s.cc)
-	stateSourceObject.Load(29, &s.rc)
+	stateSourceObject.Load(9, &s.sackedOut)
+	stateSourceObject.Load(10, &s.sndWnd)
+	stateSourceObject.Load(11, &s.sndUna)
+	stateSourceObject.Load(12, &s.sndNxt)
+	stateSourceObject.Load(13, &s.rttMeasureSeqNum)
+	stateSourceObject.Load(16, &s.closed)
+	stateSourceObject.Load(17, &s.writeNext)
+	stateSourceObject.Load(18, &s.writeList)
+	stateSourceObject.Load(19, &s.rtt)
+	stateSourceObject.Load(20, &s.rto)
+	stateSourceObject.Load(21, &s.minRTO)
+	stateSourceObject.Load(22, &s.maxRTO)
+	stateSourceObject.Load(23, &s.maxRetries)
+	stateSourceObject.Load(24, &s.maxPayloadSize)
+	stateSourceObject.Load(25, &s.gso)
+	stateSourceObject.Load(26, &s.sndWndScale)
+	stateSourceObject.Load(27, &s.maxSentAck)
+	stateSourceObject.Load(28, &s.state)
+	stateSourceObject.Load(29, &s.cc)
+	stateSourceObject.Load(30, &s.rc)
 	stateSourceObject.LoadValue(1, new(unixTime), func(y interface{}) { s.loadLastSendTime(y.(unixTime)) })
-	stateSourceObject.LoadValue(13, new(unixTime), func(y interface{}) { s.loadRttMeasureTime(y.(unixTime)) })
-	stateSourceObject.LoadValue(14, new(unixTime), func(y interface{}) { s.loadFirstRetransmittedSegXmitTime(y.(unixTime)) })
+	stateSourceObject.LoadValue(14, new(unixTime), func(y interface{}) { s.loadRttMeasureTime(y.(unixTime)) })
+	stateSourceObject.LoadValue(15, new(unixTime), func(y interface{}) { s.loadFirstRetransmittedSegXmitTime(y.(unixTime)) })
 	stateSourceObject.AfterLoad(s.afterLoad)
 }