Avoid ignoring incoming packet by demuxer on endpoint lookup failure

This fixes a race that occurs while the endpoint is being unregistered
and the transport demuxer attempts to match the incoming packet to any
endpoint. The race specifically occurs when the unregistration (and
deletion of the endpoint) occurs, after a successful endpointsByNIC
lookup and before the endpoints map is further looked up with ingress
NICID of the packet.

The fix is to notify the caller of lookup-with-NICID failure, so that
the logic falls through to handling unknown destination packets.
For TCP this can mean replying back with RST.

The syscall test in this CL catches this race as the ACK completing the
handshake could get silently dropped on a listener close, causing no
RST sent to the peer and timing out the poll waiting for POLLHUP.

Fixes #5850

PiperOrigin-RevId: 369023779

1 files changed, 9 insertions, 8 deletions

diff --git a/pkg/tcpip/stack/transport_demuxer.go b/pkg/tcpip/stack/transport_demuxer.go
index e188efccb..80ad1a9d4 100644
--- a/pkg/tcpip/stack/transport_demuxer.go
+++ b/pkg/tcpip/stack/transport_demuxer.go
@@ -150,16 +150,17 @@ func (epsByNIC *endpointsByNIC) transportEndpoints() []TransportEndpoint {
 	return eps
 }
 
-// HandlePacket is called by the stack when new packets arrive to this transport
-// endpoint.
-func (epsByNIC *endpointsByNIC) handlePacket(id TransportEndpointID, pkt *PacketBuffer) {
+// handlePacket is called by the stack when new packets arrive to this transport
+// endpoint. It returns false if the packet could not be matched to any
+// transport endpoint, true otherwise.
+func (epsByNIC *endpointsByNIC) handlePacket(id TransportEndpointID, pkt *PacketBuffer) bool {
 	epsByNIC.mu.RLock()
 
 	mpep, ok := epsByNIC.endpoints[pkt.NICID]
 	if !ok {
 		if mpep, ok = epsByNIC.endpoints[0]; !ok {
 			epsByNIC.mu.RUnlock() // Don't use defer for performance reasons.
-			return
+			return false
 		}
 	}
 
@@ -168,18 +169,19 @@ func (epsByNIC *endpointsByNIC) handlePacket(id TransportEndpointID, pkt *Packet
 	if isInboundMulticastOrBroadcast(pkt, id.LocalAddress) {
 		mpep.handlePacketAll(id, pkt)
 		epsByNIC.mu.RUnlock() // Don't use defer for performance reasons.
-		return
+		return true
 	}
 	// multiPortEndpoints are guaranteed to have at least one element.
 	transEP := selectEndpoint(id, mpep, epsByNIC.seed)
 	if queuedProtocol, mustQueue := mpep.demux.queuedProtocols[protocolIDs{mpep.netProto, mpep.transProto}]; mustQueue {
 		queuedProtocol.QueuePacket(transEP, id, pkt)
 		epsByNIC.mu.RUnlock()
-		return
+		return true
 	}
 
 	transEP.HandlePacket(id, pkt)
 	epsByNIC.mu.RUnlock() // Don't use defer for performance reasons.
+	return true
 }
 
 // handleError delivers an error to the transport endpoint identified by id.
@@ -567,8 +569,7 @@ func (d *transportDemuxer) deliverPacket(protocol tcpip.TransportProtocolNumber,
 		}
 		return false
 	}
-	ep.handlePacket(id, pkt)
-	return true
+	return ep.handlePacket(id, pkt)
 }
 
 // deliverRawPacket attempts to deliver the given packet and returns whether it