// Copyright 2020 The gVisor Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package tcp_rack_test import ( "flag" "testing" "time" "golang.org/x/sys/unix" "gvisor.dev/gvisor/pkg/abi/linux" "gvisor.dev/gvisor/pkg/binary" "gvisor.dev/gvisor/pkg/tcpip/header" "gvisor.dev/gvisor/pkg/tcpip/seqnum" "gvisor.dev/gvisor/pkg/usermem" "gvisor.dev/gvisor/test/packetimpact/testbench" ) func init() { testbench.Initialize(flag.CommandLine) } const ( // payloadSize is the size used to send packets. payloadSize = header.TCPDefaultMSS // simulatedRTT is the time delay between packets sent and acked to // increase the RTT. simulatedRTT = 30 * time.Millisecond // numPktsForRTT is the number of packets sent and acked to establish // RTT. numPktsForRTT = 10 ) func createSACKConnection(t *testing.T) (testbench.DUT, testbench.TCPIPv4, int32, int32) { dut := testbench.NewDUT(t) listenFd, remotePort := dut.CreateListener(t, unix.SOCK_STREAM, unix.IPPROTO_TCP, 1) conn := dut.Net.NewTCPIPv4(t, testbench.TCP{DstPort: &remotePort}, testbench.TCP{SrcPort: &remotePort}) // Enable SACK. opts := make([]byte, 40) optsOff := 0 optsOff += header.EncodeNOP(opts[optsOff:]) optsOff += header.EncodeNOP(opts[optsOff:]) optsOff += header.EncodeSACKPermittedOption(opts[optsOff:]) conn.ConnectWithOptions(t, opts[:optsOff]) acceptFd, _ := dut.Accept(t, listenFd) return dut, conn, acceptFd, listenFd } func closeSACKConnection(t *testing.T, dut testbench.DUT, conn testbench.TCPIPv4, acceptFd, listenFd int32) { dut.Close(t, acceptFd) dut.Close(t, listenFd) conn.Close(t) } func getRTTAndRTO(t *testing.T, dut testbench.DUT, acceptFd int32) (rtt, rto time.Duration) { info := linux.TCPInfo{} infoBytes := dut.GetSockOpt(t, acceptFd, unix.SOL_TCP, unix.TCP_INFO, int32(linux.SizeOfTCPInfo)) if got, want := len(infoBytes), linux.SizeOfTCPInfo; got != want { t.Fatalf("expected %T, got %d bytes want %d bytes", info, got, want) } binary.Unmarshal(infoBytes, usermem.ByteOrder, &info) return time.Duration(info.RTT) * time.Microsecond, time.Duration(info.RTO) * time.Microsecond } func sendAndReceive(t *testing.T, dut testbench.DUT, conn testbench.TCPIPv4, numPkts int, acceptFd int32, sendACK bool) time.Time { seqNum1 := *conn.RemoteSeqNum(t) payload := make([]byte, payloadSize) var lastSent time.Time for i, sn := 0, seqNum1; i < numPkts; i++ { lastSent = time.Now() dut.Send(t, acceptFd, payload, 0) gotOne, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(sn))}, time.Second) if err != nil { t.Fatalf("Expect #%d: %s", i+1, err) continue } if gotOne == nil { t.Fatalf("#%d: expected a packet within a second but got none", i+1) } sn.UpdateForward(seqnum.Size(payloadSize)) if sendACK { time.Sleep(simulatedRTT) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(sn))}) } } return lastSent } // TestRACKTLPAllPacketsLost tests TLP when an entire flight of data is lost. func TestRACKTLPAllPacketsLost(t *testing.T) { dut, conn, acceptFd, listenFd := createSACKConnection(t) seqNum1 := *conn.RemoteSeqNum(t) // Send ACK for data packets to establish RTT. sendAndReceive(t, dut, conn, numPktsForRTT, acceptFd, true /* sendACK */) seqNum1.UpdateForward(seqnum.Size(numPktsForRTT * payloadSize)) // We are not sending ACK for these packets. const numPkts = 5 lastSent := sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) // Probe Timeout (PTO) should be two times RTT. Check that the last // packet is retransmitted after probe timeout. rtt, _ := getRTTAndRTO(t, dut, acceptFd) pto := rtt * 2 // We expect the 5th packet (the last unacknowledged packet) to be // retransmitted. tlpProbe := testbench.Uint32(uint32(seqNum1) + uint32((numPkts-1)*payloadSize)) if _, err := conn.Expect(t, testbench.TCP{SeqNum: tlpProbe}, time.Second); err != nil { t.Fatalf("expected payload was not received: %s %v %v", err, rtt, pto) } diff := time.Now().Sub(lastSent) if diff < pto { t.Fatalf("expected payload was received before the probe timeout, got: %v, want: %v", diff, pto) } closeSACKConnection(t, dut, conn, acceptFd, listenFd) } // TestRACKTLPLost tests TLP when there are tail losses. // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-7.4 func TestRACKTLPLost(t *testing.T) { dut, conn, acceptFd, listenFd := createSACKConnection(t) seqNum1 := *conn.RemoteSeqNum(t) // Send ACK for data packets to establish RTT. sendAndReceive(t, dut, conn, numPktsForRTT, acceptFd, true /* sendACK */) seqNum1.UpdateForward(seqnum.Size(numPktsForRTT * payloadSize)) // We are not sending ACK for these packets. const numPkts = 10 lastSent := sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) // Cumulative ACK for #[1-5] packets. ackNum := seqNum1.Add(seqnum.Size(6 * payloadSize)) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(ackNum))}) // Probe Timeout (PTO) should be two times RTT. Check that the last // packet is retransmitted after probe timeout. rtt, _ := getRTTAndRTO(t, dut, acceptFd) pto := rtt * 2 // We expect the 10th packet (the last unacknowledged packet) to be // retransmitted. tlpProbe := testbench.Uint32(uint32(seqNum1) + uint32((numPkts-1)*payloadSize)) if _, err := conn.Expect(t, testbench.TCP{SeqNum: tlpProbe}, time.Second); err != nil { t.Fatalf("expected payload was not received: %s", err) } diff := time.Now().Sub(lastSent) if diff < pto { t.Fatalf("expected payload was received before the probe timeout, got: %v, want: %v", diff, pto) } closeSACKConnection(t, dut, conn, acceptFd, listenFd) } // TestRACKTLPWithSACK tests TLP by acknowledging out of order packets. // See: https://tools.ietf.org/html/draft-ietf-tcpm-rack-08#section-8.1 func TestRACKTLPWithSACK(t *testing.T) { dut, conn, acceptFd, listenFd := createSACKConnection(t) seqNum1 := *conn.RemoteSeqNum(t) // Send ACK for data packets to establish RTT. sendAndReceive(t, dut, conn, numPktsForRTT, acceptFd, true /* sendACK */) seqNum1.UpdateForward(seqnum.Size(numPktsForRTT * payloadSize)) // We are not sending ACK for these packets. const numPkts = 3 lastSent := sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) // SACK for #2 packet. sackBlock := make([]byte, 40) start := seqNum1.Add(seqnum.Size(payloadSize)) end := start.Add(seqnum.Size(payloadSize)) sbOff := 0 sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, sackBlock[sbOff:]) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1)), Options: sackBlock[:sbOff]}) // RACK marks #1 packet as lost and retransmits it. if _, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(seqNum1))}, time.Second); err != nil { t.Fatalf("expected payload was not received: %s", err) } // ACK for #1 packet. conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(end))}) // Probe Timeout (PTO) should be two times RTT. TLP will trigger for #3 // packet. RACK adds an additional timeout of 200ms if the number of // outstanding packets is equal to 1. rtt, rto := getRTTAndRTO(t, dut, acceptFd) pto := rtt*2 + (200 * time.Millisecond) if rto < pto { pto = rto } // We expect the 3rd packet (the last unacknowledged packet) to be // retransmitted. tlpProbe := testbench.Uint32(uint32(seqNum1) + uint32((numPkts-1)*payloadSize)) if _, err := conn.Expect(t, testbench.TCP{SeqNum: tlpProbe}, time.Second); err != nil { t.Fatalf("expected payload was not received: %s", err) } diff := time.Now().Sub(lastSent) if diff < pto { t.Fatalf("expected payload was received before the probe timeout, got: %v, want: %v", diff, pto) } closeSACKConnection(t, dut, conn, acceptFd, listenFd) } // TestRACKWithoutReorder tests that without reordering RACK will retransmit the // lost packets after reorder timer expires. func TestRACKWithoutReorder(t *testing.T) { dut, conn, acceptFd, listenFd := createSACKConnection(t) seqNum1 := *conn.RemoteSeqNum(t) // Send ACK for data packets to establish RTT. sendAndReceive(t, dut, conn, numPktsForRTT, acceptFd, true /* sendACK */) seqNum1.UpdateForward(seqnum.Size(numPktsForRTT * payloadSize)) // We are not sending ACK for these packets. const numPkts = 4 sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) // SACK for [3,4] packets. sackBlock := make([]byte, 40) start := seqNum1.Add(seqnum.Size(2 * payloadSize)) end := start.Add(seqnum.Size(2 * payloadSize)) sbOff := 0 sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, sackBlock[sbOff:]) time.Sleep(simulatedRTT) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1)), Options: sackBlock[:sbOff]}) // RACK marks #1 and #2 packets as lost and retransmits both after // RTT + reorderWindow. The reorderWindow initially will be a small // fraction of RTT. rtt, _ := getRTTAndRTO(t, dut, acceptFd) timeout := 2 * rtt for i, sn := 0, seqNum1; i < 2; i++ { if _, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(sn))}, timeout); err != nil { t.Fatalf("expected payload was not received: %s", err) } sn.UpdateForward(seqnum.Size(payloadSize)) } closeSACKConnection(t, dut, conn, acceptFd, listenFd) } // TestRACKWithReorder tests that RACK will retransmit segments when there is // reordering in the connection and reorder timer expires. func TestRACKWithReorder(t *testing.T) { dut, conn, acceptFd, listenFd := createSACKConnection(t) seqNum1 := *conn.RemoteSeqNum(t) // Send ACK for data packets to establish RTT. sendAndReceive(t, dut, conn, numPktsForRTT, acceptFd, true /* sendACK */) seqNum1.UpdateForward(seqnum.Size(numPktsForRTT * payloadSize)) // We are not sending ACK for these packets. const numPkts = 4 sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) time.Sleep(simulatedRTT) // SACK in reverse order for the connection to detect reorder. var start seqnum.Value var end seqnum.Value for i := 0; i < numPkts-1; i++ { sackBlock := make([]byte, 40) sbOff := 0 start = seqNum1.Add(seqnum.Size((numPkts - i - 1) * payloadSize)) end = start.Add(seqnum.Size((i + 1) * payloadSize)) sackBlock = make([]byte, 40) sbOff = 0 sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, sackBlock[sbOff:]) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1)), Options: sackBlock[:sbOff]}) } // Send a DSACK block indicating both original and retransmitted // packets are received, RACK will increase the reordering window on // every DSACK. dsackBlock := make([]byte, 40) dbOff := 0 start = seqNum1 end = start.Add(seqnum.Size(2 * payloadSize)) dbOff += header.EncodeNOP(dsackBlock[dbOff:]) dbOff += header.EncodeNOP(dsackBlock[dbOff:]) dbOff += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, dsackBlock[dbOff:]) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1 + numPkts*payloadSize)), Options: dsackBlock[:dbOff]}) seqNum1.UpdateForward(seqnum.Size(numPkts * payloadSize)) sendTime := time.Now() sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) time.Sleep(simulatedRTT) // Send SACK for [2-5] packets. sackBlock := make([]byte, 40) sbOff := 0 start = seqNum1.Add(seqnum.Size(payloadSize)) end = start.Add(seqnum.Size(3 * payloadSize)) sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, sackBlock[sbOff:]) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1)), Options: sackBlock[:sbOff]}) // Expect the retransmission of #1 packet after RTT+ReorderWindow. if _, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(seqNum1))}, time.Second); err != nil { t.Fatalf("expected payload was not received: %s", err) } rtt, _ := getRTTAndRTO(t, dut, acceptFd) diff := time.Now().Sub(sendTime) if diff < rtt { t.Fatalf("expected payload was received too sonn, within RTT") } closeSACKConnection(t, dut, conn, acceptFd, listenFd) } // TestRACKWithLostRetransmission tests that RACK will not enter RTO when a // retransmitted segment is lost and enters fast recovery. func TestRACKWithLostRetransmission(t *testing.T) { dut, conn, acceptFd, listenFd := createSACKConnection(t) seqNum1 := *conn.RemoteSeqNum(t) // Send ACK for data packets to establish RTT. sendAndReceive(t, dut, conn, numPktsForRTT, acceptFd, true /* sendACK */) seqNum1.UpdateForward(seqnum.Size(numPktsForRTT * payloadSize)) // We are not sending ACK for these packets. const numPkts = 5 sendAndReceive(t, dut, conn, numPkts, acceptFd, false /* sendACK */) // SACK for [2-5] packets. sackBlock := make([]byte, 40) start := seqNum1.Add(seqnum.Size(payloadSize)) end := start.Add(seqnum.Size(4 * payloadSize)) sbOff := 0 sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeNOP(sackBlock[sbOff:]) sbOff += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, sackBlock[sbOff:]) time.Sleep(simulatedRTT) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1)), Options: sackBlock[:sbOff]}) // RACK marks #1 packet as lost and retransmits it after // RTT + reorderWindow. The reorderWindow is bounded between a small // fraction of RTT and 1 RTT. rtt, _ := getRTTAndRTO(t, dut, acceptFd) timeout := 2 * rtt if _, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(seqNum1))}, timeout); err != nil { t.Fatalf("expected payload was not received: %s", err) } // Send #6 packet. payload := make([]byte, payloadSize) dut.Send(t, acceptFd, payload, 0) gotOne, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(seqNum1 + 5*payloadSize))}, time.Second) if err != nil { t.Fatalf("Expect #6: %s", err) } if gotOne == nil { t.Fatalf("#6: expected a packet within a second but got none") } // SACK for [2-6] packets. sackBlock1 := make([]byte, 40) start = seqNum1.Add(seqnum.Size(payloadSize)) end = start.Add(seqnum.Size(5 * payloadSize)) sbOff1 := 0 sbOff1 += header.EncodeNOP(sackBlock1[sbOff1:]) sbOff1 += header.EncodeNOP(sackBlock1[sbOff1:]) sbOff1 += header.EncodeSACKBlocks([]header.SACKBlock{{ start, end, }}, sackBlock1[sbOff1:]) time.Sleep(simulatedRTT) conn.Send(t, testbench.TCP{Flags: testbench.Uint8(header.TCPFlagAck), AckNum: testbench.Uint32(uint32(seqNum1)), Options: sackBlock1[:sbOff1]}) // Expect re-retransmission of #1 packet without entering an RTO. if _, err := conn.Expect(t, testbench.TCP{SeqNum: testbench.Uint32(uint32(seqNum1))}, timeout); err != nil { t.Fatalf("expected payload was not received: %s", err) } // Check the congestion control state. info := linux.TCPInfo{} infoBytes := dut.GetSockOpt(t, acceptFd, unix.SOL_TCP, unix.TCP_INFO, int32(linux.SizeOfTCPInfo)) if got, want := len(infoBytes), linux.SizeOfTCPInfo; got != want { t.Fatalf("expected %T, got %d bytes want %d bytes", info, got, want) } binary.Unmarshal(infoBytes, usermem.ByteOrder, &info) if info.CaState != linux.TCP_CA_Recovery { t.Fatalf("expected connection to be in fast recovery, want: %v got: %v", linux.TCP_CA_Recovery, info.CaState) } closeSACKConnection(t, dut, conn, acceptFd, listenFd) }