Update

vendor/github.com/tailscale/wireguard-go/tun/errors.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	// ErrTooManySegments is returned by Device.Read() when segmentation
+	// ErrTooManySegments is returned by Reader.Read() when segmentation
 	// overflows the length of supplied buffers. This error should not cause
 	// reads to cease.
 	ErrTooManySegments = errors.New("too many segments")

vendor/github.com/tailscale/wireguard-go/tun/offload.go

@@ -2,6 +2,7 @@ package tun
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 )
 
@@ -73,15 +74,34 @@ const (
 	ipProtoUDP = 17
 )
 
-// GSOSplit splits packets from 'in' into outBufs[<index>][outOffset:], writing
-// the size of each element into sizes. It returns the number of buffers
-// populated, and/or an error. Callers may pass an 'in' slice that overlaps with
-// the first element of outBuffers, i.e. &in[0] may be equal to
-// &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the
-// value of options.NeedsCsum. Length of each outBufs element must be greater
-// than or equal to the length of 'in', otherwise output may be silently
-// truncated.
-func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) {
+// GSOSplit splits 'in' according to options and writes the resulting segments
+// into slab. It populates packets with each packet's offset and size and returns
+// the number of populated entries.
+//
+// spacing bytes are reserved before the first packet, between adjacent packets,
+// and after the final packet.
+//
+// 'in' may overlap slab only when in begins at slab[spacing]. Overlapping input
+// is modified and must not be used after this call.
+//
+// If packets or slab cannot accommodate every segment, GSOSplit returns the
+// number successfully written together with [ErrTooManySegments]. packets[:n]
+// remains valid.
+func GSOSplit(in []byte, options GSOOptions, slab []byte, packets []ReadPacket, spacing int) (int, error) {
+	if spacing < 0 {
+		return 0, errors.New("spacing must be nonnegative")
+	}
+
+	// return early if instructed to split with no split size
+	if options.GSOType != GSONone && options.GSOSize == 0 {
+		return 0, fmt.Errorf("invalid GSOType (%v) with zero GSOSize", options.GSOType)
+	}
+
+	// return early if we can't fill a single packet
+	if len(packets) == 0 || spacing > len(slab)/2 {
+		return 0, ErrTooManySegments
+	}
+
 	cSumAt := int(options.CsumStart) + int(options.CsumOffset)
 	if cSumAt+1 >= len(in) {
 		return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in))
@@ -91,20 +111,34 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 		return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen)
 	}
 
-	// Handle the conditions where we are copying a single element to outBuffs.
+	// Handle the conditions where we are copying a single element to slab.
 	payloadLen := len(in) - int(options.HdrLen)
 	if options.GSOType == GSONone || payloadLen < int(options.GSOSize) {
-		if len(in) > len(outBufs[0][outOffset:]) {
-			return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:]))
+		// trim spacing on both ends
+		output := slab[spacing : len(slab)-spacing]
+
+		if len(in) > len(output) {
+			return 0, ErrTooManySegments
 		}
+
+		// slice to packet size, so checksum computation below is correct
+		singleOut := output[:len(in)]
+
+		// Copy before checksum computation, so that we don't mutate 'in' if
+		// 'in' and 'slab' happen to be disjoint.
+		copy(singleOut, in)
+		packets[0] = ReadPacket{
+			Offset: spacing,
+			Size:   len(singleOut),
+		}
+
 		if options.NeedsCsum {
 			// The initial value at the checksum offset should be summed with
 			// the checksum we compute. This is typically the pseudo-header sum.
-			initial := binary.BigEndian.Uint16(in[cSumAt:])
-			in[cSumAt], in[cSumAt+1] = 0, 0
-			binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial))
+			initial := binary.BigEndian.Uint16(singleOut[cSumAt:])
+			singleOut[cSumAt], singleOut[cSumAt+1] = 0, 0
+			binary.BigEndian.PutUint16(singleOut[cSumAt:], ^Checksum(singleOut[options.CsumStart:], initial))
 		}
-		sizes[0] = copy(outBufs[0][outOffset:], in)
 		return 1, nil
 	}
 
@@ -132,6 +166,62 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 		return 0, fmt.Errorf("invalid ip header version: %d", ipVersion)
 	}
 
+	// Calculate how many segments fit in slab. I found it helpful to mentally
+	// model the slab as:
+	//
+	//	[leading spacing][packet][spacing][packet]...[spacing]
+	//
+	// If every packet were full-sized, after removing the leading spacing each
+	// packet consumes fullPacketSize+spacing bytes.
+	hdrLen := int(options.HdrLen)
+	gsoSize := int(options.GSOSize)
+	// equivalent to ceil(payloadLen / gsoSize), but overflow safe if payloadLen
+	// is close to bounds
+	totalOutSegments := payloadLen / gsoSize
+	if payloadLen%gsoSize != 0 {
+		totalOutSegments++
+	}
+	// how many full gso-sized segments would fit in slab
+	fullPacketSize := hdrLen + gsoSize
+	fullFit := (len(slab) - spacing) / (fullPacketSize + spacing)
+	// this min does not yet account for a smaller tail
+	n := min(totalOutSegments, len(packets), fullFit)
+
+	// Account for the potentially shorter final segment. The exact layout needs
+	// one header per segment, the original payload bytes, and N+1 spacing
+	// regions.
+	requiredBytesLen :=
+		totalOutSegments*hdrLen +
+			payloadLen +
+			(totalOutSegments+1)*spacing
+	if len(packets) >= totalOutSegments && requiredBytesLen <= len(slab) {
+		// n was previously calculated assuming totalOutSegments were all of
+		// gsoSize, but the tail segment may be smaller than gsoSize. If a
+		// smaller than gsoSize tail fits, we can bump n back to totalOutSegments,
+		// assuming packets is long enough.
+		n = totalOutSegments
+	}
+
+	// n is the number of leading segments that fit.
+	//
+	// Since input and output may overlap, split back-to-front. Splitting
+	// front-to-back could overwrite input before it is copied.
+	//
+	// H is the header; A, B, and C are payload segments.
+	//
+	//	Input:
+	//	[ H | A | B | C ]
+	//
+	//	Desired output:
+	//	[ H | A | gap | H | B | gap | H | C ]
+	//
+	//	Front-to-back could overwrite unread B/C:
+	//	[ H | A | gap | H ... ]
+	//	          └── overwrites remaining input ──┘
+	//
+	// For the same reason, copy each segment's payload before its headers.
+
+	// Identify IP addr offsets and length, transport protocol, and transport offsets.
 	iphLen := int(options.CsumStart)
 	srcAddrOffset := ipv6SrcAddrOffset
 	addrLen := 16
@@ -152,22 +242,61 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 	} else {
 		protocol = ipProtoUDP
 	}
-	nextSegmentDataAt := int(options.HdrLen)
-	i := 0
-	for ; nextSegmentDataAt < len(in); i++ {
-		if i == len(outBufs) {
-			return i - 1, ErrTooManySegments
-		}
-		nextSegmentEnd := nextSegmentDataAt + int(options.GSOSize)
-		if nextSegmentEnd > len(in) {
-			nextSegmentEnd = len(in)
-		}
-		segmentDataLen := nextSegmentEnd - nextSegmentDataAt
-		totalLen := int(options.HdrLen) + segmentDataLen
-		sizes[i] = totalLen
-		out := outBufs[i][outOffset:]
 
-		copy(out, in[:iphLen])
+	// The segment-writing loop indexes these fixed IP and transport header
+	// fields within out[:hdrLen]. Validate that each field fits before
+	// modifying slab.
+	csumStart := int(options.CsumStart)
+	csumAt := csumStart + int(options.CsumOffset)
+	if ipVersion == 4 {
+		if hdrLen < 20 {
+			return 0, fmt.Errorf("IPv4 header exceeds GSO header length %d", hdrLen)
+		}
+	} else {
+		if hdrLen < 40 {
+			return 0, fmt.Errorf("IPv6 header exceeds GSO header length %d", hdrLen)
+		}
+	}
+	if protocol == ipProtoTCP {
+		if hdrLen < csumStart+20 {
+			return 0, fmt.Errorf(
+				"TCP header at offset %d exceeds GSO header length %d",
+				csumStart, hdrLen,
+			)
+		}
+	} else {
+		if hdrLen < csumStart+8 {
+			return 0, fmt.Errorf(
+				"UDP header at offset %d exceeds GSO header length %d",
+				csumStart, hdrLen,
+			)
+		}
+	}
+	if hdrLen < csumAt+2 {
+		return 0, fmt.Errorf(
+			"checksum field at offset %d exceeds GSO header length %d",
+			csumAt, hdrLen,
+		)
+	}
+
+	// Split back-to-front
+	for i := n - 1; i >= 0; i-- {
+		// identify input and output offsets
+		payloadStart := hdrLen + (i * gsoSize)
+		payloadEnd := min(payloadStart+gsoSize, len(in))
+		payloadSize := payloadEnd - payloadStart
+		packetSize := hdrLen + payloadSize
+		outOffset := spacing + i*(fullPacketSize+spacing)
+		out := slab[outOffset : outOffset+packetSize]
+		packets[i] = ReadPacket{
+			Offset: outOffset,
+			Size:   packetSize,
+		}
+
+		// Copy payload first because in and out may overlap.
+		copy(out[hdrLen:], in[payloadStart:payloadEnd])
+		copy(out[:hdrLen], in[:hdrLen])
+
 		if ipVersion == 4 {
 			// For IPv4 we are responsible for incrementing the ID field,
 			// updating the total len field, and recalculating the header
@@ -177,44 +306,57 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 				id += uint16(i)
 				binary.BigEndian.PutUint16(out[4:], id)
 			}
-			out[10], out[11] = 0, 0 // clear ipv4 header checksum
-			binary.BigEndian.PutUint16(out[2:], uint16(totalLen))
-			ipv4CSum := ^Checksum(out[:iphLen], 0)
-			binary.BigEndian.PutUint16(out[10:], ipv4CSum)
+			// clear ipv4 header checksum
+			out[10], out[11] = 0, 0
+			// update total len
+			binary.BigEndian.PutUint16(out[2:], uint16(packetSize))
+			// compute & set updated checksum
+			binary.BigEndian.PutUint16(out[10:], ^Checksum(out[:iphLen], 0))
 		} else {
 			// For IPv6 we are responsible for updating the payload length field.
-			binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen))
+			binary.BigEndian.PutUint16(out[4:], uint16(packetSize-iphLen))
 		}
 
-		// copy transport header
-		copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen])
-
 		if protocol == ipProtoTCP {
 			// set TCP seq and adjust TCP flags
-			tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i))
-			binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq)
-			if nextSegmentEnd != len(in) {
-				// FIN and PSH should only be set on last segment
+			seq := firstTCPSeqNum + uint32(i*gsoSize)
+			binary.BigEndian.PutUint32(out[options.CsumStart+4:], seq)
+
+			if payloadEnd != len(in) {
+				// FIN and PSH should only be set on last segment. Last
+				// specifically means last input segment, not last output, which
+				// could be last fitting, but not last input.
 				clearFlags := tcpFlagFIN | tcpFlagPSH
 				out[options.CsumStart+tcpFlagsOffset] &^= clearFlags
 			}
 		} else {
 			// set UDP header len
-			binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart))
+			udpLen := uint16(payloadSize) + options.HdrLen - options.CsumStart
+			binary.BigEndian.PutUint16(out[options.CsumStart+4:], udpLen)
 		}
 
-		// payload
-		copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd])
-
 		// transport checksum
 		out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum
 		transportHeaderLen := int(options.HdrLen - options.CsumStart)
-		lenForPseudo := uint16(transportHeaderLen + segmentDataLen)
-		transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo)
-		transportCSum = ^Checksum(out[options.CsumStart:totalLen], transportCSum)
-		binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum)
-
-		nextSegmentDataAt += int(options.GSOSize)
+		pseudoLen := uint16(transportHeaderLen + payloadSize)
+		transportCSum := PseudoHeaderChecksum(
+			protocol,
+			in[srcAddrOffset:srcAddrOffset+addrLen],
+			in[srcAddrOffset+addrLen:srcAddrOffset+2*addrLen],
+			pseudoLen,
+		)
+		transportCSum = ^Checksum(
+			out[options.CsumStart:packetSize],
+			transportCSum,
+		)
+		binary.BigEndian.PutUint16(
+			out[options.CsumStart+options.CsumOffset:],
+			transportCSum,
+		)
 	}
-	return i, nil
+
+	if n < totalOutSegments {
+		return n, ErrTooManySegments
+	}
+	return n, nil
 }

vendor/github.com/tailscale/wireguard-go/tun/offload_linux.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"slices"
 	"unsafe"
 
 	"github.com/tailscale/wireguard-go/conn"
@@ -72,8 +73,76 @@ const (
 	// virtioNetHdrLen is the length in bytes of virtioNetHdr. This matches the
 	// shape of the C ABI for its kernel counterpart -- sizeof(virtio_net_hdr).
 	virtioNetHdrLen = int(unsafe.Sizeof(virtioNetHdr{}))
+
+	// Vector layout: [virtioHdr | headPacket | coalescedPayloadFragments...]
+	iovVirtioNetHdrIdx         = 0
+	iovEmptyVectorLen          = 1
+	iovHeadPacketIdx           = 1
+	iovSinglePacketLen         = 2
+	iovFirstPayloadFragmentIdx = 2
+
+	maxScatterGatherFragments = 1024 // Limited by UIO_MAXIOV of 1024.
 )
 
+// groToWrite holds the write-ordered scatter-gather IO vectors for
+// writev.
+type groToWrite struct {
+	// Each iov is a [][]byte:
+	// - iovs[i][0] is the pre-allocated virtio header, fixed to [virtioNetHdrLen]
+	// - iovs[i][1] is the head packet with transport headers
+	// - iovs[i][2:] are coalesced payload fragments
+	//
+	// Here and elsewhere, "packet" refers to the full IP packet including transport headers,
+	// and "payload fragment" refers to the data without transport headers.
+	//
+	// Empty iov is length 1 (iovEmptyVectorLen), with the first element (iovVirtioNetHdrIdx) zeroed.
+	// Single packet iov is length 2 (iovSinglePacketLen),
+	// with the first element zeroed, and the full packet at index 1 (iovHeadPacketIdx).
+	// Coalesced packet is length >2, with the first element written from [virtioNetHdr],
+	// the second element the head packet,
+	// and the remaining elements (iovFirstPayloadFragmentIdx:) coalesced payload fragments.
+	iovs      [][][]byte
+	allocated int
+}
+
+func newGROToWrite() groToWrite {
+	wi := groToWrite{
+		iovs: make([][][]byte, 0, conn.IdealBatchSize),
+	}
+	for range cap(wi.iovs) {
+		wi.appendIov(nil) // pre-allocate virtio headers
+	}
+	wi.iovs = wi.iovs[:0]
+	return wi
+}
+
+// appendIov extends iovs by one, reusing the pre-allocated backing.
+// Returns the index of the new item's [][]byte.
+func (w *groToWrite) appendIov(pkt []byte) int {
+	n := len(w.iovs)
+	if n < w.allocated {
+		w.iovs = w.iovs[:n+1]
+	} else {
+		iov := make([][]byte, 1, conn.IdealBatchSize)
+		iov[iovVirtioNetHdrIdx] = make([]byte, virtioNetHdrLen)
+		w.iovs = append(w.iovs, iov)
+		w.allocated++
+	}
+	if pkt != nil {
+		// Nil is passed only in [newGROToWrite] and tests.
+		w.iovs[n] = append(w.iovs[n], pkt)
+	}
+	return n
+}
+
+func (w *groToWrite) reset() {
+	for i := range w.iovs {
+		clear(w.iovs[i][iovVirtioNetHdrIdx])
+		w.iovs[i] = w.iovs[i][:iovEmptyVectorLen] // keep virtio header alloc
+	}
+	w.iovs = w.iovs[:0]
+}
+
 // tcpFlowKey represents the key for a TCP flow.
 type tcpFlowKey struct {
 	srcAddr, dstAddr [16]byte
@@ -114,28 +183,30 @@ func newTCPFlowKey(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset int) tcp
 // lookupOrInsert looks up a flow for the provided packet and metadata,
 // returning the packets found for the flow, or inserting a new one if none
 // is found.
-func (t *tcpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex int) ([]tcpGROItem, bool) {
+func (t *tcpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen int, wi *groToWrite) ([]tcpGROItem, bool) {
 	key := newTCPFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset)
 	items, ok := t.itemsByFlow[key]
 	if ok {
 		return items, ok
 	}
 	// TODO: insert() performs another map lookup. This could be rearranged to avoid.
-	t.insert(pkt, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex)
+	t.insert(pkt, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, wi)
 	return nil, false
 }
 
 // insert an item in the table for the provided packet and packet metadata.
-func (t *tcpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex int) {
+func (t *tcpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen int, wi *groToWrite) {
 	key := newTCPFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset)
+	idx := wi.appendIov(pkt)
 	item := tcpGROItem{
-		key:       key,
-		bufsIndex: uint16(bufsIndex),
-		gsoSize:   uint16(len(pkt[tcphOffset+tcphLen:])),
-		iphLen:    uint8(tcphOffset),
-		tcphLen:   uint8(tcphLen),
-		sentSeq:   binary.BigEndian.Uint32(pkt[tcphOffset+4:]),
-		pshSet:    pkt[tcphOffset+tcpFlagsOffset]&tcpFlagPSH != 0,
+		key:        key,
+		outputIdx:  uint16(idx),
+		gsoSize:    uint16(len(pkt[tcphOffset+tcphLen:])),
+		iphLen:     uint8(tcphOffset),
+		tcphLen:    uint8(tcphLen),
+		sentSeq:    binary.BigEndian.Uint32(pkt[tcphOffset+4:]),
+		pshSet:     pkt[tcphOffset+tcpFlagsOffset]&tcpFlagPSH != 0,
+		payloadLen: uint16(len(pkt[tcphOffset+tcphLen:])),
 	}
 	items, ok := t.itemsByFlow[key]
 	if !ok {
@@ -159,14 +230,14 @@ func (t *tcpGROTable) deleteAt(key tcpFlowKey, i int) {
 // tcpGROItem represents bookkeeping data for a TCP packet during the lifetime
 // of a GRO evaluation across a vector of packets.
 type tcpGROItem struct {
-	key       tcpFlowKey
-	sentSeq   uint32 // the sequence number
-	bufsIndex uint16 // the index into the original bufs slice
-	numMerged uint16 // the number of packets merged into this item
-	gsoSize   uint16 // payload size
-	iphLen    uint8  // ip header len
-	tcphLen   uint8  // tcp header len
-	pshSet    bool   // psh flag is set
+	key        tcpFlowKey
+	sentSeq    uint32 // the sequence number
+	outputIdx  uint16 // index into groToWrite
+	payloadLen uint16 // accumulated payload bytes
+	gsoSize    uint16 // payload size
+	iphLen     uint8  // ip header len
+	tcphLen    uint8  // tcp header len
+	pshSet     bool   // psh flag is set
 }
 
 func (t *tcpGROTable) newItems() []tcpGROItem {
@@ -221,26 +292,28 @@ func newUDPFlowKey(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int) udp
 // lookupOrInsert looks up a flow for the provided packet and metadata,
 // returning the packets found for the flow, or inserting a new one if none
 // is found.
-func (u *udpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex int) ([]udpGROItem, bool) {
+func (u *udpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int, wi *groToWrite) ([]udpGROItem, bool) {
 	key := newUDPFlowKey(pkt, srcAddrOffset, dstAddrOffset, udphOffset)
 	items, ok := u.itemsByFlow[key]
 	if ok {
 		return items, ok
 	}
 	// TODO: insert() performs another map lookup. This could be rearranged to avoid.
-	u.insert(pkt, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex, false)
+	u.insert(pkt, srcAddrOffset, dstAddrOffset, udphOffset, wi, false)
 	return nil, false
 }
 
 // insert an item in the table for the provided packet and packet metadata.
-func (u *udpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex int, cSumKnownInvalid bool) {
+func (u *udpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int, wi *groToWrite, cSumKnownInvalid bool) {
 	key := newUDPFlowKey(pkt, srcAddrOffset, dstAddrOffset, udphOffset)
+	idx := wi.appendIov(pkt)
 	item := udpGROItem{
 		key:              key,
-		bufsIndex:        uint16(bufsIndex),
+		outputIdx:        uint16(idx),
 		gsoSize:          uint16(len(pkt[udphOffset+udphLen:])),
 		iphLen:           uint8(udphOffset),
 		cSumKnownInvalid: cSumKnownInvalid,
+		payloadLen:       uint16(len(pkt[udphOffset+udphLen:])),
 	}
 	items, ok := u.itemsByFlow[key]
 	if !ok {
@@ -259,8 +332,8 @@ func (u *udpGROTable) updateAt(item udpGROItem, i int) {
 // of a GRO evaluation across a vector of packets.
 type udpGROItem struct {
 	key              udpFlowKey
-	bufsIndex        uint16 // the index into the original bufs slice
-	numMerged        uint16 // the number of packets merged into this item
+	outputIdx        uint16 // index into groToWrite
+	payloadLen       uint16 // accumulated payload bytes
 	gsoSize          uint16 // payload size
 	iphLen           uint8  // ip header len
 	cSumKnownInvalid bool   // UDP header checksum validity; a false value DOES NOT imply valid, just unknown.
@@ -325,15 +398,16 @@ func ipHeadersCanCoalesce(pktA, pktB []byte) bool {
 }
 
 // udpPacketsCanCoalesce evaluates if pkt can be coalesced with the packet
-// described by item. iphLen and gsoSize describe pkt. bufs is the vector of
-// packets involved in the current GRO evaluation. bufsOffset is the offset at
-// which packet data begins within bufs.
-func udpPacketsCanCoalesce(pkt []byte, iphLen uint8, gsoSize uint16, item udpGROItem, bufs [][]byte, bufsOffset int) canCoalesce {
-	pktTarget := bufs[item.bufsIndex][bufsOffset:]
-	if !ipHeadersCanCoalesce(pkt, pktTarget) {
+// described by item. iphLen and gsoSize describe pkt.
+func udpPacketsCanCoalesce(pkt []byte, gsoSize uint16, item udpGROItem, wi *groToWrite) canCoalesce {
+	if len(wi.iovs[item.outputIdx]) >= maxScatterGatherFragments {
 		return coalesceUnavailable
 	}
-	if len(pktTarget[iphLen+udphLen:])%int(item.gsoSize) != 0 {
+	headPacket := wi.iovs[item.outputIdx][iovHeadPacketIdx]
+	if !ipHeadersCanCoalesce(pkt, headPacket) {
+		return coalesceUnavailable
+	}
+	if item.payloadLen%item.gsoSize != 0 {
 		// A smaller than gsoSize packet has been appended previously.
 		// Nothing can come after a smaller packet on the end.
 		return coalesceUnavailable
@@ -342,37 +416,44 @@ func udpPacketsCanCoalesce(pkt []byte, iphLen uint8, gsoSize uint16, item udpGRO
 		// We cannot have a larger packet following a smaller one.
 		return coalesceUnavailable
 	}
+	if int(item.iphLen)+udphLen+int(item.payloadLen)+int(gsoSize) > maxUint16 {
+		return coalesceUnavailable
+	}
 	return coalesceAppend
 }
 
 // tcpPacketsCanCoalesce evaluates if pkt can be coalesced with the packet
 // described by item. This function makes considerations that match the kernel's
 // GRO self tests, which can be found in tools/testing/selftests/net/gro.c.
-func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet bool, gsoSize uint16, item tcpGROItem, bufs [][]byte, bufsOffset int) canCoalesce {
-	pktTarget := bufs[item.bufsIndex][bufsOffset:]
+func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet bool, gsoSize uint16, item tcpGROItem, wi *groToWrite) canCoalesce {
+	if len(wi.iovs[item.outputIdx]) >= maxScatterGatherFragments {
+		return coalesceUnavailable
+	}
+	headPacket := wi.iovs[item.outputIdx][iovHeadPacketIdx]
 	if tcphLen != item.tcphLen {
 		// cannot coalesce with unequal tcp options len
 		return coalesceUnavailable
 	}
 	if tcphLen > 20 {
-		if !bytes.Equal(pkt[iphLen+20:iphLen+tcphLen], pktTarget[item.iphLen+20:iphLen+tcphLen]) {
+		if !bytes.Equal(pkt[iphLen+20:iphLen+tcphLen], headPacket[item.iphLen+20:iphLen+tcphLen]) {
 			// cannot coalesce with unequal tcp options
 			return coalesceUnavailable
 		}
 	}
-	if !ipHeadersCanCoalesce(pkt, pktTarget) {
+	if !ipHeadersCanCoalesce(pkt, headPacket) {
+		return coalesceUnavailable
+	}
+	if int(item.iphLen)+int(item.tcphLen)+int(item.payloadLen)+int(gsoSize) > maxUint16 {
 		return coalesceUnavailable
 	}
 	// seq adjacency
-	lhsLen := item.gsoSize
-	lhsLen += item.numMerged * item.gsoSize
-	if seq == item.sentSeq+uint32(lhsLen) { // pkt aligns following item from a seq num perspective
+	if seq == item.sentSeq+uint32(item.payloadLen) { // pkt aligns following item from a seq num perspective
 		if item.pshSet {
 			// We cannot append to a segment that has the PSH flag set, PSH
 			// can only be set on the final segment in a reassembled group.
 			return coalesceUnavailable
 		}
-		if len(pktTarget[iphLen+tcphLen:])%int(item.gsoSize) != 0 {
+		if item.payloadLen%item.gsoSize != 0 {
 			// A smaller than gsoSize packet has been appended previously.
 			// Nothing can come after a smaller packet on the end.
 			return coalesceUnavailable
@@ -392,7 +473,7 @@ func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet
 			// We cannot have a larger packet following a smaller one.
 			return coalesceUnavailable
 		}
-		if gsoSize > item.gsoSize && item.numMerged > 0 {
+		if gsoSize > item.gsoSize && len(wi.iovs[item.outputIdx]) > iovSinglePacketLen {
 			// There's at least one previous merge, and we're larger than all
 			// previous. This would put multiple smaller packets on the end.
 			return coalesceUnavailable
@@ -414,13 +495,11 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool {
 	return ^Checksum(pkt[iphLen:], cSum) == 0
 }
 
-// coalesceResult represents the result of attempting to coalesce two TCP
-// packets.
+// coalesceResult represents the result of attempting to coalesce two packets.
 type coalesceResult int
 
 const (
-	coalesceInsufficientCap coalesceResult = iota
-	coalescePSHEnding
+	coalescePSHEnding coalesceResult = iota
 	coalesceItemInvalidCSum
 	coalescePktInvalidCSum
 	coalesceSuccess
@@ -428,54 +507,33 @@ const (
 
 // coalesceUDPPackets attempts to coalesce pkt with the packet described by
 // item, and returns the outcome.
-func coalesceUDPPackets(pkt []byte, item *udpGROItem, bufs [][]byte, bufsOffset int, isV6 bool) coalesceResult {
-	pktHead := bufs[item.bufsIndex][bufsOffset:] // the packet that will end up at the front
-	headersLen := item.iphLen + udphLen
-	coalescedLen := len(bufs[item.bufsIndex][bufsOffset:]) + len(pkt) - int(headersLen)
-
-	if cap(pktHead)-bufsOffset < coalescedLen {
-		// We don't want to allocate a new underlying array if capacity is
-		// too small.
-		return coalesceInsufficientCap
-	}
-	if item.numMerged == 0 {
-		if item.cSumKnownInvalid || !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_UDP, isV6) {
+func coalesceUDPPackets(pkt []byte, item *udpGROItem, wi *groToWrite, isV6 bool) coalesceResult {
+	headersLen := int(item.iphLen) + udphLen
+	iov := &wi.iovs[item.outputIdx]
+	if len(*iov) == iovSinglePacketLen {
+		if item.cSumKnownInvalid || !checksumValid((*iov)[iovHeadPacketIdx], item.iphLen, unix.IPPROTO_UDP, isV6) {
 			return coalesceItemInvalidCSum
 		}
 	}
 	if !checksumValid(pkt, item.iphLen, unix.IPPROTO_UDP, isV6) {
 		return coalescePktInvalidCSum
 	}
-	extendBy := len(pkt) - int(headersLen)
-	bufs[item.bufsIndex] = append(bufs[item.bufsIndex], make([]byte, extendBy)...)
-	copy(bufs[item.bufsIndex][bufsOffset+len(pktHead):], pkt[headersLen:])
-
-	item.numMerged++
+	*iov = append(*iov, pkt[headersLen:])
+	item.payloadLen += uint16(len(pkt) - headersLen)
 	return coalesceSuccess
 }
 
 // coalesceTCPPackets attempts to coalesce pkt with the packet described by
-// item, and returns the outcome. This function may swap bufs elements in the
-// event of a prepend as item's bufs index is already being tracked for writing
-// to a Device.
-func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize uint16, seq uint32, pshSet bool, item *tcpGROItem, bufs [][]byte, bufsOffset int, isV6 bool) coalesceResult {
-	var pktHead []byte // the packet that will end up at the front
-	headersLen := item.iphLen + item.tcphLen
-	coalescedLen := len(bufs[item.bufsIndex][bufsOffset:]) + len(pkt) - int(headersLen)
-
-	// Copy data
+// item, and returns the outcome.
+func coalesceTCPPackets(mode canCoalesce, pkt []byte, gsoSize uint16, seq uint32, pshSet bool, item *tcpGROItem, wi *groToWrite, isV6 bool) coalesceResult {
+	headersLen := int(item.iphLen) + int(item.tcphLen)
+	iov := &wi.iovs[item.outputIdx]
 	if mode == coalescePrepend {
-		pktHead = pkt
-		if cap(pkt)-bufsOffset < coalescedLen {
-			// We don't want to allocate a new underlying array if capacity is
-			// too small.
-			return coalesceInsufficientCap
-		}
 		if pshSet {
 			return coalescePSHEnding
 		}
-		if item.numMerged == 0 {
-			if !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_TCP, isV6) {
+		if len(*iov) == iovSinglePacketLen {
+			if !checksumValid((*iov)[iovHeadPacketIdx], item.iphLen, unix.IPPROTO_TCP, isV6) {
 				return coalesceItemInvalidCSum
 			}
 		}
@@ -483,21 +541,15 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize
 			return coalescePktInvalidCSum
 		}
 		item.sentSeq = seq
-		extendBy := coalescedLen - len(pktHead)
-		bufs[pktBuffsIndex] = append(bufs[pktBuffsIndex], make([]byte, extendBy)...)
-		copy(bufs[pktBuffsIndex][bufsOffset+len(pkt):], bufs[item.bufsIndex][bufsOffset+int(headersLen):])
-		// Flip the slice headers in bufs as part of prepend. The index of item
-		// is already being tracked for writing.
-		bufs[item.bufsIndex], bufs[pktBuffsIndex] = bufs[pktBuffsIndex], bufs[item.bufsIndex]
-	} else {
-		pktHead = bufs[item.bufsIndex][bufsOffset:]
-		if cap(pktHead)-bufsOffset < coalescedLen {
-			// We don't want to allocate a new underlying array if capacity is
-			// too small.
-			return coalesceInsufficientCap
+		oldHead := (*iov)[iovHeadPacketIdx]
+		(*iov)[iovHeadPacketIdx] = pkt
+		oldHeadPayload := oldHead[headersLen:]
+		if len(oldHeadPayload) > 0 {
+			*iov = slices.Insert(*iov, iovFirstPayloadFragmentIdx, oldHeadPayload)
 		}
-		if item.numMerged == 0 {
-			if !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_TCP, isV6) {
+	} else {
+		if len(*iov) == iovSinglePacketLen {
+			if !checksumValid((*iov)[iovHeadPacketIdx], item.iphLen, unix.IPPROTO_TCP, isV6) {
 				return coalesceItemInvalidCSum
 			}
 		}
@@ -507,18 +559,16 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize
 		if pshSet {
 			// We are appending a segment with PSH set.
 			item.pshSet = pshSet
-			pktHead[item.iphLen+tcpFlagsOffset] |= tcpFlagPSH
+			(*iov)[iovHeadPacketIdx][item.iphLen+tcpFlagsOffset] |= tcpFlagPSH
 		}
-		extendBy := len(pkt) - int(headersLen)
-		bufs[item.bufsIndex] = append(bufs[item.bufsIndex], make([]byte, extendBy)...)
-		copy(bufs[item.bufsIndex][bufsOffset+len(pktHead):], pkt[headersLen:])
+		*iov = append(*iov, pkt[headersLen:])
 	}
 
 	if gsoSize > item.gsoSize {
 		item.gsoSize = gsoSize
 	}
 
-	item.numMerged++
+	item.payloadLen += uint16(len(pkt) - headersLen)
 	return coalesceSuccess
 }
 
@@ -538,13 +588,12 @@ const (
 	groResultCoalesced
 )
 
-// tcpGRO evaluates the TCP packet at pktI in bufs for coalescing with
+// tcpGRO evaluates the TCP packet for coalescing with
 // existing packets tracked in table. It returns a groResultNoop when no
 // action was taken, groResultTableInsert when the evaluated packet was
 // inserted into table, and groResultCoalesced when the evaluated packet was
 // coalesced with another packet in table.
-func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) groResult {
-	pkt := bufs[pktI][offset:]
+func tcpGRO(pkt []byte, table *tcpGROTable, wi *groToWrite, isV6 bool) groResult {
 	if len(pkt) > maxUint16 {
 		// A valid IPv4 or IPv6 packet will never exceed this.
 		return groResultNoop
@@ -599,7 +648,7 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool)
 		srcAddrOffset = ipv6SrcAddrOffset
 		addrLen = 16
 	}
-	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI)
+	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, wi)
 	if !existing {
 		return groResultTableInsert
 	}
@@ -613,9 +662,9 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool)
 		// sequence number perspective, however once an item is inserted into
 		// the table it is never compared across other items later.
 		item := items[i]
-		can := tcpPacketsCanCoalesce(pkt, uint8(iphLen), uint8(tcphLen), seq, pshSet, gsoSize, item, bufs, offset)
+		can := tcpPacketsCanCoalesce(pkt, uint8(iphLen), uint8(tcphLen), seq, pshSet, gsoSize, item, wi)
 		if can != coalesceUnavailable {
-			result := coalesceTCPPackets(can, pkt, pktI, gsoSize, seq, pshSet, &item, bufs, offset, isV6)
+			result := coalesceTCPPackets(can, pkt, gsoSize, seq, pshSet, &item, wi, isV6)
 			switch result {
 			case coalesceSuccess:
 				table.updateAt(item, i)
@@ -631,16 +680,19 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool)
 		}
 	}
 	// failed to coalesce with any other packets; store the item in the flow
-	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI)
+	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, wi)
 	return groResultTableInsert
 }
 
-// applyTCPCoalesceAccounting updates bufs to account for coalescing based on the
+// applyTCPCoalesceAccounting updates headers to account for coalescing based on the
 // metadata found in table.
-func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) error {
+func applyTCPCoalesceAccounting(wi *groToWrite, table *tcpGROTable) error {
 	for _, items := range table.itemsByFlow {
 		for _, item := range items {
-			if item.numMerged > 0 {
+			iov := wi.iovs[item.outputIdx]
+			pkt := iov[iovHeadPacketIdx]
+			if len(iov) > iovSinglePacketLen {
+				totalLen := uint16(item.iphLen) + uint16(item.tcphLen) + item.payloadLen
 				hdr := virtioNetHdr{
 					flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb
 					hdrLen:     uint16(item.iphLen + item.tcphLen),
@@ -648,21 +700,20 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
 					csumStart:  uint16(item.iphLen),
 					csumOffset: 16,
 				}
-				pkt := bufs[item.bufsIndex][offset:]
 
 				// Recalculate the total len (IPv4) or payload len (IPv6).
 				// Recalculate the (IPv4) header checksum.
 				if item.key.isV6 {
 					hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6
-					binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len
+					binary.BigEndian.PutUint16(pkt[4:], totalLen-uint16(item.iphLen)) // set new IPv6 header payload len
 				} else {
 					hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4
 					pkt[10], pkt[11] = 0, 0
-					binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
-					iphCSum := ^Checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
-					binary.BigEndian.PutUint16(pkt[10:], iphCSum)         // set IPv4 header checksum field
+					binary.BigEndian.PutUint16(pkt[2:], totalLen) // set new total length
+					iphCSum := ^Checksum(pkt[:item.iphLen], 0)    // compute IPv4 header checksum
+					binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field
 				}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
+				err := hdr.encode(iov[iovVirtioNetHdrIdx])
 				if err != nil {
 					return err
 				}
@@ -676,56 +727,51 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
 					addrLen = 16
 					addrOffset = ipv6SrcAddrOffset
 				}
-				srcAddrAt := offset + addrOffset
-				srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
-				dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
-				psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
+				srcAddr := pkt[addrOffset : addrOffset+addrLen]
+				dstAddr := pkt[addrOffset+addrLen : addrOffset+addrLen*2]
+				psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, totalLen-uint16(item.iphLen))
 				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
-			} else {
-				hdr := virtioNetHdr{}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
-				if err != nil {
-					return err
-				}
 			}
 		}
 	}
 	return nil
 }
 
-// applyUDPCoalesceAccounting updates bufs to account for coalescing based on the
+// applyUDPCoalesceAccounting updates headers to account for coalescing based on the
 // metadata found in table.
-func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) error {
+func applyUDPCoalesceAccounting(wi *groToWrite, table *udpGROTable) error {
 	for _, items := range table.itemsByFlow {
 		for _, item := range items {
-			if item.numMerged > 0 {
+			iov := wi.iovs[item.outputIdx]
+			pkt := iov[iovHeadPacketIdx]
+			if len(iov) > iovSinglePacketLen {
+				totalLen := uint16(item.iphLen) + udphLen + item.payloadLen
 				hdr := virtioNetHdr{
 					flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb
-					hdrLen:     uint16(item.iphLen + udphLen),
+					hdrLen:     uint16(item.iphLen) + udphLen,
 					gsoSize:    item.gsoSize,
 					csumStart:  uint16(item.iphLen),
 					csumOffset: 6,
 				}
-				pkt := bufs[item.bufsIndex][offset:]
 
 				// Recalculate the total len (IPv4) or payload len (IPv6).
 				// Recalculate the (IPv4) header checksum.
 				hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_UDP_L4
 				if item.key.isV6 {
-					binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len
+					binary.BigEndian.PutUint16(pkt[4:], totalLen-uint16(item.iphLen)) // set new IPv6 header payload len
 				} else {
 					pkt[10], pkt[11] = 0, 0
-					binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
-					iphCSum := ^Checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
-					binary.BigEndian.PutUint16(pkt[10:], iphCSum)         // set IPv4 header checksum field
+					binary.BigEndian.PutUint16(pkt[2:], totalLen) // set new total length
+					iphCSum := ^Checksum(pkt[:item.iphLen], 0)    // compute IPv4 header checksum
+					binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field
 				}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
+				err := hdr.encode(iov[iovVirtioNetHdrIdx])
 				if err != nil {
 					return err
 				}
 
 				// Recalculate the UDP len field value
-				binary.BigEndian.PutUint16(pkt[item.iphLen+4:], uint16(len(pkt[item.iphLen:])))
+				binary.BigEndian.PutUint16(pkt[item.iphLen+4:], udphLen+item.payloadLen)
 
 				// Calculate the pseudo header checksum and place it at the UDP
 				// checksum offset. Downstream checksum offloading will combine
@@ -736,17 +782,10 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e
 					addrLen = 16
 					addrOffset = ipv6SrcAddrOffset
 				}
-				srcAddrAt := offset + addrOffset
-				srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
-				dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
-				psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
+				srcAddr := pkt[addrOffset : addrOffset+addrLen]
+				dstAddr := pkt[addrOffset+addrLen : addrOffset+addrLen*2]
+				psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, totalLen-uint16(item.iphLen))
 				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
-			} else {
-				hdr := virtioNetHdr{}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
-				if err != nil {
-					return err
-				}
 			}
 		}
 	}
@@ -793,13 +832,12 @@ const (
 	udphLen = 8
 )
 
-// udpGRO evaluates the UDP packet at pktI in bufs for coalescing with
+// udpGRO evaluates the UDP packet for coalescing with
 // existing packets tracked in table. It returns a groResultNoop when no
 // action was taken, groResultTableInsert when the evaluated packet was
 // inserted into table, and groResultCoalesced when the evaluated packet was
 // coalesced with another packet in table.
-func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) groResult {
-	pkt := bufs[pktI][offset:]
+func udpGRO(pkt []byte, table *udpGROTable, wi *groToWrite, isV6 bool) groResult {
 	if len(pkt) > maxUint16 {
 		// A valid IPv4 or IPv6 packet will never exceed this.
 		return groResultNoop
@@ -840,7 +878,7 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
 		srcAddrOffset = ipv6SrcAddrOffset
 		addrLen = 16
 	}
-	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, pktI)
+	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, wi)
 	if !existing {
 		return groResultTableInsert
 	}
@@ -848,10 +886,10 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
 	// for a given flow. We must also always insert a new item, or successfully
 	// coalesce with an existing item, for the same reason.
 	item := items[len(items)-1]
-	can := udpPacketsCanCoalesce(pkt, uint8(iphLen), gsoSize, item, bufs, offset)
+	can := udpPacketsCanCoalesce(pkt, gsoSize, item, wi)
 	var pktCSumKnownInvalid bool
 	if can == coalesceAppend {
-		result := coalesceUDPPackets(pkt, &item, bufs, offset, isV6)
+		result := coalesceUDPPackets(pkt, &item, wi, isV6)
 		switch result {
 		case coalesceSuccess:
 			table.updateAt(item, len(items)-1)
@@ -868,44 +906,40 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
 		}
 	}
 	// failed to coalesce with any other packets; store the item in the flow
-	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, pktI, pktCSumKnownInvalid)
+	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, wi, pktCSumKnownInvalid)
 	return groResultTableInsert
 }
 
-// handleGRO evaluates bufs for GRO, and writes the indices of the resulting
-// packets into toWrite. toWrite, tcpTable, and udpTable should initially be
+// handleGRO evaluates bufs for GRO, and populates wi with the resulting
+// io vectors for writev. wi, tcpTable, and udpTable should initially be
 // empty (but non-nil), and are passed in to save allocs as the caller may reset
 // and recycle them across vectors of packets. gro indicates if TCP and UDP GRO
 // are supported/enabled.
-func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error {
+func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, wi *groToWrite) error {
 	for i := range bufs {
 		if offset < virtioNetHdrLen || offset > len(bufs[i])-1 {
 			return errors.New("invalid offset")
 		}
+		pkt := bufs[i][offset:]
 		var result groResult
-		switch packetIsGROCandidate(bufs[i][offset:], gro) {
+		switch packetIsGROCandidate(pkt, gro) {
 		case tcp4GROCandidate:
-			result = tcpGRO(bufs, offset, i, tcpTable, false)
+			result = tcpGRO(pkt, tcpTable, wi, false)
 		case tcp6GROCandidate:
-			result = tcpGRO(bufs, offset, i, tcpTable, true)
+			result = tcpGRO(pkt, tcpTable, wi, true)
 		case udp4GROCandidate:
-			result = udpGRO(bufs, offset, i, udpTable, false)
+			result = udpGRO(pkt, udpTable, wi, false)
 		case udp6GROCandidate:
-			result = udpGRO(bufs, offset, i, udpTable, true)
+			result = udpGRO(pkt, udpTable, wi, true)
 		}
 		switch result {
 		case groResultNoop:
-			hdr := virtioNetHdr{}
-			err := hdr.encode(bufs[i][offset-virtioNetHdrLen:])
-			if err != nil {
-				return err
-			}
-			fallthrough
+			wi.appendIov(pkt)
 		case groResultTableInsert:
-			*toWrite = append(*toWrite, i)
+			// already in wi via table insert
 		}
 	}
-	errTCP := applyTCPCoalesceAccounting(bufs, offset, tcpTable)
-	errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable)
+	errTCP := applyTCPCoalesceAccounting(wi, tcpTable)
+	errUDP := applyUDPCoalesceAccounting(wi, udpTable)
 	return errors.Join(errTCP, errUDP)
 }

vendor/github.com/tailscale/wireguard-go/tun/tun.go

@@ -17,18 +17,55 @@ const (
 	EventMTUUpdate
 )
 
-type Device interface {
-	// File returns the file descriptor of the device.
+// ReadPacket describes a packet read by [Reader.Read].
+type ReadPacket struct {
+	// Offset is the starting byte offset.
+	Offset int
+	// Size is the size of the packet.
+	Size int
+}
+
+// ReadPacketSpacing is the number of bytes reserved before the first packet,
+// between adjacent packets, and after the final packet filled by [Reader.Read].
+const ReadPacketSpacing = 64
+
+// Reader reads packets from a single queue of a [Device].
+type Reader interface {
+	// Read reads one or more packets from the queue into slab. On return, it
+	// populates packets and returns the number of entries to evaluate. Those
+	// entries are valid even when err is non-nil. Callers must provide at least
+	// [Device.BatchSize] entries.
+	//
+	// Read reserves [ReadPacketSpacing] bytes before the first packet, between
+	// adjacent packets, and after the final packet. The contents of the reserved
+	// space are unspecified; the [Device] may use them while Read is executing.
+	// Callers must provide a slab of at least 2*[ReadPacketSpacing] bytes.
+	//
+	// Read returns [ErrTooManySegments] if packets or slab cannot accommodate
+	// all packets produced by the read.
+	Read(slab []byte, packets []ReadPacket) (n int, err error)
+}
+
+// Queue is a single read queue of a [Device]. Distinct Queues of one Device
+// may be used concurrently.
+//
+// A [Device] implementation may return read-only Queues in addition to those
+// backed by file descriptors. [MultiQueueDevice.WriteTo] is the way to
+// dispatch writes to the queues that accept them.
+type Queue interface {
+	Reader
+
+	// File returns the queue's file descriptor, or nil if the queue is not
+	// backed by one.
 	File() *os.File
+}
 
-	// Read one or more packets from the Device (without any additional headers).
-	// On a successful read it returns the number of packets read, and sets
-	// packet lengths within the sizes slice. len(sizes) must be >= len(bufs).
-	// A nonzero offset can be used to instruct the Device on where to begin
-	// reading into each element of the bufs slice.
-	Read(bufs [][]byte, sizes []int, offset int) (n int, err error)
+type Device interface {
+	// Queue is the Device's first queue. A [MultiQueueDevice] exposes the
+	// rest.
+	Queue
 
-	// Write one or more packets to the device (without any additional headers).
+	// Write one or more packets to the Device.
 	// On a successful write it returns the number of packets written. A nonzero
 	// offset can be used to instruct the Device on where to begin writing from
 	// each packet contained within the bufs slice.
@@ -52,6 +89,82 @@ type Device interface {
 	BatchSize() int
 }
 
+// MultiQueueDevice is a Device that exposes more than one kernel queue, each
+// backed by its own file descriptor. The count is fixed for the Device's
+// lifetime.
+type MultiQueueDevice interface {
+	Device
+
+	// Queues returns one [Queue] per kernel queue with at least one entry.
+	// Entry 0 is equivalent to the Device's own [Reader.Read].
+	Queues() []Queue
+
+	// WriteTo is [Device.Write] directed at one of the Device's write queues.
+	// Data written with the same flow id lands on the same write queue.
+	WriteTo(flow int, bufs [][]byte, offset int) (int, error)
+}
+
+// WriteToOf returns dev's queue-aware write, or [Device.Write] for a Device
+// with one write queue.
+func WriteToOf(dev Device) func(flow int, bufs [][]byte, offset int) (int, error) {
+	if w, ok := dev.(MultiQueueDevice); ok {
+		return w.WriteTo
+	}
+	return func(_ int, bufs [][]byte, offset int) (int, error) {
+		return dev.Write(bufs, offset)
+	}
+}
+
+// QueuesOf returns dev's queues, or dev itself as the single queue. The result
+// always has at least one entry.
+func QueuesOf(dev Device) []Queue {
+	if mq, ok := dev.(MultiQueueDevice); ok {
+		if qs := mq.Queues(); len(qs) > 0 {
+			return qs
+		}
+	}
+	return []Queue{dev}
+}
+
+// An Option configures a [Device] at creation time.
+type Option interface {
+	apply(*config)
+}
+
+type optionFunc func(*config)
+
+func (f optionFunc) apply(config *config) {
+	f(config)
+}
+
+type config struct {
+	queues int // Tun fd's in IFF_MULTI_QUEUE group to open.
+}
+
+func defaultConfig() config {
+	return config{
+		queues: 1,
+	}
+}
+
+// WithQueues requests that the [Device] be created with n queues. n must be at
+// least 1.
+//
+// Only Linux implements multiqueue TUN. Errors are returned if more queues
+// are requested than the Linux kernel supports (at the time of writing,
+// v7.0 has MAX_TAP_QUEUES at 256) or if the per-process limit was reached.
+//
+// Implementations may choose to return more read queues than was requested.
+// Callers must consult [QueuesOf] rather than assume they got n.
+//
+// EXPERIMENTAL: Current implementation introduces nonce reordering with more
+// than one queue.
+func WithQueues(n int) Option {
+	return optionFunc(func(config *config) {
+		config.queues = n
+	})
+}
+
 // GRODevice is a Device extended with methods for disabling GRO. Certain OS
 // versions may have offload bugs. Where these bugs negatively impact throughput
 // or break connectivity entirely we can use these methods to disable the

vendor/github.com/tailscale/wireguard-go/tun/tun_bsd.go

@@ -0,0 +1,23 @@
+//go:build freebsd || openbsd || darwin
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package tun
+
+import (
+	"fmt"
+	"os"
+)
+
+func createTUNFromFilesNoMQ(files []*os.File, mtu int) (Device, error) {
+	if len(files) != 1 {
+		for _, f := range files {
+			f.Close()
+		}
+		return nil, fmt.Errorf("got %d TUN queue files: multiqueue TUN is only supported on Linux", len(files))
+	}
+	return CreateTUNFromFile(files[0], mtu)
+}

vendor/github.com/tailscale/wireguard-go/tun/tun_darwin.go

@@ -98,7 +98,7 @@ func (tun *NativeTun) routineRouteListener(tunIfindex int) {
 	}
 }
 
-func CreateTUN(name string, mtu int) (Device, error) {
+func CreateTUN(name string, mtu int, _ ...Option) (Device, error) {
 	ifIndex := -1
 	if name != "utun" {
 		_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
@@ -148,6 +148,10 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	return tun, err
 }
 
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	return createTUNFromFilesNoMQ(files, mtu)
+}
+
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 	tun := &NativeTun{
 		tunFile: file,
@@ -217,7 +221,7 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	// TODO: the BSDs look very similar in Read() and Write(). They should be
 	// collapsed, with platform-specific files containing the varying parts of
 	// their implementations.
@@ -225,12 +229,13 @@ func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error)
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		buf := bufs[0][offset-4:]
+		buf := slab[ReadPacketSpacing-4 : len(slab)-ReadPacketSpacing]
 		n, err := tun.tunFile.Read(buf[:])
 		if n < 4 {
 			return 0, err
 		}
-		sizes[0] = n - 4
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n - 4
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_freebsd.go

@@ -159,7 +159,7 @@ func tunDestroy(name string) error {
 	return nil
 }
 
-func CreateTUN(name string, mtu int) (Device, error) {
+func CreateTUN(name string, mtu int, _ ...Option) (Device, error) {
 	if len(name) > unix.IFNAMSIZ-1 {
 		return nil, errors.New("interface name too long")
 	}
@@ -261,6 +261,10 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	return CreateTUNFromFile(tunFile, mtu)
 }
 
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	return createTUNFromFilesNoMQ(files, mtu)
+}
+
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 	tun := &NativeTun{
 		tunFile: file,
@@ -333,17 +337,18 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		buf := bufs[0][offset-4:]
+		buf := slab[ReadPacketSpacing-4 : len(slab)-ReadPacketSpacing]
 		n, err := tun.tunFile.Read(buf[:])
 		if n < 4 {
 			return 0, err
 		}
-		sizes[0] = n - 4
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n - 4
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_linux.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"os"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 	"unsafe"
@@ -28,12 +29,14 @@ const (
 )
 
 type NativeTun struct {
-	tunFile                 *os.File
-	index                   int32      // if index
-	errors                  chan error // async error handling
-	events                  chan Event // device related events
-	netlinkSock             int
-	netlinkCancel           *rwcancel.RWCancel
+	queues []*tunQueue // queues[0] always exists and is used by Read and Write
+
+	index         int32      // if index
+	errors        chan error // async error handling
+	events        chan Event // device related events
+	netlinkSock   int
+	netlinkCancel *rwcancel.RWCancel
+
 	hackListenerClosed      sync.Mutex
 	statusListenersShutdown chan struct{}
 	batchSize               int
@@ -45,14 +48,52 @@ type NativeTun struct {
 	nameCache string    // name of interface
 	nameErr   error
 
-	readOpMu sync.Mutex                    // readOpMu guards readBuff
-	readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr
+	gro atomic.Int32 // groDisablementFlags for cross-queue access
+}
+
+// tunQueue is a single kernel queue of a tun device, and is the [Queue]
+// handed out by [NativeTun.Queues].
+// An individual tunQueue is not concurrency-safe, but distinct queues can
+// be read or written concurrently.
+type tunQueue struct {
+	tun     *NativeTun
+	file    *os.File
+	rawConn syscall.RawConn
 
 	writeOpMu   sync.Mutex // writeOpMu guards the following fields
-	toWrite     []int
+	toWrite     groToWrite
 	tcpGROTable *tcpGROTable
 	udpGROTable *udpGROTable
-	gro         groDisablementFlags
+}
+
+// appendQueue wraps file as a new queue of tun. It must be called before tun is
+// published to other goroutines, and only appends on success.
+func appendQueue(tun *NativeTun, file *os.File) error {
+	q := &tunQueue{
+		tun:         tun,
+		file:        file,
+		tcpGROTable: newTCPGROTable(),
+		udpGROTable: newUDPGROTable(),
+		toWrite:     newGROToWrite(),
+	}
+	var err error
+	q.rawConn, err = file.SyscallConn()
+	if err != nil {
+		return err
+	}
+	tun.queues = append(tun.queues, q)
+	return nil
+}
+
+func (q *tunQueue) File() *os.File { return q.file }
+
+// Queues implements [MultiQueueDevice].
+func (tun *NativeTun) Queues() []Queue {
+	qs := make([]Queue, 0, len(tun.queues))
+	for _, q := range tun.queues {
+		qs = append(qs, q)
+	}
+	return qs
 }
 
 type groDisablementFlags int
@@ -62,24 +103,16 @@ const (
 	udpGRODisabled
 )
 
-func (g *groDisablementFlags) disableTCPGRO() {
-	*g |= tcpGRODisabled
-}
-
 func (g *groDisablementFlags) canTCPGRO() bool {
 	return (*g)&tcpGRODisabled == 0
 }
 
-func (g *groDisablementFlags) disableUDPGRO() {
-	*g |= udpGRODisabled
-}
-
 func (g *groDisablementFlags) canUDPGRO() bool {
 	return (*g)&udpGRODisabled == 0
 }
 
 func (tun *NativeTun) File() *os.File {
-	return tun.tunFile
+	return tun.queues[0].file
 }
 
 func (tun *NativeTun) routineHackListener() {
@@ -93,7 +126,7 @@ func (tun *NativeTun) routineHackListener() {
 		down = 2
 	)
 	for {
-		sysconn, err := tun.tunFile.SyscallConn()
+		sysconn, err := tun.File().SyscallConn()
 		if err != nil {
 			return
 		}
@@ -326,9 +359,13 @@ func (tun *NativeTun) initNameCache() {
 }
 
 func (tun *NativeTun) nameSlow() (string, error) {
-	sysconn, err := tun.tunFile.SyscallConn()
+	return tunNameOf(tun.File())
+}
+
+func tunNameOf(file *os.File) (string, error) {
+	sysconn, err := file.SyscallConn()
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to get name of TUN device: %w", err)
 	}
 	var ifr [ifReqSize]byte
 	var errno syscall.Errno
@@ -350,35 +387,73 @@ func (tun *NativeTun) nameSlow() (string, error) {
 }
 
 func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
-	tun.writeOpMu.Lock()
+	return tun.queues[0].write(bufs, offset)
+}
+
+// WriteTo implements [MultiQueueDevice].
+func (tun *NativeTun) WriteTo(flow int, bufs [][]byte, offset int) (int, error) {
+	return tun.queues[uint(flow)%uint(len(tun.queues))].write(bufs, offset)
+}
+
+func (q *tunQueue) write(bufs [][]byte, offset int) (int, error) {
+	q.writeOpMu.Lock()
 	defer func() {
-		tun.tcpGROTable.reset()
-		tun.udpGROTable.reset()
-		tun.writeOpMu.Unlock()
+		q.tcpGROTable.reset()
+		q.udpGROTable.reset()
+		q.toWrite.reset()
+		q.writeOpMu.Unlock()
 	}()
 	var (
 		errs  error
 		total int
 	)
-	tun.toWrite = tun.toWrite[:0]
-	if tun.vnetHdr {
-		err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite)
-		if err != nil {
-			return 0, err
-		}
-		offset -= virtioNetHdrLen
-	} else {
+	if !q.tun.vnetHdr {
 		for i := range bufs {
-			tun.toWrite = append(tun.toWrite, i)
+			n, err := q.file.Write(bufs[i][offset:])
+			if errors.Is(err, syscall.EBADFD) {
+				return total, os.ErrClosed
+			}
+			if err != nil {
+				errs = errors.Join(errs, err)
+			} else {
+				total += n
+			}
 		}
+		return total, errs
 	}
-	for _, bufsI := range tun.toWrite {
-		n, err := tun.tunFile.Write(bufs[bufsI][offset:])
-		if errors.Is(err, syscall.EBADFD) {
+	err := handleGRO(
+		bufs,
+		offset,
+		q.tcpGROTable,
+		q.udpGROTable,
+		groDisablementFlags(q.tun.gro.Load()),
+		&q.toWrite,
+	)
+	if err != nil {
+		return 0, err
+	}
+	for _, nb := range q.toWrite.iovs {
+		var werr error
+		var n int
+		err := q.rawConn.Write(func(fd uintptr) bool {
+			for {
+				n, werr = unix.Writev(int(fd), nb)
+				if errors.Is(werr, syscall.EINTR) {
+					continue // quick retry on interrupt, EINTR is never returned with partial writes
+				}
+				return !errors.Is(werr, syscall.EAGAIN) // poller retry on "would block"
+			}
+		})
+		// err is a poller error (e.g. fd closed before the syscall)
+		// werr is the Writev syscall error itself.
+		if err != nil {
+			return total, err
+		}
+		if errors.Is(werr, syscall.EBADFD) {
 			return total, os.ErrClosed
 		}
-		if err != nil {
-			errs = errors.Join(errs, err)
+		if werr != nil {
+			errs = errors.Join(errs, werr)
 		} else {
 			total += n
 		}
@@ -386,10 +461,11 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
 	return total, errs
 }
 
-// handleVirtioRead splits in into bufs, leaving offset bytes at the front of
-// each buffer. It mutates sizes to reflect the size of each element of bufs,
-// and returns the number of packets read.
-func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, error) {
+// handleVirtioRead splits in into slab, leaving [ReadPacketSpacing] bytes at
+// the front of the first packet, between every adjacent packet, and at the end
+// of the last packet. It mutates packets to reflect the size and offset of each
+// packet in slab, and returns the number of packets read.
+func handleVirtioRead(in []byte, slab []byte, packets []ReadPacket) (int, error) {
 	var hdr virtioNetHdr
 	err := hdr.decode(in)
 	if err != nil {
@@ -421,31 +497,39 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e
 		options.HdrLen = options.CsumStart + tcpHLen
 	}
 
-	return GSOSplit(in, options, bufs, sizes, offset)
+	return GSOSplit(in, options, slab, packets, ReadPacketSpacing)
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
-	tun.readOpMu.Lock()
-	defer tun.readOpMu.Unlock()
+// assert that [ReadPacketSpacing] is >= [virtioNetHdrLen], as [NativeTun.Read]
+// assumes so, and uses headroom for [virtioNetHdr].
+const _ = uint(ReadPacketSpacing - virtioNetHdrLen)
+
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
+	return tun.queues[0].Read(slab, packets)
+}
+
+func (q *tunQueue) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
-	case err := <-tun.errors:
+	case err := <-q.tun.errors:
 		return 0, err
 	default:
-		readInto := bufs[0][offset:]
-		if tun.vnetHdr {
-			readInto = tun.readBuff[:]
+		start := ReadPacketSpacing
+		if q.tun.vnetHdr {
+			start -= virtioNetHdrLen
 		}
-		n, err := tun.tunFile.Read(readInto)
+		readInto := slab[start : len(slab)-ReadPacketSpacing]
+		n, err := q.file.Read(readInto)
 		if errors.Is(err, syscall.EBADFD) {
 			err = os.ErrClosed
 		}
 		if err != nil {
 			return 0, err
 		}
-		if tun.vnetHdr {
-			return handleVirtioRead(readInto[:n], bufs, sizes, offset)
+		if q.tun.vnetHdr {
+			return handleVirtioRead(readInto[:n], slab, packets)
 		} else {
-			sizes[0] = n
+			packets[0].Size = n
+			packets[0].Offset = ReadPacketSpacing
 			return 1, nil
 		}
 	}
@@ -466,7 +550,11 @@ func (tun *NativeTun) Close() error {
 		} else if tun.events != nil {
 			close(tun.events)
 		}
-		err2 = tun.tunFile.Close()
+		for _, q := range tun.queues {
+			if cerr := q.file.Close(); cerr != nil && err2 == nil {
+				err2 = cerr
+			}
+		}
 	})
 	if err1 != nil {
 		return err1
@@ -481,17 +569,13 @@ func (tun *NativeTun) BatchSize() int {
 // DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface
 // for cases where it should be called.
 func (tun *NativeTun) DisableUDPGRO() {
-	tun.writeOpMu.Lock()
-	tun.gro.disableUDPGRO()
-	tun.writeOpMu.Unlock()
+	tun.gro.Or(int32(udpGRODisabled))
 }
 
 // DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface
 // for cases where it should be called.
 func (tun *NativeTun) DisableTCPGRO() {
-	tun.writeOpMu.Lock()
-	tun.gro.disableTCPGRO()
-	tun.writeOpMu.Unlock()
+	tun.gro.Or(int32(tcpGRODisabled))
 }
 
 const (
@@ -501,7 +585,7 @@ const (
 )
 
 func (tun *NativeTun) initFromFlags(name string) error {
-	sc, err := tun.tunFile.SyscallConn()
+	sc, err := tun.File().SyscallConn()
 	if err != nil {
 		return err
 	}
@@ -530,7 +614,7 @@ func (tun *NativeTun) initFromFlags(name string) error {
 			// tunUDPOffloads were added in Linux v6.2. We do not return an
 			// error if they are unsupported at runtime.
 			if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil {
-				tun.gro.disableUDPGRO()
+				tun.DisableUDPGRO()
 			}
 		} else {
 			tun.batchSize = 1
@@ -541,8 +625,8 @@ func (tun *NativeTun) initFromFlags(name string) error {
 	return err
 }
 
-// CreateTUN creates a Device with the provided name and MTU.
-func CreateTUN(name string, mtu int) (Device, error) {
+// openTUNFile opens /dev/net/tun and attaches it to name.
+func openTUNFile(name string, flags uint16) (*os.File, error) {
 	nfd, err := unix.Open(cloneDevicePath, unix.O_RDWR|unix.O_CLOEXEC, 0)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -553,13 +637,13 @@ func CreateTUN(name string, mtu int) (Device, error) {
 
 	ifr, err := unix.NewIfreq(name)
 	if err != nil {
+		unix.Close(nfd)
 		return nil, err
 	}
-	// IFF_VNET_HDR enables the "tun status hack" via routineHackListener()
-	// where a null write will return EINVAL indicating the TUN is up.
-	ifr.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR)
+	ifr.SetUint16(flags)
 	err = unix.IoctlIfreq(nfd, unix.TUNSETIFF, ifr)
 	if err != nil {
+		unix.Close(nfd)
 		return nil, err
 	}
 
@@ -570,46 +654,122 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	}
 
 	// Note that the above -- open,ioctl,nonblock -- must happen prior to handing it to netpoll as below this line.
+	return os.NewFile(uintptr(nfd), cloneDevicePath), nil
+}
 
-	fd := os.NewFile(uintptr(nfd), cloneDevicePath)
-	return CreateTUNFromFile(fd, mtu)
+// CreateTUN creates a Device with the provided name and MTU.
+func CreateTUN(name string, mtu int, opts ...Option) (Device, error) {
+	config := defaultConfig()
+	for _, opt := range opts {
+		opt.apply(&config)
+	}
+	if config.queues < 1 {
+		return nil, fmt.Errorf("%d tun queues requested, must be at least 1", config.queues)
+	}
+	// IFF_VNET_HDR enables the "tun status hack" via routineHackListener()
+	// where a null write will return EINVAL indicating the TUN is up.
+	flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR)
+	if config.queues > 1 {
+		// all members must declare multiqueue
+		flags |= unix.IFF_MULTI_QUEUE
+	}
+	first, err := openTUNFile(name, flags)
+	if err != nil {
+		return nil, err
+	}
+	files := []*os.File{first}
+	closeAll := func() {
+		for _, f := range files {
+			f.Close()
+		}
+	}
+	if config.queues > 1 {
+		// Resolve name wildcards (%d) to not spawn new groups.
+		resolved, err := tunNameOf(first)
+		if err != nil {
+			closeAll()
+			return nil, err
+		}
+		for i := range config.queues - 1 {
+			f, err := openTUNFile(resolved, flags)
+			if err != nil {
+				closeAll()
+				return nil, fmt.Errorf("attaching extra tun queue %d of %d: %w", i, config.queues-1, err)
+			}
+			files = append(files, f)
+		}
+	}
+	return CreateTUNFromFiles(files, mtu)
 }
 
 // CreateTUNFromFile creates a Device from an os.File with the provided MTU.
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
+	return CreateTUNFromFiles([]*os.File{file}, mtu)
+}
+
+// CreateTUNFromFiles creates a Device with the provided MTU from one file per
+// kernel queue. It satisfies [MultiQueueDevice] when there is more than one.
+// Files must be distinct and ownership transfers.
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	if len(files) == 0 {
+		return nil, errors.New("CreateTUNFromFiles requires at least one file")
+	}
 	tun := &NativeTun{
-		tunFile:                 file,
 		events:                  make(chan Event, 5),
 		errors:                  make(chan error, 5),
 		statusListenersShutdown: make(chan struct{}),
-		tcpGROTable:             newTCPGROTable(),
-		udpGROTable:             newUDPGROTable(),
-		toWrite:                 make([]int, 0, conn.IdealBatchSize),
+	}
+	// Append every queue before spawning routineHackListener et al.
+	for _, file := range files {
+		if err := appendQueue(tun, file); err != nil {
+			for _, f := range files {
+				f.Close()
+			}
+			return nil, err
+		}
 	}
 
 	name, err := tun.Name()
 	if err != nil {
+		tun.Close()
 		return nil, err
 	}
+	// Verify that all fds are attached to the same interface.
+	for i, file := range files[1:] {
+		got, err := tunNameOf(file)
+		if err != nil {
+			tun.Close()
+			return nil, fmt.Errorf("tun queue %d: %w", i+1, err)
+		}
+		if got != name {
+			tun.Close()
+			return nil, fmt.Errorf("tun queue %d is attached to %q, want %q", i+1, got, name)
+		}
+	}
 
-	err = tun.initFromFlags(name)
-	if err != nil {
+	// Offload is negotiated once for the whole device, on queue 0.
+	if err := tun.initFromFlags(name); err != nil {
+		tun.Close()
 		return nil, err
 	}
 
 	// start event listener
 	tun.index, err = getIFIndex(name)
 	if err != nil {
+		tun.Close()
 		return nil, err
 	}
 
 	tun.netlinkSock, err = createNetlinkSocket()
 	if err != nil {
+		tun.Close()
 		return nil, err
 	}
 	tun.netlinkCancel, err = rwcancel.NewRWCancel(tun.netlinkSock)
 	if err != nil {
+		// No goroutine owns netlinkSock yet, and tun.Close won't reach it.
 		unix.Close(tun.netlinkSock)
+		tun.Close()
 		return nil, err
 	}
 
@@ -619,7 +779,7 @@ func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 
 	err = tun.setMTU(mtu)
 	if err != nil {
-		unix.Close(tun.netlinkSock)
+		tun.Close()
 		return nil, err
 	}
 
@@ -629,26 +789,76 @@ func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 // CreateUnmonitoredTUNFromFD creates a Device from the provided file
 // descriptor.
 func CreateUnmonitoredTUNFromFD(fd int) (Device, string, error) {
-	err := unix.SetNonblock(fd, true)
-	if err != nil {
-		return nil, "", err
+	return CreateUnmonitoredTUNFromFDs([]int{fd})
+}
+
+// CreateUnmonitoredTUNFromFDs creates a Device from one descriptor per kernel
+// queue. It satisfies [MultiQueueDevice] when there is more than one.
+func CreateUnmonitoredTUNFromFDs(fds []int) (Device, string, error) {
+	if len(fds) == 0 {
+		return nil, "", errors.New("CreateUnmonitoredTUNFromFDs requires at least one fd")
+	}
+	// Error if passed duplicate FDs.
+	var dupes []int
+	set := map[int]struct{}{}
+	for _, v := range fds {
+		if _, dup := set[v]; dup {
+			dupes = append(dupes, v)
+			continue
+		}
+		set[v] = struct{}{}
+	}
+	if len(dupes) != 0 {
+		for fd := range set {
+			unix.Close(fd)
+		}
+		return nil, "", fmt.Errorf("passed duplicate file descriptors: %v", dupes)
 	}
-	file := os.NewFile(uintptr(fd), "/dev/tun")
 	tun := &NativeTun{
-		tunFile:     file,
-		events:      make(chan Event, 5),
-		errors:      make(chan error, 5),
-		tcpGROTable: newTCPGROTable(),
-		udpGROTable: newUDPGROTable(),
-		toWrite:     make([]int, 0, conn.IdealBatchSize),
+		events: make(chan Event, 5),
+		errors: make(chan error, 5),
 	}
+	for i, fd := range fds {
+		// See openTUNFile: this must precede os.NewFile.
+		if err := unix.SetNonblock(fd, true); err != nil {
+			tun.Close()
+			for _, rest := range fds[i:] {
+				unix.Close(rest)
+			}
+			return nil, "", err
+		}
+		file := os.NewFile(uintptr(fd), "/dev/tun")
+		if err := appendQueue(tun, file); err != nil {
+			file.Close() // not saved, so tun.Close won't reach it
+			tun.Close()
+			for _, rest := range fds[i+1:] {
+				unix.Close(rest)
+			}
+			return nil, "", err
+		}
+	}
+
 	name, err := tun.Name()
 	if err != nil {
+		tun.Close()
 		return nil, "", err
 	}
-	err = tun.initFromFlags(name)
-	if err != nil {
+	for i, q := range tun.queues[1:] {
+		got, err := tunNameOf(q.file)
+		if err != nil {
+			tun.Close()
+			return nil, "", fmt.Errorf("tun queue %d: %w", i+1, err)
+		}
+		if got != name {
+			tun.Close()
+			return nil, "", fmt.Errorf("tun queue %d is attached to %q, want %q", i+1, got, name)
+		}
+	}
+
+	// Offload is negotiated once for the whole device, on queue 0.
+	if err := tun.initFromFlags(name); err != nil {
+		tun.Close()
 		return nil, "", err
 	}
-	return tun, name, err
+	return tun, name, nil
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_openbsd.go

@@ -101,7 +101,7 @@ func (tun *NativeTun) routineRouteListener(tunIfindex int) {
 	}
 }
 
-func CreateTUN(name string, mtu int) (Device, error) {
+func CreateTUN(name string, mtu int, _ ...Option) (Device, error) {
 	ifIndex := -1
 	if name != "tun" {
 		_, err := fmt.Sscanf(name, "tun%d", &ifIndex)
@@ -140,6 +140,10 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	return tun, err
 }
 
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	return createTUNFromFilesNoMQ(files, mtu)
+}
+
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 	tun := &NativeTun{
 		tunFile: file,
@@ -204,17 +208,18 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		buf := bufs[0][offset-4:]
+		buf := slab[ReadPacketSpacing-4 : len(slab)-ReadPacketSpacing]
 		n, err := tun.tunFile.Read(buf[:])
 		if n < 4 {
 			return 0, err
 		}
-		sizes[0] = n - 4
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n - 4
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_plan9.go

@@ -23,7 +23,7 @@ type NativeTun struct {
 	closeOnce sync.Once
 }
 
-func CreateTUN(_ string, mtu int) (Device, error) {
+func CreateTUN(_ string, mtu int, _ ...Option) (Device, error) {
 	ctl, err := os.OpenFile("/net/ipifc/clone", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -81,18 +81,18 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		n, err := tun.dataFile.Read(bufs[0][offset:])
-		if n == 1 && bufs[0][offset] == 0 {
+		n, err := tun.dataFile.Read(slab[ReadPacketSpacing : len(slab)-ReadPacketSpacing])
+		if n == 1 && slab[ReadPacketSpacing] == 0 {
 			// EOF
-			err = io.EOF
-			n = 0
+			return 0, io.EOF
 		}
-		sizes[0] = n
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_windows.go

@@ -59,7 +59,7 @@ func nanotime() int64
 
 // CreateTUN creates a Wintun interface with the given name. Should a Wintun
 // interface with the same name exist, it is reused.
-func CreateTUN(ifname string, mtu int) (Device, error) {
+func CreateTUN(ifname string, mtu int, _ ...Option) (Device, error) {
 	return CreateTUNWithRequestedGUID(ifname, WintunStaticRequestedGUID, mtu)
 }
 
@@ -144,7 +144,7 @@ func (tun *NativeTun) BatchSize() int {
 
 // Note: Read() and Write() assume the caller comes only from a single thread; there's no locking.
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	tun.running.Add(1)
 	defer tun.running.Done()
 retry:
@@ -160,11 +160,16 @@ retry:
 		packet, err := tun.session.ReceivePacket()
 		switch err {
 		case nil:
-			packetSize := len(packet)
-			copy(bufs[0][offset:], packet)
-			sizes[0] = packetSize
+			dst := slab[ReadPacketSpacing : len(slab)-ReadPacketSpacing]
+			if len(packet) > len(dst) {
+				tun.session.ReleaseReceivePacket(packet)
+				return 0, ErrTooManySegments
+			}
+			n := copy(dst, packet)
+			packets[0].Offset = ReadPacketSpacing
+			packets[0].Size = n
 			tun.session.ReleaseReceivePacket(packet)
-			tun.rate.update(uint64(packetSize))
+			tun.rate.update(uint64(n))
 			return 1, nil
 		case windows.ERROR_NO_MORE_ITEMS:
 			if !shouldSpin || uint64(nanotime()-start) >= spinloopDuration {