Update

vendor/github.com/tailscale/certstore/certstore_darwin.go

@@ -155,24 +155,31 @@ func (i *macIdentity) CertificateChain() ([]*x509.Certificate, error) {
 	policy := C.SecPolicyCreateSSL(0, nilCFStringRef)
 
 	var trustRef C.SecTrustRef
-	if err := osStatusError(C.SecTrustCreateWithCertificates(C.CFTypeRef(certRef), C.CFTypeRef(policy), &trustRef)); err != nil {
+	osStatusResult := C.SecTrustCreateWithCertificates(C.CFTypeRef(certRef), C.CFTypeRef(policy), &trustRef)
+	if err := osStatusError(osStatusResult); err != nil {
 		return nil, err
 	}
 	defer C.CFRelease(C.CFTypeRef(trustRef))
 
-	var status C.SecTrustResultType
-	if err := osStatusError(C.SecTrustEvaluate(trustRef, &status)); err != nil {
-		return nil, err
+	// Evaluate trust to populate the certificate chain; ignore the trust
+	// result since we only need the chain structure, not trust validation.
+	var cfTrustErr C.CFErrorRef
+	C.SecTrustEvaluateWithError(trustRef, &cfTrustErr)
+	if cfTrustErr != nilCFErrorRef {
+		// ignore the error since we only care about the chain, not trust validation.
 	}
 
-	var (
-		nchain = C.SecTrustGetCertificateCount(trustRef)
-		chain  = make([]*x509.Certificate, 0, int(nchain))
-	)
+	certChain := C.SecTrustCopyCertificateChain(trustRef)
+	if certChain == nilCFArrayRef {
+		return nil, errors.New("error getting certificate chain")
+	}
+	defer C.CFRelease(C.CFTypeRef(certChain))
 
-	for i := C.CFIndex(0); i < nchain; i++ {
-		// TODO: do we need to release these?
-		chainCertref := C.SecTrustGetCertificateAtIndex(trustRef, i)
+	nchain := C.CFArrayGetCount(certChain)
+	chain := make([]*x509.Certificate, 0, int(nchain))
+
+	for j := C.CFIndex(0); j < nchain; j++ {
+		chainCertref := C.SecCertificateRef(C.CFArrayGetValueAtIndex(certChain, j))
 		if chainCertref == nilSecCertificateRef {
 			return nil, errors.New("nil certificate in chain")
 		}

vendor/github.com/tailscale/hujson/README.md

@@ -1,5 +1,7 @@
 # HuJSON - "Human JSON" ([JWCC](https://nigeltao.github.io/blog/2021/json-with-commas-comments.html))
 
+[![Go Reference](https://pkg.go.dev/badge/github.com/tailscale/hujson.svg)](https://pkg.go.dev/github.com/tailscale/hujson)
+
 The `github.com/tailscale/hujson` package implements
 the [JWCC](https://nigeltao.github.io/blog/2021/json-with-commas-comments.html) extension
 of [standard JSON](https://datatracker.ietf.org/doc/html/rfc8259).
@@ -15,6 +17,14 @@ For details, see the JWCC docs at:
 
 https://nigeltao.github.io/blog/2021/json-with-commas-comments.html
 
+## Formatting with hujsonfmt
+
+`hujsonfmt` is a program that formats HuJSON files. Install it by running:
+
+```
+go install github.com/tailscale/hujson/cmd/hujsonfmt@latest
+```
+
 ## Visual Studio Code association
 
 Visual Studio Code supports a similar `jsonc` (JSON with comments) format. To

vendor/github.com/tailscale/hujson/format.go

@@ -88,21 +88,6 @@ func (v *Value) Format() {
 	v.UpdateOffsets()
 }
 
-// Range iterates through a Value in depth-first order and
-// calls f for each value (including the root value).
-// It stops iteration when f returns false.
-func (v *Value) Range(f func(v *Value) bool) bool {
-	if !f(v) {
-		return false
-	}
-	if comp, ok := v.Value.(composite); ok {
-		return comp.rangeValues(func(v2 *Value) bool {
-			return v2.Range(f)
-		})
-	}
-	return true
-}
-
 // normalize performs simple normalization changes. In particular, it:
 //   - normalizes strings,
 //   - normalizes empty objects and arrays as simply {} or [],
@@ -129,15 +114,16 @@ func (v *Value) normalize() bool {
 
 		// If there is only whitespace between the name and colon,
 		// or between the value and comma, then remove the whitespace.
-		v2.rangeValues(func(v *Value) bool {
-			if !v.AfterExtra.hasComment() {
-				v.AfterExtra = nil
+		for v3 := range v2.allValues() {
+			if !v3.AfterExtra.hasComment() {
+				v3.AfterExtra = nil
 			}
-			return true
-		})
+		}
 
 		// Normalize all sub-values.
-		v2.rangeValues((*Value).normalize)
+		for v3 := range v2.allValues() {
+			v3.normalize()
+		}
 	}
 	return true
 }
@@ -583,7 +569,9 @@ func (v *Value) alignObjectValues() bool {
 
 	// Recursively align all sub-objects.
 	if comp, ok := v.Value.(composite); ok {
-		comp.rangeValues((*Value).alignObjectValues)
+		for v2 := range comp.allValues() {
+			v2.alignObjectValues()
+		}
 	}
 	return true
 }
@@ -593,9 +581,11 @@ func (v Value) hasNewline(checkTopLevelExtra bool) bool {
 		return true
 	}
 	if comp, ok := v.Value.(composite); ok {
-		return !comp.rangeValues(func(v *Value) bool {
-			return !v.hasNewline(true)
-		})
+		for v := range comp.allValues() {
+			if v.hasNewline(true) {
+				return true
+			}
+		}
 	}
 	return false
 }

vendor/github.com/tailscale/hujson/go.work

@@ -1,4 +1,4 @@
-go 1.18
+go 1.26
 
 use (
 	.

vendor/github.com/tailscale/hujson/parse.go

@@ -221,13 +221,24 @@ func consumeExtra(n int, b []byte) (int, error) {
 			n += consumeWhitespace(b[n:])
 		// Skip past comments.
 		case '/':
+			isLineComment := bytes.HasPrefix(b[n:], lineCommentStart)
 			switch nc := consumeComment(b[n:]); {
 			case nc == 0:
 				return n, nil
 			case nc < 0:
+				if isLineComment {
+					if i := indexLineSeparator(b[n:]); i >= 0 {
+						return n + i, newInvalidCharacterError(b[n+i:], "in line comment")
+					}
+				}
 				return n, fmt.Errorf("parsing comment: %w", io.ErrUnexpectedEOF)
 			case !utf8.Valid(b[n : n+nc]):
 				return n, fmt.Errorf("invalid UTF-8 in comment")
+			case isLineComment:
+				if i := indexLineSeparator(b[n : n+nc]); i >= 0 {
+					return n + i, newInvalidCharacterError(b[n+i:], "in line comment")
+				}
+				n += nc
 			default:
 				n += nc
 			}
@@ -245,6 +256,15 @@ func consumeWhitespace(b []byte) (n int) {
 	return n
 }
 
+func indexLineSeparator(b []byte) int {
+	i := bytes.IndexRune(b, '\u2028')
+	j := bytes.IndexRune(b, '\u2029')
+	if i < 0 || (j >= 0 && j < i) {
+		return j
+	}
+	return i
+}
+
 // consumeComment consumes a line or block comment start in b.
 // It returns the length of the comment if valid, otherwise
 // it returns 0 if it is not a comment and -1 if it is invalid.

vendor/github.com/tailscale/hujson/patch.go

@@ -417,8 +417,8 @@ func (b *Extra) extractTrailingcomments(readonly bool) (trailing Extra) {
 
 // classifyComments classifies comments as belonging to the previous element
 // or belonging to the current element such that:
-//	* b[:prevEnd] belongs to the previous element, and
-//	* b[currStart:] belongs to the current element.
+//   - b[:prevEnd] belongs to the previous element, and
+//   - b[currStart:] belongs to the current element.
 //
 // Invariant: prevEnd <= currStart
 func (b Extra) classifyComments() (prevEnd, currStart int) {

vendor/github.com/tailscale/hujson/standard.go

@@ -14,8 +14,10 @@ func (v *Value) isStandard() bool {
 		return false
 	}
 	if comp, ok := v.Value.(composite); ok {
-		if !comp.rangeValues((*Value).isStandard) {
-			return false
+		for v2 := range comp.allValues() {
+			if !v2.isStandard() {
+				return false
+			}
 		}
 		if hasTrailingComma(comp) || !comp.afterExtra().IsStandard() {
 			return false
@@ -41,15 +43,16 @@ func (v *Value) Minimize() {
 	v.minimize()
 	v.UpdateOffsets()
 }
-func (v *Value) minimize() bool {
+func (v *Value) minimize() {
 	v.BeforeExtra = nil
 	if v2, ok := v.Value.(composite); ok {
-		v2.rangeValues((*Value).minimize)
+		for v3 := range v2.allValues() {
+			v3.minimize()
+		}
 		setTrailingComma(v2, false)
 		*v2.afterExtra() = nil
 	}
 	v.AfterExtra = nil
-	return true
 }
 
 // Standardize strips any features specific to HuJSON from v,
@@ -60,10 +63,12 @@ func (v *Value) Standardize() {
 	v.standardize()
 	v.UpdateOffsets() // should be noop if offsets are already correct
 }
-func (v *Value) standardize() bool {
+func (v *Value) standardize() {
 	v.BeforeExtra.standardize()
 	if comp, ok := v.Value.(composite); ok {
-		comp.rangeValues((*Value).standardize)
+		for v2 := range comp.allValues() {
+			v2.standardize()
+		}
 		if last := comp.lastValue(); last != nil && last.AfterExtra != nil {
 			*comp.afterExtra() = append(append(last.AfterExtra, ' '), *comp.afterExtra()...)
 			last.AfterExtra = nil
@@ -71,7 +76,6 @@ func (v *Value) standardize() bool {
 		comp.afterExtra().standardize()
 	}
 	v.AfterExtra.standardize()
-	return true
 }
 func (b *Extra) standardize() {
 	for i, c := range *b {

vendor/github.com/tailscale/hujson/types.go

@@ -12,8 +12,7 @@
 //
 // See https://nigeltao.github.io/blog/2021/json-with-commas-comments.html
 //
-//
-// Functionality
+// # Functionality
 //
 // The Parse function parses HuJSON input as a Value,
 // which is a syntax tree exactly representing the input.
@@ -32,8 +31,7 @@
 // but instead for the HuJSON and standard JSON format.
 // The Patch method applies a JSON Patch (RFC 6902) to the receiving value.
 //
-//
-// Grammar
+// # Grammar
 //
 // The changes to the JSON grammar are:
 //
@@ -72,8 +70,7 @@
 //	 	'000A' ws
 //	 	'000D' ws
 //
-//
-// Use with the Standard Library
+// # Use with the Standard Library
 //
 // This package operates with HuJSON as an AST. In order to parse HuJSON
 // into arbitrary Go types, use this package to parse HuJSON input as an AST,
@@ -95,6 +92,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"iter"
 	"math"
 	"strconv"
 	"unicode/utf8"
@@ -147,6 +145,45 @@ func (v Value) Clone() Value {
 	return v
 }
 
+// Range iterates through a Value in depth-first order and
+// calls f for each value (including the root value).
+// It stops iteration when f returns false.
+//
+// Deprecated: Use [All] instead.
+func (v *Value) Range(f func(v *Value) bool) bool {
+	for v2 := range v.All() {
+		if !f(v2) {
+			return false
+		}
+	}
+	return true
+}
+
+// All returns an iterator over all values in depth-first order,
+// starting with v itself.
+func (v *Value) All() iter.Seq[*Value] {
+	return func(yield func(*Value) bool) {
+		v.walkAll(yield)
+	}
+}
+
+// walkAll recursively yields v and its descendants in depth-first order.
+func (v *Value) walkAll(yield func(*Value) bool) bool {
+	if !yield(v) {
+		return false
+	}
+
+	if comp, ok := v.Value.(composite); ok {
+		for v2 := range comp.allValues() {
+			if !v2.walkAll(yield) {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
 // ValueTrimmed is a JSON value without surrounding whitespace or comments.
 // This is a sum type consisting of Literal, *Object, or *Array.
 type ValueTrimmed interface {
@@ -335,6 +372,7 @@ type Object struct {
 	// after the preceding open brace or comma and before the closing brace.
 	AfterExtra Extra
 }
+
 type ObjectMember struct {
 	Name, Value Value
 }
@@ -342,35 +380,43 @@ type ObjectMember struct {
 func (obj Object) length() int {
 	return len(obj.Members)
 }
+
 func (obj Object) firstValue() *Value {
 	if len(obj.Members) > 0 {
 		return &obj.Members[0].Name
 	}
 	return nil
 }
-func (obj Object) rangeValues(f func(*Value) bool) bool {
-	for i := range obj.Members {
-		if !f(&obj.Members[i].Name) {
-			return false
-		}
-		if !f(&obj.Members[i].Value) {
-			return false
+
+// allValues iterates all members of the object,
+// interleaved between the member name and the member value.
+func (obj Object) allValues() iter.Seq[*Value] {
+	return func(yield func(*Value) bool) {
+		for i := range obj.Members {
+			if !yield(&obj.Members[i].Name) {
+				return
+			}
+			if !yield(&obj.Members[i].Value) {
+				return
+			}
 		}
 	}
-	return true
 }
+
 func (obj Object) lastValue() *Value {
 	if len(obj.Members) > 0 {
 		return &obj.Members[len(obj.Members)-1].Value
 	}
 	return nil
 }
+
 func (obj *Object) beforeExtraAt(i int) *Extra {
 	if i < len(obj.Members) {
 		return &obj.Members[i].Name.BeforeExtra
 	}
 	return &obj.AfterExtra
 }
+
 func (obj *Object) afterExtra() *Extra {
 	return &obj.AfterExtra
 }
@@ -401,37 +447,45 @@ type Array struct {
 	// after the preceding open bracket or comma and before the closing bracket.
 	AfterExtra Extra
 }
+
 type ArrayElement = Value
 
 func (arr Array) length() int {
 	return len(arr.Elements)
 }
+
 func (arr Array) firstValue() *Value {
 	if len(arr.Elements) > 0 {
 		return &arr.Elements[0]
 	}
 	return nil
 }
-func (arr Array) rangeValues(f func(*Value) bool) bool {
-	for i := range arr.Elements {
-		if !f(&arr.Elements[i]) {
-			return false
+
+// allValues iterates all elements of the array.
+func (arr Array) allValues() iter.Seq[*Value] {
+	return func(yield func(*Value) bool) {
+		for i := range arr.Elements {
+			if !yield(&arr.Elements[i]) {
+				return
+			}
 		}
 	}
-	return true
 }
+
 func (arr Array) lastValue() *Value {
 	if len(arr.Elements) > 0 {
 		return &arr.Elements[len(arr.Elements)-1]
 	}
 	return nil
 }
+
 func (arr *Array) beforeExtraAt(i int) *Extra {
 	if i < len(arr.Elements) {
 		return &arr.Elements[i].BeforeExtra
 	}
 	return &arr.AfterExtra
 }
+
 func (arr *Array) afterExtra() *Extra {
 	return &arr.AfterExtra
 }
@@ -457,7 +511,7 @@ type composite interface {
 	length() int
 
 	firstValue() *Value
-	rangeValues(func(*Value) bool) bool
+	allValues() iter.Seq[*Value]
 	lastValue() *Value
 
 	getAt(int) ValueTrimmed

vendor/github.com/tailscale/web-client-prebuilt/build/index.html

@@ -6,8 +6,8 @@
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <link rel="shortcut icon" href="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAQAAADZc7J/AAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAAAmJLR0QA/4ePzL8AAAAHdElNRQflAx4QGA4EvmzDAAAA30lEQVRIx2NgGAWMCKa8JKM4A8Ovt88ekyLCDGOoyDBJMjExMbFy8zF8/EKsCAMDE8yAPyIwFps48SJIBpAL4AZwvoSx/r0lXgQpDN58EWL5x/7/H+vL20+JFxluQKVe5b3Ke5V+0kQQCamfoYKBg4GDwUKI8d0BYkWQkrLKewYBKPPDHUFiRaiZkBgmwhj/F5IgggyUJ6i8V3mv0kCayDAAeEsklXqGAgYGhgV3CnGrwVciYSYk0kokhgS44/JxqqFpiYSZbEgskd4dEBRk1GD4wdB5twKXmlHAwMDAAACdEZau06NQUwAAACV0RVh0ZGF0ZTpjcmVhdGUAMjAyMC0wNy0xNVQxNTo1Mzo0MCswMDowMCVXsDIAAAAldEVYdGRhdGU6bW9kaWZ5ADIwMjAtMDctMTVUMTU6NTM6NDArMDA6MDBUCgiOAAAAAElFTkSuQmCC" />
     <link rel="preload" as="font" href="./assets/Inter.var.latin-Dxq58mVK.woff2" type="font/woff2" crossorigin />
-    <script type="module" crossorigin src="./assets/index-BbZBz4S-.js"></script>
-    <link rel="stylesheet" crossorigin href="./assets/index-DVk8gqX9.css">
+    <script type="module" crossorigin src="./assets/index-BbgPCR4_.js"></script>
+    <link rel="stylesheet" crossorigin href="./assets/index-UKv3Effy.css">
   </head>
   <body class="px-2">
     <noscript>

vendor/github.com/tailscale/wireguard-go/conn/bind_std.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/netip"
 	"runtime"
@@ -202,9 +203,17 @@ again:
 	return fns, uint16(port), nil
 }
 
-func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) {
-	for i := range *msgs {
-		(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
+// putMessages resets each message in (*msgs)[:usedMsgs] for reuse, then returns
+// msgs to its [sync.Pool].
+func (s *StdNetBind) putMessages(msgs *[]ipv6.Message, usedMsgs int) {
+	for i := range (*msgs)[:usedMsgs] {
+		// Clear references to previously used packet buffers, otherwise they
+		// may never be GC'd.
+		clear((*msgs)[i].Buffers)
+		(*msgs)[i] = ipv6.Message{
+			Buffers: (*msgs)[i].Buffers[:1], // Non-coalesced write paths require a single element.
+			OOB:     (*msgs)[i].OOB,
+		}
 	}
 	s.msgsPool.Put(msgs)
 }
@@ -226,69 +235,62 @@ type batchWriter interface {
 	WriteBatch([]ipv6.Message, int) (int, error)
 }
 
+const maxDatagramSize = 1<<16 - 1
+
 func (s *StdNetBind) receiveIP(
 	br batchReader,
 	conn *net.UDPConn,
 	rxOffload bool,
-	bufs [][]byte,
-	sizes []int,
-	eps []Endpoint,
+	slab []byte,
+	packets []ReceivedPacket,
 ) (n int, err error) {
 	msgs := s.getMessages()
-	for i := range bufs {
-		(*msgs)[i].Buffers[0] = bufs[i]
-		(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
+	usedMsgs := 1
+	if runtime.GOOS == "linux" {
+		// set a floor of 1 in case len(slab) < maxDatagramSize
+		usedMsgs = max(1, len(slab)/maxDatagramSize)
+		// we can't read more datagrams than what we can describe in packets
+		usedMsgs = min(len(packets), usedMsgs)
 	}
-	defer s.putMessages(msgs)
+	defer s.putMessages(msgs, usedMsgs)
 	var numMsgs int
 	if runtime.GOOS == "linux" {
-		if rxOffload {
-			readAt := len(*msgs) - 2
-			numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0)
-			if err != nil {
-				return 0, err
-			}
-			numMsgs, err = splitCoalescedMessages(*msgs, readAt, getGSOSize)
-			if err != nil {
-				return 0, err
-			}
-		} else {
-			numMsgs, err = br.ReadBatch(*msgs, 0)
-			if err != nil {
-				return 0, err
+		rem := slab
+		for i := range usedMsgs {
+			end := maxDatagramSize
+			if len(rem) < maxDatagramSize {
+				end = len(rem)
 			}
+			(*msgs)[i].Buffers[0] = rem[:end]
+			(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
+			rem = rem[end:]
+		}
+		numMsgs, err = br.ReadBatch((*msgs)[:usedMsgs], 0)
+		if err != nil {
+			return 0, err
 		}
 	} else {
 		msg := &(*msgs)[0]
+		msg.Buffers[0] = slab
+		msg.OOB = msg.OOB[:cap(msg.OOB)]
 		msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
 		if err != nil {
 			return 0, err
 		}
 		numMsgs = 1
 	}
-	for i := 0; i < numMsgs; i++ {
-		msg := &(*msgs)[i]
-		sizes[i] = msg.N
-		if sizes[i] == 0 {
-			continue
-		}
-		addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
-		ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
-		getSrcFromControl(msg.OOB[:msg.NN], ep)
-		eps[i] = ep
-	}
-	return numMsgs, nil
+	return fillReceivedPackets((*msgs)[:numMsgs], maxDatagramSize, packets, rxOffload, getGSOSize)
 }
 
 func (s *StdNetBind) makeReceiveIPv4(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc {
-	return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) {
-		return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps)
+	return func(slab []byte, packets []ReceivedPacket) (n int, err error) {
+		return s.receiveIP(pc, conn, rxOffload, slab, packets)
 	}
 }
 
 func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc {
-	return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) {
-		return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps)
+	return func(slab []byte, packets []ReceivedPacket) (n int, err error) {
+		return s.receiveIP(pc, conn, rxOffload, slab, packets)
 	}
 }
 
@@ -365,7 +367,8 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
 	}
 
 	msgs := s.getMessages()
-	defer s.putMessages(msgs)
+	usedMsgs := len(bufs)
+	defer s.putMessages(msgs, usedMsgs)
 	ua := s.udpAddrPool.Get().(*net.UDPAddr)
 	defer s.udpAddrPool.Put(ua)
 	if is6 {
@@ -445,17 +448,34 @@ const (
 	maxIPv6PayloadLen = 1<<16 - 1 - 8
 
 	// This is a hard limit imposed by the kernel.
+	// As long as we use one fragment per datagram, this also serves as a
+	// limit for the number of fragments we can coalesce during scatter-gather writes.
+	//
+	// 64 is below the 1024 of IOV_MAX (Linux) or UIO_MAXIOV (BSD),
+	// and the 256 of WSABUF_MAX_COUNT (Windows).
+	//
+	// (2026-04) If we begin shipping datagrams in more than one fragment,
+	// an independent fragment count limit needs to be implemented.
 	udpSegmentMaxDatagrams = 64
 )
 
 type setGSOFunc func(control *[]byte, gsoSize uint16)
 
+// coalesceMessages iterates 'buffs', setting and coalescing them in 'msgs'
+// where possible while maintaining datagram order.
+//
+// It aggregates message components as a list of buffers without copying,
+// and expects to be used only on Linux with scatter-gather writes via sendmmsg(2).
+//
+// All msgs[i].Buffers len must be one. Will panic if there is not enough msgs
+// to coalesce all buffs.
 func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offset int, msgs []ipv6.Message, setGSO setGSOFunc) int {
 	var (
-		base     = -1 // index of msg we are currently coalescing into
-		gsoSize  int  // segmentation size of msgs[base]
-		dgramCnt int  // number of dgrams coalesced into msgs[base]
-		endBatch bool // tracking flag to start a new batch on next iteration of bufs
+		base         = -1 // index of msg we are currently coalescing into
+		gsoSize      int  // segmentation size of msgs[base]
+		dgramCnt     int  // number of dgrams coalesced into msgs[base]
+		endBatch     bool // tracking flag to start a new batch on next iteration of bufs
+		coalescedLen int  // bytes coalesced into msgs[base]
 	)
 	maxPayloadLen := maxIPv4PayloadLen
 	if ep.DstIP().Is6() {
@@ -465,18 +485,18 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
 		buf = buf[offset:]
 		if i > 0 {
 			msgLen := len(buf)
-			baseLenBefore := len(msgs[base].Buffers[0])
-			freeBaseCap := cap(msgs[base].Buffers[0]) - baseLenBefore
-			if msgLen+baseLenBefore <= maxPayloadLen &&
+			if msgLen+coalescedLen <= maxPayloadLen &&
 				msgLen <= gsoSize &&
-				msgLen <= freeBaseCap &&
 				dgramCnt < udpSegmentMaxDatagrams &&
 				!endBatch {
-				msgs[base].Buffers[0] = append(msgs[base].Buffers[0], buf...)
+				// msgs[base].Buffers[0] is set to buf[i] when a new base is set.
+				// This appends a struct iovec element in the underlying struct msghdr (scatter-gather).
+				msgs[base].Buffers = append(msgs[base].Buffers, buf)
 				if i == len(bufs)-1 {
 					setGSO(&msgs[base].OOB, uint16(gsoSize))
 				}
 				dgramCnt++
+				coalescedLen += msgLen
 				if msgLen < gsoSize {
 					// A smaller than gsoSize packet on the tail is legal, but
 					// it must end the batch.
@@ -497,51 +517,48 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
 		msgs[base].Buffers[0] = buf
 		msgs[base].Addr = addr
 		dgramCnt = 1
+		coalescedLen = len(buf)
 	}
 	return base + 1
 }
 
 type getGSOFunc func(control []byte) (int, error)
 
-func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFunc) (n int, err error) {
-	for i := firstMsgAt; i < len(msgs); i++ {
-		msg := &msgs[i]
-		if msg.N == 0 {
-			return n, err
-		}
+func fillReceivedPackets(msgs []ipv6.Message, slabOffset int, packets []ReceivedPacket, rxOffload bool, getGSO getGSOFunc) (n int, err error) {
+	for i, msg := range msgs {
 		var (
 			gsoSize    int
 			start      int
-			end        = msg.N
 			numToSplit = 1
 		)
-		gsoSize, err = getGSO(msg.OOB[:msg.NN])
-		if err != nil {
-			return n, err
-		}
-		if gsoSize > 0 {
-			numToSplit = (msg.N + gsoSize - 1) / gsoSize
-			end = gsoSize
-		}
-		for j := 0; j < numToSplit; j++ {
-			if n > i {
-				return n, errors.New("splitting coalesced packet resulted in overflow")
+		if rxOffload {
+			gsoSize, err = getGSO(msg.OOB[:msg.NN])
+			if err != nil {
+				return n, err
 			}
-			copied := copy(msgs[n].Buffers[0], msg.Buffers[0][start:end])
-			msgs[n].N = copied
-			msgs[n].Addr = msg.Addr
-			start = end
-			end += gsoSize
-			if end > msg.N {
-				end = msg.N
+			if gsoSize > 0 {
+				numToSplit = (msg.N + gsoSize - 1) / gsoSize
+			}
+		}
+		addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
+		ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
+		getSrcFromControl(msg.OOB[:msg.NN], ep)
+		regionOffset := i * slabOffset // region may contain multiple coalesced packets
+		for j := 0; j < numToSplit; j++ {
+			if n >= len(packets) {
+				return n, fmt.Errorf("%w: filling received packet metadata resulted in overflow", io.ErrShortBuffer)
+			}
+			end := msg.N
+			if gsoSize > 0 && start+gsoSize < end {
+				end = start + gsoSize
+			}
+			packets[n] = ReceivedPacket{
+				Offset:   regionOffset + start,
+				Size:     end - start,
+				Endpoint: ep,
 			}
 			n++
-		}
-		if i != n-1 {
-			// It is legal for bytes to move within msg.Buffers[0] as a result
-			// of splitting, so we only zero the source msg len when it is not
-			// the destination of the last split operation above.
-			msg.N = 0
+			start = end
 		}
 	}
 	return n, nil

vendor/github.com/tailscale/wireguard-go/conn/bind_windows.go

@@ -416,21 +416,25 @@ retry:
 	return n, &ep, nil
 }
 
-func (bind *WinRingBind) receiveIPv4(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
+func (bind *WinRingBind) receiveIPv4(slab []byte, packets []ReceivedPacket) (int, error) {
 	bind.mu.RLock()
 	defer bind.mu.RUnlock()
-	n, ep, err := bind.v4.Receive(bufs[0], &bind.isOpen)
-	sizes[0] = n
-	eps[0] = ep
+	n, ep, err := bind.v4.Receive(slab, &bind.isOpen)
+	packets[0] = ReceivedPacket{
+		Size:     n,
+		Endpoint: ep,
+	}
 	return 1, err
 }
 
-func (bind *WinRingBind) receiveIPv6(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
+func (bind *WinRingBind) receiveIPv6(slab []byte, packets []ReceivedPacket) (int, error) {
 	bind.mu.RLock()
 	defer bind.mu.RUnlock()
-	n, ep, err := bind.v6.Receive(bufs[0], &bind.isOpen)
-	sizes[0] = n
-	eps[0] = ep
+	n, ep, err := bind.v6.Receive(slab, &bind.isOpen)
+	packets[0] = ReceivedPacket{
+		Size:     n,
+		Endpoint: ep,
+	}
 	return 1, err
 }
 

vendor/github.com/tailscale/wireguard-go/conn/conn.go

@@ -19,13 +19,25 @@ const (
 	IdealBatchSize = 128 // maximum number of packets handled per read and write
 )
 
+// ReceivedPacket describes a packet read by a [ReceiveFunc].
+type ReceivedPacket struct {
+	// Offset is the starting byte offset.
+	Offset int
+	// Size is the size of the packet.
+	Size int
+	// Endpoint is the associated [Endpoint].
+	Endpoint Endpoint
+}
+
+func (r *ReceivedPacket) Bytes(slab []byte) []byte {
+	return slab[r.Offset : r.Offset+r.Size : r.Offset+r.Size]
+}
+
 // A ReceiveFunc receives at least one packet from the network and writes them
-// into packets. On a successful read it returns the number of elements of
-// sizes, packets, and endpoints that should be evaluated. Some elements of
-// sizes may be zero, and callers should ignore them. Callers must pass a sizes
-// and eps slice with a length greater than or equal to the length of packets.
-// These lengths must not exceed the length of the associated Bind.BatchSize().
-type ReceiveFunc func(packets [][]byte, sizes []int, eps []Endpoint) (n int, err error)
+// into slab. On a successful read it returns the number of elements of
+// packets that should be evaluated. Callers must pass a length of packets equal
+// to the associated [Bind.BatchSize].
+type ReceiveFunc func(slab []byte, packets []ReceivedPacket) (n int, err error)
 
 // A Bind listens on a port for both IPv6 and IPv4 UDP traffic.
 //

vendor/github.com/tailscale/wireguard-go/device/aead_amd64.go

@@ -0,0 +1,41 @@
+//go:build amd64 && gc && !purego
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"crypto/cipher"
+	"os"
+
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/sys/cpu"
+
+	asmAEAD "github.com/tailscale/wireguard-go/tsasm/amd64/chacha20poly1305"
+)
+
+/*
+shouldUseTSAsm reports whether the data-path AEAD should come from tsasm/amd64. x/crypto gates its remaining amd64 kernel on HasSSSE3 && HasAVX2 && HasBMI2 and otherwise runs generic Go, so only CPUs with SSSE3 and no AVX2 are routed here. Pure so a test can cover every combination; the CPU variables cannot be written.
+*/
+func shouldUseTSAsm(hasSSSE3, hasAVX2, hasBMI2 bool) bool {
+	return hasSSSE3 && !(hasAVX2 && hasBMI2)
+}
+
+var useTSAsm = shouldUseTSAsm(asmAEAD.Available(), cpu.X86.HasAVX2, cpu.X86.HasBMI2)
+
+/*
+chacha20poly1305New returns a ChaCha20-Poly1305 AEAD. On amd64 CPUs with SSSE3 but no AVX2 it uses the assembly kernel from tsasm/amd64/chacha20poly1305.
+
+The cookie path (which uses the extended-nonce variant via chacha20poly1305.NewX) is left on the x/crypto path because it is not on the per-packet hot path.
+
+As an escape hatch for hardware regressions or asm bugs, setting the environment variable TS_WG_ASM=0 forces the x/crypto implementation instead.
+*/
+func chacha20poly1305New(key []byte) (cipher.AEAD, error) {
+	if !useTSAsm || os.Getenv("TS_WG_ASM") == "0" {
+		return chacha20poly1305.New(key)
+	}
+	return asmAEAD.New(key)
+}

vendor/github.com/tailscale/wireguard-go/device/aead_arm.go

@@ -0,0 +1,33 @@
+//go:build arm
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"crypto/cipher"
+	"os"
+
+	"golang.org/x/crypto/chacha20poly1305"
+
+	asmAEAD "github.com/tailscale/wireguard-go/tsasm/arm/chacha20poly1305"
+)
+
+// chacha20poly1305New returns a ChaCha20-Poly1305 AEAD. On GOARCH=arm
+// (32-bit, any GOOS) it uses the assembly kernels from tsasm/arm/.
+// The cookie path (which uses the extended-nonce variant via
+// chacha20poly1305.NewX) is left on the x/crypto path because it is
+// not on the per-packet hot path.
+//
+// As an escape hatch for hardware regressions or asm bugs, setting
+// the environment variable TS_WG_ASM=0 forces the pure-Go x/crypto
+// implementation instead.
+func chacha20poly1305New(key []byte) (cipher.AEAD, error) {
+	if os.Getenv("TS_WG_ASM") == "0" {
+		return chacha20poly1305.New(key)
+	}
+	return asmAEAD.New(key)
+}

vendor/github.com/tailscale/wireguard-go/device/aead_other.go

@@ -0,0 +1,18 @@
+//go:build !arm && !(amd64 && gc && !purego)
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"crypto/cipher"
+
+	"golang.org/x/crypto/chacha20poly1305"
+)
+
+func chacha20poly1305New(key []byte) (cipher.AEAD, error) {
+	return chacha20poly1305.New(key)
+}

vendor/github.com/tailscale/wireguard-go/device/allowedips.go

@@ -55,6 +55,25 @@ func commonBits(ip1, ip2 []byte) uint8 {
 	}
 }
 
+func commonBits4(ip1 []byte, ip2 [4]byte) uint8 {
+	a := binary.BigEndian.Uint32(ip1)
+	b := binary.BigEndian.Uint32(ip2[:])
+	return uint8(bits.LeadingZeros32(a ^ b))
+}
+
+func commonBits6(ip1 []byte, ip2 [16]byte) uint8 {
+	a := binary.BigEndian.Uint64(ip1)
+	b := binary.BigEndian.Uint64(ip2[:])
+	x := a ^ b
+	if x != 0 {
+		return uint8(bits.LeadingZeros64(x))
+	}
+	a = binary.BigEndian.Uint64(ip1[8:])
+	b = binary.BigEndian.Uint64(ip2[8:])
+	x = a ^ b
+	return 64 + uint8(bits.LeadingZeros64(x))
+}
+
 func (node *trieEntry) addToPeerEntries() {
 	node.perPeerElem = node.peer.trieEntries.PushBack(node)
 }
@@ -188,7 +207,37 @@ func (trie parentIndirection) insert(ip []byte, cidr uint8, peer *Peer) {
 	}
 }
 
-func (node *trieEntry) lookup(ip []byte) *Peer {
+func (node *trieEntry) lookup4(ip [4]byte) *Peer {
+	var found *Peer
+	for node != nil && commonBits4(node.bits, ip) >= node.cidr {
+		if node.peer != nil {
+			found = node.peer
+		}
+		if node.bitAtByte == 4 {
+			break
+		}
+		bit := (ip[node.bitAtByte] >> node.bitAtShift) & 1
+		node = node.child[bit]
+	}
+	return found
+}
+
+func (node *trieEntry) lookup6(ip [16]byte) *Peer {
+	var found *Peer
+	for node != nil && commonBits6(node.bits, ip) >= node.cidr {
+		if node.peer != nil {
+			found = node.peer
+		}
+		if node.bitAtByte == 16 {
+			break
+		}
+		bit := (ip[node.bitAtByte] >> node.bitAtShift) & 1
+		node = node.child[bit]
+	}
+	return found
+}
+
+func (node *trieEntry) lookup(ip net.IP) *Peer {
 	var found *Peer
 	size := uint8(len(ip))
 	for node != nil && commonBits(node.bits, ip) >= node.cidr {
@@ -205,14 +254,17 @@ func (node *trieEntry) lookup(ip []byte) *Peer {
 }
 
 type AllowedIPs struct {
-	IPv4  *trieEntry
-	IPv6  *trieEntry
-	mutex sync.RWMutex
+	mu   sync.RWMutex
+	ipv4 *trieEntry
+	ipv6 *trieEntry
+
+	peerByIPPacketFunc PeerByIPPacketFunc // if non-nil, called to look up peers by IP
+	device             *Device            // back-reference to parent device; non-nil only if peerByIPPacketFunc is set
 }
 
 func (table *AllowedIPs) EntriesForPeer(peer *Peer, cb func(prefix netip.Prefix) bool) {
-	table.mutex.RLock()
-	defer table.mutex.RUnlock()
+	table.mu.RLock()
+	defer table.mu.RUnlock()
 
 	for elem := peer.trieEntries.Front(); elem != nil; elem = elem.Next() {
 		node := elem.Value.(*trieEntry)
@@ -223,10 +275,25 @@ func (table *AllowedIPs) EntriesForPeer(peer *Peer, cb func(prefix netip.Prefix)
 	}
 }
 
-func (table *AllowedIPs) RemoveByPeer(peer *Peer) {
-	table.mutex.Lock()
-	defer table.mutex.Unlock()
+// setPeerPrefixes atomically removes all of peer's existing prefixes and adds
+// the provided ones.
+func (table *AllowedIPs) setPeerPrefixes(peer *Peer, prefixes []netip.Prefix) {
+	table.mu.Lock()
+	defer table.mu.Unlock()
 
+	table.removeByPeerLocked(peer)
+	for _, prefix := range prefixes {
+		table.insertLocked(prefix, peer)
+	}
+}
+
+func (table *AllowedIPs) RemoveByPeer(peer *Peer) {
+	table.mu.Lock()
+	defer table.mu.Unlock()
+	table.removeByPeerLocked(peer)
+}
+
+func (table *AllowedIPs) removeByPeerLocked(peer *Peer) {
 	var next *list.Element
 	for elem := peer.trieEntries.Front(); elem != nil; elem = next {
 		next = elem.Next()
@@ -266,29 +333,139 @@ func (table *AllowedIPs) RemoveByPeer(peer *Peer) {
 }
 
 func (table *AllowedIPs) Insert(prefix netip.Prefix, peer *Peer) {
-	table.mutex.Lock()
-	defer table.mutex.Unlock()
+	table.mu.Lock()
+	defer table.mu.Unlock()
+	table.insertLocked(prefix, peer)
+}
 
+func (table *AllowedIPs) insertLocked(prefix netip.Prefix, peer *Peer) {
 	if prefix.Addr().Is6() {
 		ip := prefix.Addr().As16()
-		parentIndirection{&table.IPv6, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
+		parentIndirection{&table.ipv6, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
 	} else if prefix.Addr().Is4() {
 		ip := prefix.Addr().As4()
-		parentIndirection{&table.IPv4, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
+		parentIndirection{&table.ipv4, 2}.insert(ip[:], uint8(prefix.Bits()), peer)
 	} else {
 		panic(errors.New("inserting unknown address type"))
 	}
 }
 
-func (table *AllowedIPs) Lookup(ip []byte) *Peer {
-	table.mutex.RLock()
-	defer table.mutex.RUnlock()
-	switch len(ip) {
-	case net.IPv6len:
-		return table.IPv6.lookup(ip)
-	case net.IPv4len:
-		return table.IPv4.lookup(ip)
+// LookupFromPacket looks up the peer to which an outbound IP packet should be
+// sent. It lives on [AllowedIPs] for legacy/structural reasons: historically
+// WireGuard's only peer-selection mechanism was the AllowedIPs trie, and the
+// send path already had a reference to the table. When a [PeerByIPPacketFunc]
+// has been registered via [Device.SetPeerByIPPacketFunc], that callback is used
+// instead of the trie and the AllowedIPs table is not consulted at all.
+//
+// When no callback is registered, only dst is used (standard WireGuard
+// AllowedIPs trie lookup). When a callback is registered, all three
+// parameters are forwarded to it; see [PeerByIPPacketFunc] for details.
+func (table *AllowedIPs) LookupFromPacket(src, dst netip.Addr, ipPkt []byte) *Peer {
+	table.mu.RLock()
+	if f := table.peerByIPPacketFunc; f != nil {
+		device := table.device
+		table.mu.RUnlock()
+
+		if pubk, ok := f(src, dst, ipPkt); ok {
+			return device.LookupPeer(pubk)
+		}
+		return nil
+	}
+	defer table.mu.RUnlock()
+
+	switch {
+	case dst.Is6():
+		return table.ipv6.lookup6(dst.As16())
+	case dst.Is4():
+		return table.ipv4.lookup4(dst.As4())
 	default:
 		panic(errors.New("looking up unknown address type"))
 	}
 }
+
+// Deprecated: Lookup is only used by legacy tests. It does not call
+// [PeerByIPPacketFunc]; use [AllowedIPs.LookupFromPacket] for production lookups.
+func (table *AllowedIPs) Lookup(ip []byte) *Peer {
+	table.mu.RLock()
+	defer table.mu.RUnlock()
+	return table.lookupLocked(ip)
+}
+
+// lookupLocked looks up the peer associated with the given IP address.
+// It assumes the caller holds the read lock (or doesn't hold it, but also
+// doesn't concurrently mutate AllowedIP).
+//
+// It returns nil if no peer is associated with the given IP address.
+func (table *AllowedIPs) lookupLocked(ip []byte) *Peer {
+	switch len(ip) {
+	case net.IPv6len:
+		return table.ipv6.lookup(ip)
+	case net.IPv4len:
+		return table.ipv4.lookup(ip)
+	default:
+		panic(errors.New("looking up unknown address type"))
+	}
+}
+
+// AllowedPeerSourceIP reports whether the given source IP address is allowed
+// for the given peer.
+func (peer *Peer) AllowedPeerSourceIP(src netip.Addr) bool {
+	if f := peer.state.testAllowedIP.Load(); f != nil {
+		return (*f)(src)
+	}
+
+	table := &peer.device.allowedips
+	table.mu.RLock()
+	defer table.mu.RUnlock()
+	switch {
+	case src.Is6():
+		return table.ipv6.lookup6(src.As16()) == peer
+	case src.Is4():
+		return table.ipv4.lookup4(src.As4()) == peer
+	}
+	return false
+}
+
+// cidrLinearSearchMax is the largest number of CIDRs for which
+// mkIPInCIDRsTestFunc does a linear search instead of building a trie.
+const cidrLinearSearchMax = 4
+
+// mkIPInCIDRsTestFunc returns a function that tests whether an IP address is
+// contained in any of the given CIDRs.
+func mkIPInCIDRsTestFunc(cidrs []netip.Prefix) func(netip.Addr) bool {
+	if len(cidrs) == 0 {
+		return func(netip.Addr) bool { return false }
+	}
+	if len(cidrs) == 1 {
+		return func(addr netip.Addr) bool { return cidrs[0].Contains(addr) }
+	}
+	if len(cidrs) <= cidrLinearSearchMax {
+		// For small numbers of CIDRs, just do a linear search. The trie construction
+		// is more expensive than the linear search, and the test function is faster
+		// than the trie lookup, so this is a net win.
+		return func(addr netip.Addr) bool {
+			for _, c := range cidrs {
+				if c.Contains(addr) {
+					return true
+				}
+			}
+			return false
+		}
+	}
+	// Make a trie for faster lookups. Use a local dummy Peer so its
+	// trieEntries and the whole trie can be collected once the returned
+	// closure is gone.
+	fakePeer := new(Peer)
+	var a AllowedIPs
+	for _, c := range cidrs {
+		a.Insert(c, fakePeer)
+	}
+	return func(addr netip.Addr) bool {
+		switch {
+		case addr.Is4():
+			return a.ipv4.lookup4(addr.As4()) == fakePeer
+		default:
+			return a.ipv6.lookup6(addr.As16()) == fakePeer
+		}
+	}
+}

vendor/github.com/tailscale/wireguard-go/device/channels.go

@@ -6,7 +6,6 @@
 package device
 
 import (
-	"runtime"
 	"sync"
 )
 
@@ -23,9 +22,9 @@ type outboundQueue struct {
 	wg sync.WaitGroup
 }
 
-func newOutboundQueue() *outboundQueue {
+func newOutboundQueue(capacity int) *outboundQueue {
 	q := &outboundQueue{
-		c: make(chan *QueueOutboundElementsContainer, QueueOutboundSize),
+		c: make(chan *QueueOutboundElementsContainer, capacity),
 	}
 	q.wg.Add(1)
 	go func() {
@@ -41,9 +40,9 @@ type inboundQueue struct {
 	wg sync.WaitGroup
 }
 
-func newInboundQueue() *inboundQueue {
+func newInboundQueue(capacity int) *inboundQueue {
 	q := &inboundQueue{
-		c: make(chan *QueueInboundElementsContainer, QueueInboundSize),
+		c: make(chan *QueueInboundElementsContainer, capacity),
 	}
 	q.wg.Add(1)
 	go func() {
@@ -59,9 +58,9 @@ type handshakeQueue struct {
 	wg sync.WaitGroup
 }
 
-func newHandshakeQueue() *handshakeQueue {
+func newHandshakeQueue(capacity int) *handshakeQueue {
 	q := &handshakeQueue{
-		c: make(chan QueueHandshakeElement, QueueHandshakeSize),
+		c: make(chan QueueHandshakeElement, capacity),
 	}
 	q.wg.Add(1)
 	go func() {
@@ -70,68 +69,3 @@ func newHandshakeQueue() *handshakeQueue {
 	}()
 	return q
 }
-
-type autodrainingInboundQueue struct {
-	c chan *QueueInboundElementsContainer
-}
-
-// newAutodrainingInboundQueue returns a channel that will be drained when it gets GC'd.
-// It is useful in cases in which is it hard to manage the lifetime of the channel.
-// The returned channel must not be closed. Senders should signal shutdown using
-// some other means, such as sending a sentinel nil values.
-func newAutodrainingInboundQueue(device *Device) *autodrainingInboundQueue {
-	q := &autodrainingInboundQueue{
-		c: make(chan *QueueInboundElementsContainer, QueueInboundSize),
-	}
-	runtime.SetFinalizer(q, device.flushInboundQueue)
-	return q
-}
-
-func (device *Device) flushInboundQueue(q *autodrainingInboundQueue) {
-	for {
-		select {
-		case elemsContainer := <-q.c:
-			elemsContainer.Lock()
-			for _, elem := range elemsContainer.elems {
-				device.PutMessageBuffer(elem.buffer)
-				device.PutInboundElement(elem)
-			}
-			device.PutInboundElementsContainer(elemsContainer)
-		default:
-			return
-		}
-	}
-}
-
-type autodrainingOutboundQueue struct {
-	c chan *QueueOutboundElementsContainer
-}
-
-// newAutodrainingOutboundQueue returns a channel that will be drained when it gets GC'd.
-// It is useful in cases in which is it hard to manage the lifetime of the channel.
-// The returned channel must not be closed. Senders should signal shutdown using
-// some other means, such as sending a sentinel nil values.
-// All sends to the channel must be best-effort, because there may be no receivers.
-func newAutodrainingOutboundQueue(device *Device) *autodrainingOutboundQueue {
-	q := &autodrainingOutboundQueue{
-		c: make(chan *QueueOutboundElementsContainer, QueueOutboundSize),
-	}
-	runtime.SetFinalizer(q, device.flushOutboundQueue)
-	return q
-}
-
-func (device *Device) flushOutboundQueue(q *autodrainingOutboundQueue) {
-	for {
-		select {
-		case elemsContainer := <-q.c:
-			elemsContainer.Lock()
-			for _, elem := range elemsContainer.elems {
-				device.PutMessageBuffer(elem.buffer)
-				device.PutOutboundElement(elem)
-			}
-			device.PutOutboundElementsContainer(elemsContainer)
-		default:
-			return
-		}
-	}
-}

vendor/github.com/tailscale/wireguard-go/device/device.go

@@ -6,6 +6,8 @@
 package device
 
 import (
+	"errors"
+	"net/netip"
 	"runtime"
 	"sync"
 	"sync/atomic"
@@ -18,6 +20,8 @@ import (
 )
 
 type Device struct {
+	config config
+
 	state struct {
 		// state holds the device's state. It is accessed atomically.
 		// Use the device.deviceState method to read it.
@@ -44,7 +48,6 @@ type Device struct {
 		netlinkCancel *rwcancel.RWCancel
 		port          uint16 // listening port
 		fwmark        uint32 // mark value (0 = disabled)
-		brokenRoaming bool
 	}
 
 	staticIdentity struct {
@@ -56,8 +59,12 @@ type Device struct {
 	peers struct {
 		sync.RWMutex // protects keyMap
 		keyMap       map[NoisePublicKey]*Peer
+		lookupFunc   PeerLookupFunc // or nil if unused
 	}
 
+	peerStateFn   atomic.Pointer[PeerSessionStateFunc]    // observes peer session state changes, nil if unset
+	priorityMsgFn atomic.Pointer[PeerPriorityMessageFunc] // returns a priority message to be sent around session establishment, nil if unset
+
 	rate struct {
 		underLoadUntil atomic.Int64
 		limiter        ratelimiter.Ratelimiter
@@ -70,9 +77,12 @@ type Device struct {
 	pool struct {
 		inboundElementsContainer  *WaitPool
 		outboundElementsContainer *WaitPool
-		messageBuffers            *WaitPool
 		inboundElements           *WaitPool
 		outboundElements          *WaitPool
+		packetBufs                *WaitPool
+		// smallPacketBufs aliases packetBufs when no distinct small packet pool
+		// is used. See [Device.PopulatePools].
+		smallPacketBufs *WaitPool
 	}
 
 	queue struct {
@@ -82,8 +92,12 @@ type Device struct {
 	}
 
 	tun struct {
-		device tun.Device
-		mtu    atomic.Int32
+		// Fields in this block are read-only after instantiation.
+		device  tun.Device
+		queues  []tun.Queue // device's read queues, at least one, see [tun.QueuesOf]
+		writeTo func(flow int, bufs [][]byte, offset int) (int, error)
+
+		mtu atomic.Int32
 	}
 
 	ipcMutex sync.RWMutex
@@ -91,6 +105,152 @@ type Device struct {
 	log      *Logger
 }
 
+type config struct {
+	queueStagedSize            int
+	queueOutboundSize          int
+	queueInboundSize           int
+	queueHandshakeSize         int
+	preallocatedBuffersPerPool uint32
+	metrics                    Metrics
+}
+
+func defaultConfig() config {
+	return config{
+		queueStagedSize:            DefaultQueueStagedSize,
+		queueOutboundSize:          DefaultQueueOutboundSize,
+		queueInboundSize:           DefaultQueueInboundSize,
+		queueHandshakeSize:         DefaultQueueHandshakeSize,
+		preallocatedBuffersPerPool: DefaultPreallocatedBuffersPerPool,
+	}
+}
+
+// An Option configures a [Device].
+type Option interface {
+	apply(*config)
+}
+
+type optionFunc func(*config)
+
+func (f optionFunc) apply(config *config) {
+	f(config)
+}
+
+// WithQueueStagedSize sets the capacity of each peer's staged packet queue.
+// Staged packet queues must be buffered, so max(tun readers, size) is applied
+// to the user-supplied value. [DefaultQueueStagedSize] is the default.
+func WithQueueStagedSize(size int) Option {
+	return optionFunc(func(config *config) {
+		config.queueStagedSize = max(1, size)
+	})
+}
+
+// WithQueueOutboundSize sets the capacity of each peer's outbound packet queue.
+// It also sets the capacity of the device-wide outbound queue, whose total
+// capacity is size multiplied by the number of TUN queues.
+// [DefaultQueueOutboundSize] is the default.
+func WithQueueOutboundSize(size int) Option {
+	return optionFunc(func(config *config) {
+		config.queueOutboundSize = size
+	})
+}
+
+// WithQueueInboundSize sets the capacity of the device and per-peer inbound
+// packet queues. [DefaultQueueInboundSize] is the default.
+func WithQueueInboundSize(size int) Option {
+	return optionFunc(func(config *config) {
+		config.queueInboundSize = size
+	})
+}
+
+// WithQueueHandshakeSize sets the capacity of the device's handshake queue.
+// [DefaultQueueHandshakeSize] is the default.
+func WithQueueHandshakeSize(size int) Option {
+	return optionFunc(func(config *config) {
+		config.queueHandshakeSize = size
+	})
+}
+
+// WithPreallocatedBuffersPerPool sets the maximum number of packet memory
+// pool outstanding items. A value of zero is unlimited. Care must be taken to
+// not set a size that is too small, which can lead to immediate deadlock, or
+// increase the probability of deadlock once packets start flowing.
+// See tailscale/corp#46396. [DefaultPreallocatedBuffersPerPool] is the default.
+func WithPreallocatedBuffersPerPool(size uint32) Option {
+	return optionFunc(func(config *config) {
+		config.preallocatedBuffersPerPool = size
+	})
+}
+
+// Counter is a monotonically increasing counter used in [Metrics].
+type Counter interface {
+	// Add increments the Counter's value by n. n must not be negative. Add is
+	// called in performance-sensitive contexts, therefore it must be cheap. Add
+	// may be called concurrently.
+	Add(n int64)
+}
+
+// Metrics contains metrics that can be exported via [WithMetrics].
+type Metrics struct {
+	// MessageInitiationTXAttemptInitial counts non-retry handshake
+	// initiation messages passed to the network send path.
+	MessageInitiationTXAttemptInitial Counter
+	// MessageInitiationTXAttemptRetry counts retry handshake initiation
+	// messages passed to the network send path.
+	MessageInitiationTXAttemptRetry Counter
+	// MessageResponseTXAttempt counts handshake response messages passed to the
+	// network send path.
+	MessageResponseTXAttempt Counter
+	// MessageCookieReplyTXAttempt counts cookie reply messages passed to the
+	// network send path.
+	MessageCookieReplyTXAttempt Counter
+	// HandshakeInitiatorCompleted counts symmetric session establishment on the
+	// initiator side.
+	HandshakeInitiatorCompleted Counter
+	// HandshakeResponderCompleted counts responder-side handshake completions,
+	// observed when an authenticated transport packet confirms the new keypair.
+	HandshakeResponderCompleted Counter
+	// MessageTransportRXDroppedReplay counts inbound transport packets rejected by the
+	// replay filter.
+	MessageTransportRXDroppedReplay Counter
+}
+
+// noopCounter is a noop implementation of [Counter].
+type noopCounter struct {
+}
+
+func (noopCounter) Add(int64) {}
+
+func (m *Metrics) fillNils() {
+	if m.MessageInitiationTXAttemptInitial == nil {
+		m.MessageInitiationTXAttemptInitial = noopCounter{}
+	}
+	if m.MessageInitiationTXAttemptRetry == nil {
+		m.MessageInitiationTXAttemptRetry = noopCounter{}
+	}
+	if m.MessageResponseTXAttempt == nil {
+		m.MessageResponseTXAttempt = noopCounter{}
+	}
+	if m.MessageCookieReplyTXAttempt == nil {
+		m.MessageCookieReplyTXAttempt = noopCounter{}
+	}
+	if m.HandshakeInitiatorCompleted == nil {
+		m.HandshakeInitiatorCompleted = noopCounter{}
+	}
+	if m.HandshakeResponderCompleted == nil {
+		m.HandshakeResponderCompleted = noopCounter{}
+	}
+	if m.MessageTransportRXDroppedReplay == nil {
+		m.MessageTransportRXDroppedReplay = noopCounter{}
+	}
+}
+
+// WithMetrics sets the provided [Metrics] to be used at instrumentation points.
+func WithMetrics(metrics Metrics) Option {
+	return optionFunc(func(config *config) {
+		config.metrics = metrics
+	})
+}
+
 // deviceState represents the state of a Device.
 // There are three states: down, up, closed.
 // Transitions:
@@ -179,14 +339,22 @@ func (device *Device) upLocked() error {
 	device.ipcMutex.Lock()
 	defer device.ipcMutex.Unlock()
 
+	// Collect peers under RLock and then release before calling into them,
+	// because SendKeepalive can reach CreateMessageInitiation which acquires
+	// staticIdentity.RLock; holding peers.RLock across that path would
+	// invert the staticIdentity < peers hierarchy (see lock-ordering.md).
 	device.peers.RLock()
+	peers := make([]*Peer, 0, len(device.peers.keyMap))
 	for _, peer := range device.peers.keyMap {
+		peers = append(peers, peer)
+	}
+	device.peers.RUnlock()
+	for _, peer := range peers {
 		peer.Start()
 		if peer.persistentKeepaliveInterval.Load() > 0 {
 			peer.SendKeepalive()
 		}
 	}
-	device.peers.RUnlock()
 	return nil
 }
 
@@ -217,7 +385,9 @@ func (device *Device) Down() error {
 func (device *Device) IsUnderLoad() bool {
 	// check if currently under load
 	now := time.Now()
-	underLoad := len(device.queue.handshake.c) >= QueueHandshakeSize/8
+	// max(1, ...) is required on the right hand side, otherwise underLoad would
+	// always be true when queueHandshakeSize < 8.
+	underLoad := len(device.queue.handshake.c) >= max(1, device.config.queueHandshakeSize/8)
 	if underLoad {
 		device.rate.underLoadUntil.Store(now.Add(UnderLoadAfterTime).UnixNano())
 		return true
@@ -281,8 +451,12 @@ func (device *Device) SetPrivateKey(sk NoisePrivateKey) error {
 	return nil
 }
 
-func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device {
-	device := new(Device)
+func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger, opts ...Option) *Device {
+	device := &Device{config: defaultConfig()}
+	for _, opt := range opts {
+		opt.apply(&device.config)
+	}
+	device.config.metrics.fillNils()
 	device.state.state.Store(uint32(deviceStateDown))
 	device.closed = make(chan struct{})
 	device.log = logger
@@ -294,17 +468,31 @@ func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device {
 		mtu = DefaultMTU
 	}
 	device.tun.mtu.Store(int32(mtu))
+	device.tun.queues = tun.QueuesOf(tunDevice)
+	device.tun.writeTo = tun.WriteToOf(tunDevice)
 	device.peers.keyMap = make(map[NoisePublicKey]*Peer)
 	device.rate.limiter.Init()
 	device.indexTable.Init()
 
+	if want := len(device.tun.queues); want > device.config.queueStagedSize {
+		device.log.Errorf(
+			"Raising staged queue size to fit concurrent tun readers: %d -> %d",
+			device.config.queueStagedSize,
+			want,
+		)
+		device.config.queueStagedSize = want
+	}
+
 	device.PopulatePools()
 
 	// create queues
 
-	device.queue.handshake = newHandshakeQueue()
-	device.queue.encryption = newOutboundQueue()
-	device.queue.decryption = newInboundQueue()
+	device.queue.handshake = newHandshakeQueue(device.config.queueHandshakeSize)
+	// Scale to the number of producers
+	device.queue.encryption = newOutboundQueue(
+		device.config.queueOutboundSize * len(device.tun.queues),
+	)
+	device.queue.decryption = newInboundQueue(device.config.queueInboundSize)
 
 	// start workers
 
@@ -317,9 +505,11 @@ func NewDevice(tunDevice tun.Device, bind conn.Bind, logger *Logger) *Device {
 		go device.RoutineHandshake(i + 1)
 	}
 
-	device.state.stopping.Add(1)      // RoutineReadFromTUN
-	device.queue.encryption.wg.Add(1) // RoutineReadFromTUN
-	go device.RoutineReadFromTUN()
+	device.state.stopping.Add(len(device.tun.queues))      // RoutineReadFromTUN
+	device.queue.encryption.wg.Add(len(device.tun.queues)) // RoutineReadFromTUN
+	for i, q := range device.tun.queues {
+		go device.RoutineReadFromTUN(i, q)
+	}
 	go device.RoutineTUNEventReader()
 
 	return device
@@ -338,13 +528,69 @@ func (device *Device) BatchSize() int {
 	return size
 }
 
+// LookupPeer looks up a peer by its public key.
+//
+// If the peer does not exist and a [PeerLookupFunc] is set (via
+// [Device.SetPeerLookupFunc]), then that function is used to create the peer
+// before returning it. Peers created via this mechanism exist only until their
+// state machine reaches idle, and then the peers are removed.
+//
+// If the peer does not exist and no [PeerLookupFunc] is set, nil is returned.
+//
+// Use [Device.LookupActivePeer] to only return already-existing peers, without
+// using a [PeerLookupFunc].
 func (device *Device) LookupPeer(pk NoisePublicKey) *Peer {
 	device.peers.RLock()
-	defer device.peers.RUnlock()
+	p, ok := device.peers.keyMap[pk]
+	lookupFunc := device.peers.lookupFunc
+	device.peers.RUnlock()
+	if ok || lookupFunc == nil {
+		return p
+	}
 
-	return device.peers.keyMap[pk]
+	conf, ok := lookupFunc(pk)
+	if !ok || conf == nil {
+		return nil
+	}
+
+	p, err := device.NewPeer(pk)
+	if err != nil {
+		if errors.Is(err, errAddExistingPeer) {
+			device.peers.RLock()
+			defer device.peers.RUnlock()
+			return device.peers.keyMap[pk]
+		}
+		device.log.Errorf("Failed to create peer: %v", err)
+		return nil
+	}
+	p.SetPresharedKey(conf.PresharedKey)
+	p.SetAllowedIPs(conf.AllowedIPs)
+	p.deleteOnIdle = true
+	if conf.Endpoint != nil {
+		p.endpoint.Lock()
+		p.endpoint.val = conf.Endpoint
+		p.endpoint.Unlock()
+	}
+	p.Start()
+	return p
 }
 
+// LookupActivePeer looks up a peer by its public key.
+//
+// Unlike [Device.LookupPeer], this function does not use a [PeerLookupFunc] to
+// create the peer if it does not already exist.
+//
+// If the peer does not exist or was created lazily via [PeerLookupFunc]
+// and has subsequently idled away, it returns (nil, false).
+func (device *Device) LookupActivePeer(pk NoisePublicKey) (_ *Peer, ok bool) {
+	device.peers.RLock()
+	defer device.peers.RUnlock()
+	p, ok := device.peers.keyMap[pk]
+	return p, ok
+}
+
+var errAddExistingPeer = errors.New("adding existing peer")
+
 func (device *Device) RemovePeer(key NoisePublicKey) {
 	device.peers.Lock()
 	defer device.peers.Unlock()
@@ -367,6 +613,175 @@ func (device *Device) RemoveAllPeers() {
 	device.peers.keyMap = make(map[NoisePublicKey]*Peer)
 }
 
+// RemoveMatchingPeers removes all peers for which shouldRemove returns true.
+//
+// It returns the number of peers removed.
+func (device *Device) RemoveMatchingPeers(shouldRemove func(NoisePublicKey) bool) (numRemoved int) {
+	device.peers.Lock()
+	defer device.peers.Unlock()
+
+	for key, peer := range device.peers.keyMap {
+		if shouldRemove(key) {
+			removePeerLocked(device, peer, key)
+			numRemoved++
+		}
+	}
+	return numRemoved
+}
+
+// NewPeerConfig are the configuration parameters for a new peer created via a
+// [PeerLookupFunc] func.
+type NewPeerConfig struct {
+	// AllowedIPs is the initial set of allowed IPs for the new peer.
+	AllowedIPs []netip.Prefix
+
+	// PresharedKey is the initial pre-shared key for the new peer. The zero
+	// value disables the optional WireGuard pre-shared-key layer.
+	PresharedKey NoisePresharedKey
+
+	// Endpoint, if non-nil, sets the endpoint for newly created peers.
+	// The endpoint is pinned for the lifetime of the peer.
+	Endpoint conn.Endpoint
+}
+
+// PeerLookupFunc is the type of function used to look up peers by public key
+// when receiving packets for unknown peers.
+//
+// If it returns nil, the peer is not known.
+//
+// Otherwise, returning non-nil signals that wireguard-go should create the peer
+// with the provided allowed IPs.
+//
+// See [Device.SetPeerLookupFunc] and [Device.LookupPeer].
+type PeerLookupFunc func(NoisePublicKey) (_ *NewPeerConfig, ok bool)
+
+// PeerByIPPacketFunc is the type of function used to look up a peer to send to
+// for a given src/dst IP pair. The ipPkt parameter is the raw IP packet being
+// routed; callers needing transport-layer ports or other header fields may parse
+// them from ipPkt, but must handle IP fragmentation (ports may be absent on
+// non-first fragments) and protocols that do not use ports (e.g. ICMP).
+//
+// Except for experimental use cases, dst is the only address
+// that should be relied upon when looking up a peer.
+//
+// If it returns ok=false, the peer is not known.
+//
+// See [Device.SetPeerByIPPacketFunc] and [Device.SetPeerLookupFunc].
+type PeerByIPPacketFunc func(src, dst netip.Addr, ipPkt []byte) (_ NoisePublicKey, ok bool)
+
+// PeerSessionState is the current WireGuard session state for a peer.
+type PeerSessionState uint8
+
+const (
+	// PeerSessionNone means there is no handshake in progress and no session key
+	// material retained for this peer.
+	PeerSessionNone PeerSessionState = iota
+
+	// PeerSessionHandshake means a handshake is in progress for this peer, but
+	// there is not currently a usable WireGuard session.
+	PeerSessionHandshake
+
+	// PeerSessionEstablished means the peer has a completed WireGuard session
+	// with usable session key material.
+	PeerSessionEstablished
+
+	// PeerSessionExpired means the peer's session key material is no longer
+	// considered usable, but final key cleanup or lazy peer removal may not have
+	// happened yet.
+	PeerSessionExpired
+)
+
+// PeerSessionStateFunc is called when a peer's WireGuard session state changes.
+//
+// Calls are serialized per peer and delivered in that peer's transition order. The
+// callback must be cheap and must not call back into Device.
+type PeerSessionStateFunc func(peer NoisePublicKey, state PeerSessionState)
+
+// SetPeerLookupFunc sets the function used to look up peers by public key
+// when receiving packets for unknown peers.
+func (device *Device) SetPeerLookupFunc(f PeerLookupFunc) {
+	device.peers.Lock()
+	defer device.peers.Unlock()
+	device.peers.lookupFunc = f
+}
+
+// SetPeerByIPPacketFunc sets the function used to look up peers by IP address
+// when sending packets to unknown peers.
+func (device *Device) SetPeerByIPPacketFunc(f PeerByIPPacketFunc) {
+	device.allowedips.mu.Lock()
+	defer device.allowedips.mu.Unlock()
+	device.allowedips.peerByIPPacketFunc = f
+	device.allowedips.device = device
+}
+
+// SetSessionStateFunc sets the function used to observe peer WireGuard session
+// state changes.
+//
+// It does not replay current state. Callers that need a complete view should set
+// it before peers are started or lazily created, and maintain any snapshots,
+// sequence numbers, and pubsub state outside wireguard-go.
+//
+// The callback must be concurrent-safe and must not call back into Device.
+func (device *Device) SetSessionStateFunc(f PeerSessionStateFunc) {
+	if f == nil {
+		device.peerStateFn.Store(nil)
+		return
+	}
+	device.peerStateFn.Store(&f)
+}
+
+// MaxPriorityMessageContentSize is the maximum size of a message returned by a
+// [PeerPriorityMessageFunc]. It's a power of 2 that leaves significant space
+// when accounting for all WireGuard overhead and encapsulating network protocol
+// headers. Future adjustments to this value should consider all these overheads
+// and any [conn.Bind] implementation limitations.
+const MaxPriorityMessageContentSize = 512
+
+// PeerPriorityMessageFunc is called when a peer's WireGuard session keypair is
+// established (or re-keyed) for forward data transmission.
+//
+// The returned message is transmitted to the peer in priority fashion. Priority
+// means it cannot be evicted from the staged packet queue by non-priority
+// (read from [tun.Device]) packets. It avoids the staged queue altogether.
+//
+// The callback must be cheap and must not call back into [Device]. A zero length
+// message or a message whose length exceeds [MaxPriorityMessageContentSize] will
+// be silently dropped. Message should start with an IPv4 or IPv6 header as it
+// is subject to allowed IPs lookup on the receiver, same as any other transport
+// message.
+type PeerPriorityMessageFunc func(peer NoisePublicKey) (msg []byte)
+
+// SetPriorityMessageOnEstablishmentFunc sets a function to be used for sending
+// a priority message around session establishment. See [PeerPriorityMessageFunc]
+// docs for more details. A nil value clears any previously set value.
+func (device *Device) SetPriorityMessageOnEstablishmentFunc(f PeerPriorityMessageFunc) {
+	if f == nil {
+		device.priorityMsgFn.Store(nil)
+		return
+	}
+	device.priorityMsgFn.Store(&f)
+}
+
+// ScheduleHandshakeOnUserSend marks the peer for handshake initiation immediately
+// following its next outbound transport message. It is a no-op if the peer is
+// unknown or currently holds no key material.
+//
+// The initiation remains subject to [RekeyTimeout] elapsing since the last
+// handshake message was sent. A request blocked by that window stays armed for
+// a later transport message rather than being dropped.
+func (device *Device) ScheduleHandshakeOnUserSend(peer NoisePublicKey) {
+	device.peers.RLock()
+	defer device.peers.RUnlock()
+	p, ok := device.peers.keyMap[peer]
+	if !ok {
+		return
+	}
+	if !p.hasKeyMaterial() {
+		return
+	}
+	p.handshakeOnUserSend.Store(true)
+}
+
 func (device *Device) Close() {
 	device.state.Lock()
 	defer device.state.Unlock()
@@ -408,16 +823,25 @@ func (device *Device) SendKeepalivesToPeersWithCurrentKeypair() {
 		return
 	}
 
+	// Collect the set of peers to keepalive under peers.RLock, then release
+	// before invoking SendKeepalive. SendKeepalive can reach
+	// CreateMessageInitiation which acquires staticIdentity.RLock; holding
+	// peers.RLock across that path would invert the
+	// staticIdentity < peers hierarchy (see lock-ordering.md).
+	var peers []*Peer
 	device.peers.RLock()
 	for _, peer := range device.peers.keyMap {
 		peer.keypairs.RLock()
 		sendKeepalive := peer.keypairs.current != nil && !peer.keypairs.current.created.Add(RejectAfterTime).Before(time.Now())
 		peer.keypairs.RUnlock()
 		if sendKeepalive {
-			peer.SendKeepalive()
+			peers = append(peers, peer)
 		}
 	}
 	device.peers.RUnlock()
+	for _, peer := range peers {
+		peer.SendKeepalive()
+	}
 }
 
 // closeBindLocked closes the device's net.bind.

vendor/github.com/tailscale/wireguard-go/device/mobilequirks.go

@@ -1,19 +0,0 @@
-/* SPDX-License-Identifier: MIT
- *
- * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
- */
-
-package device
-
-// DisableSomeRoamingForBrokenMobileSemantics should ideally be called before peers are created,
-// though it will try to deal with it, and race maybe, if called after.
-func (device *Device) DisableSomeRoamingForBrokenMobileSemantics() {
-	device.net.brokenRoaming = true
-	device.peers.RLock()
-	for _, peer := range device.peers.keyMap {
-		peer.endpoint.Lock()
-		peer.endpoint.disableRoaming = peer.endpoint.val != nil
-		peer.endpoint.Unlock()
-	}
-	device.peers.RUnlock()
-}

vendor/github.com/tailscale/wireguard-go/device/noise-protocol.go

@@ -308,7 +308,7 @@ func (device *Device) CreateMessageInitiation(peer *Peer) (*MessageInitiation, e
 		handshake.chainKey[:],
 		ss[:],
 	)
-	aead, _ := chacha20poly1305.New(key[:])
+	aead, _ := chacha20poly1305New(key[:])
 	aead.Seal(msg.Static[:0], ZeroNonce[:], device.staticIdentity.publicKey[:], handshake.hash[:])
 	handshake.mixHash(msg.Static[:])
 
@@ -323,7 +323,7 @@ func (device *Device) CreateMessageInitiation(peer *Peer) (*MessageInitiation, e
 		handshake.precomputedStaticStatic[:],
 	)
 	timestamp := tai64n.Now()
-	aead, _ = chacha20poly1305.New(key[:])
+	aead, _ = chacha20poly1305New(key[:])
 	aead.Seal(msg.Timestamp[:0], ZeroNonce[:], timestamp[:], handshake.hash[:])
 
 	// assign index
@@ -349,22 +349,27 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation, endpoint
 		return nil
 	}
 
+	// Snapshot staticIdentity so we don't hold the RLock across LookupPeer,
+	// which may call NewPeer (reentrant RLock deadlocks against a pending
+	// SetPrivateKey writer; see lock-ordering.md).
 	device.staticIdentity.RLock()
-	defer device.staticIdentity.RUnlock()
+	publicKey := device.staticIdentity.publicKey
+	privateKey := device.staticIdentity.privateKey
+	device.staticIdentity.RUnlock()
 
-	mixHash(&hash, &InitialHash, device.staticIdentity.publicKey[:])
+	mixHash(&hash, &InitialHash, publicKey[:])
 	mixHash(&hash, &hash, msg.Ephemeral[:])
 	mixKey(&chainKey, &InitialChainKey, msg.Ephemeral[:])
 
 	// decrypt static key
 	var peerPK NoisePublicKey
 	var key [chacha20poly1305.KeySize]byte
-	ss, err := device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral)
+	ss, err := privateKey.sharedSecret(msg.Ephemeral)
 	if err != nil {
 		return nil
 	}
 	KDF2(&chainKey, &key, chainKey[:], ss[:])
-	aead, _ := chacha20poly1305.New(key[:])
+	aead, _ := chacha20poly1305New(key[:])
 	_, err = aead.Open(peerPK[:0], ZeroNonce[:], msg.Static[:], hash[:])
 	if err != nil {
 		return nil
@@ -379,7 +384,7 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation, endpoint
 	}
 
 	peer := device.LookupPeer(peerPK)
-	if peer == nil || !peer.isRunning.Load() {
+	if peer == nil || !peer.runningState.isRunning.Load() {
 		return nil
 	}
 
@@ -401,7 +406,7 @@ func (device *Device) ConsumeMessageInitiation(msg *MessageInitiation, endpoint
 		chainKey[:],
 		handshake.precomputedStaticStatic[:],
 	)
-	aead, _ = chacha20poly1305.New(key[:])
+	aead, _ = chacha20poly1305New(key[:])
 	_, err = aead.Open(timestamp[:0], ZeroNonce[:], msg.Timestamp[:], hash[:])
 	if err != nil {
 		handshake.mutex.RUnlock()
@@ -507,7 +512,7 @@ func (device *Device) CreateMessageResponse(peer *Peer) (*MessageResponse, error
 
 	handshake.mixHash(tau[:])
 
-	aead, _ := chacha20poly1305.New(key[:])
+	aead, _ := chacha20poly1305New(key[:])
 	aead.Seal(msg.Empty[:0], ZeroNonce[:], nil, handshake.hash[:])
 	handshake.mixHash(msg.Empty[:])
 
@@ -534,6 +539,14 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 		chainKey [blake2s.Size]byte
 	)
 
+	// Snapshot the static private key before acquiring handshake.mutex so
+	// that handshake.mutex is never held while acquiring staticIdentity
+	// (which would invert the staticIdentity < handshake.mutex hierarchy;
+	// see lock-ordering.md).
+	device.staticIdentity.RLock()
+	privateKey := device.staticIdentity.privateKey
+	device.staticIdentity.RUnlock()
+
 	ok := func() bool {
 		// lock handshake state
 
@@ -544,11 +557,6 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 			return false
 		}
 
-		// lock private key for reading
-
-		device.staticIdentity.RLock()
-		defer device.staticIdentity.RUnlock()
-
 		// finish 3-way DH
 
 		mixHash(&hash, &handshake.hash, msg.Ephemeral[:])
@@ -561,7 +569,7 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 		mixKey(&chainKey, &chainKey, ss[:])
 		setZero(ss[:])
 
-		ss, err = device.staticIdentity.privateKey.sharedSecret(msg.Ephemeral)
+		ss, err = privateKey.sharedSecret(msg.Ephemeral)
 		if err != nil {
 			return false
 		}
@@ -583,7 +591,7 @@ func (device *Device) ConsumeMessageResponse(msg *MessageResponse) *Peer {
 
 		// authenticate transcript
 
-		aead, _ := chacha20poly1305.New(key[:])
+		aead, _ := chacha20poly1305New(key[:])
 		_, err = aead.Open(nil, ZeroNonce[:], msg.Empty[:], hash[:])
 		if err != nil {
 			return false
@@ -658,8 +666,8 @@ func (peer *Peer) BeginSymmetricSession() error {
 	// create AEAD instances
 
 	keypair := new(Keypair)
-	keypair.send, _ = chacha20poly1305.New(sendKey[:])
-	keypair.receive, _ = chacha20poly1305.New(recvKey[:])
+	keypair.send, _ = chacha20poly1305New(sendKey[:])
+	keypair.receive, _ = chacha20poly1305New(recvKey[:])
 
 	setZero(sendKey[:])
 	setZero(recvKey[:])

vendor/github.com/tailscale/wireguard-go/device/packetbuf.go

@@ -0,0 +1,66 @@
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package device
+
+import (
+	"fmt"
+	"sync/atomic"
+
+	"github.com/tailscale/wireguard-go/tun"
+	"golang.org/x/crypto/poly1305"
+)
+
+const (
+	outboundPlaintextHeadroom = MessageEncapsulatingTransportSize + MessageTransportHeaderSize
+
+	outboundPlaintextTailroom = PaddingMultiple - 1 + poly1305.TagSize
+
+	outboundPlaintextSpacing = max(outboundPlaintextHeadroom+outboundPlaintextTailroom, tun.ReadPacketSpacing)
+
+	// singlePacketSlabSize is the size to be used for [packetBuf.slab] when
+	// neither [tun.Device] nor [conn.Bind] support batched reads.
+	singlePacketSlabSize = MaxContentSize + 2*tun.ReadPacketSpacing
+
+	// batchingSlabSize is the size to be used for [packetBuf.slab] when either
+	// [tun.Device] or [conn.Bind] support batched reads. This covers the
+	// [tun.GSOSplit] worst case for minimum-sized MTU/MSS, which is documented
+	// in [tun.TestGSOSplitMinimumGSOSize].
+	batchingSlabSize = 2 * (1<<16 - 1)
+)
+
+// The tun package cannot import device, so we assert [tun.ReadPacketSpacing] is
+// greater than or equal to [outboundPlaintextSpacing] instead.
+const _ = uint(tun.ReadPacketSpacing - outboundPlaintextSpacing)
+
+// packetBuf is a reference counted packet buffer.
+type packetBuf struct {
+	slab     []byte
+	refCount atomic.Int32
+	cleanup  func(*packetBuf)
+}
+
+// newPacketBuf returns a new [*packetBuf] of the provided size with its
+// reference count set to zero. cleanup runs for the returned [*packetBuf] when
+// its ref count moves 1->0.
+func newPacketBuf(size int, cleanup func(*packetBuf)) *packetBuf {
+	return &packetBuf{
+		slab:    make([]byte, size),
+		cleanup: cleanup,
+	}
+}
+
+func (p *packetBuf) incRef() {
+	p.refCount.Add(1)
+}
+
+func (p *packetBuf) decRef() {
+	c := p.refCount.Add(-1)
+	if c == 0 {
+		p.cleanup(p)
+	} else if c < 0 {
+		panic(fmt.Sprintf("packetBuf.decRef below zero (%d)", c))
+	}
+}

vendor/github.com/tailscale/wireguard-go/device/peer.go

@@ -7,7 +7,11 @@ package device
 
 import (
 	"container/list"
+	"encoding/binary"
 	"errors"
+	"net/netip"
+	"runtime"
+	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -16,26 +20,53 @@ import (
 )
 
 type Peer struct {
-	isRunning         atomic.Bool
-	keypairs          Keypairs
-	handshake         Handshake
-	device            *Device
-	stopping          sync.WaitGroup // routines pending stop
-	txBytes           atomic.Uint64  // bytes send to peer (endpoint)
-	rxBytes           atomic.Uint64  // bytes received from peer
-	lastHandshakeNano atomic.Int64   // nano seconds since epoch
+	runningState struct {
+		// isRunning is swapped across [Peer.Start] and [Peer.Stop]. Direct
+		// access loads are best-effort and inherently racey.
+		isRunning atomic.Bool
+		// queueWriters is the count of outstanding goroutines performing
+		// potential write operations against [Peer.queue].* channels.
+		// It is semantically similar to a [sync.WaitGroup], but free
+		// from the event order constraints of [sync.WaitGroup.Add]. Increments
+		// from zero may run concurrent to the "waiting" side.
+		queueWriters atomic.Int32
+		// queueReaders is the count of outstanding goroutines performing
+		// potential read operations against [Peer.queue.inbound] and
+		// [Peer.queue.outbound].
+		queueReaders sync.WaitGroup
+	}
+
+	keypairs            Keypairs
+	handshake           Handshake
+	device              *Device
+	txBytes             atomic.Uint64 // bytes send to peer (endpoint)
+	rxBytes             atomic.Uint64 // bytes received from peer
+	lastHandshakeNano   atomic.Int64  // nanoseconds since epoch
+	handshakeOnUserSend atomic.Bool   // attempt handshake following next outbound transport
+
+	sessionState struct {
+		sync.Mutex
+		current        PeerSessionState
+		sessionExpires time.Time
+	}
+
+	// deleteOnIdle indicates whether the peer should be deleted when idle
+	// because it was auto-created via a Device.PeerLookupFunc.
+	//
+	// This field should only be set once, before the peer is started.
+	deleteOnIdle bool
 
 	endpoint struct {
 		sync.Mutex
-		val            conn.Endpoint
-		clearSrcOnTx   bool // signal to val.ClearSrc() prior to next packet transmission
-		disableRoaming bool
+		val          conn.Endpoint
+		clearSrcOnTx bool // signal to val.ClearSrc() prior to next packet transmission
 	}
 
 	timers struct {
 		retransmitHandshake     *Timer
 		sendKeepalive           *Timer
 		newHandshake            *Timer
+		sessionExpired          *Timer
 		zeroKeyMaterial         *Timer
 		persistentKeepalive     *Timer
 		handshakeAttempts       atomic.Uint32
@@ -44,18 +75,43 @@ type Peer struct {
 	}
 
 	state struct {
-		sync.Mutex // protects against concurrent Start/Stop
+		sync.Mutex // protects against concurrent Start/Stop, and fields below
+
+		allowedIPs []netip.Prefix
+
+		// testAllowedIP, if non-nil, is used to test whether the peer is
+		// allowed to send a packet from the given IP address. It can be read
+		// without locking, but must be set with the state mutex locked.
+		testAllowedIP atomic.Pointer[func(netip.Addr) bool]
 	}
 
+	// queue channel write operations must be wrapped by [Peer.doIfRunning]
 	queue struct {
 		staged   chan *QueueOutboundElementsContainer // staged packets before a handshake is available
-		outbound *autodrainingOutboundQueue           // sequential ordering of udp transmission
-		inbound  *autodrainingInboundQueue            // sequential ordering of tun writing
+		outbound chan *QueueOutboundElementsContainer // sequential ordering of udp transmission
+		inbound  chan *QueueInboundElementsContainer  // sequential ordering of tun writing
 	}
 
 	cookieGenerator             CookieGenerator
 	trieEntries                 list.List
 	persistentKeepaliveInterval atomic.Uint32
+
+	// flowID pins this peer's decrypted packets to a single TUN write queue
+	// when the device is a [tun.MultiQueueDevice].
+	// See [Peer.writeTUN].
+	//
+	// The kernel picks a TUN queue by 5-tuple for new (or quiescent,
+	// TUN_FLOW_EXPIRE=3s) flows. A pin learned from our writes overrides
+	// the hash. Kernel delivers a packet to the stack before recording the pin,
+	// so the first reply to a flow we write leaves on the hashed queue and the
+	// rest on ours. Both queues then carry that flow briefly, which may
+	// introduce overlay reordering.
+	// See https://elixir.bootlin.com/linux/v7.0/source/drivers/net/tun.c#L461
+	//
+	// TODO match or influence kernel hashing when writing to TUN to avoid
+	// reorders. Note that the kernel hash is siphash keyed by an unexported
+	// random value. This can be overridden with TUNSETSTEERINGEBPF ioctl.
+	flowID uint32
 }
 
 func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
@@ -80,14 +136,17 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 
 	peer.cookieGenerator.Init(pk)
 	peer.device = device
-	peer.queue.outbound = newAutodrainingOutboundQueue(device)
-	peer.queue.inbound = newAutodrainingInboundQueue(device)
-	peer.queue.staged = make(chan *QueueOutboundElementsContainer, QueueStagedSize)
+	// Public keys are uniformly distributed, so the low bytes spread fine.
+	peer.flowID = binary.LittleEndian.Uint32(pk[:4])
+
+	// staged is never closed, and it can be accessed concurrent to [Peer.Start],
+	// so we init here instead of [Peer.Start].
+	peer.queue.staged = make(chan *QueueOutboundElementsContainer, device.config.queueStagedSize)
 
 	// map public key
 	_, ok := device.peers.keyMap[pk]
 	if ok {
-		return nil, errors.New("adding existing peer")
+		return nil, errAddExistingPeer
 	}
 
 	// pre-compute DH
@@ -100,7 +159,6 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 	// reset endpoint
 	peer.endpoint.Lock()
 	peer.endpoint.val = nil
-	peer.endpoint.disableRoaming = false
 	peer.endpoint.clearSrcOnTx = false
 	peer.endpoint.Unlock()
 
@@ -113,6 +171,35 @@ func (device *Device) NewPeer(pk NoisePublicKey) (*Peer, error) {
 	return peer, nil
 }
 
+// SetAllowedIPs sets the allowed IP prefixes for this peer.
+//
+// If the allowedIPs are unchanged since the last call, this method is a no-op.
+// It's the caller's responsibility to ensure that no two peers have duplicate
+// allowed IPs. If so, the last writer wins.
+func (p *Peer) SetAllowedIPs(allowedIPs []netip.Prefix) {
+	p.state.Lock()
+	defer p.state.Unlock()
+
+	if slices.Equal(p.state.allowedIPs, allowedIPs) {
+		return
+	}
+	p.device.allowedips.setPeerPrefixes(p, allowedIPs)
+
+	allowedIPs = slices.Clone(allowedIPs) // avoid retaining caller's slice
+	p.state.allowedIPs = allowedIPs
+
+	f := mkIPInCIDRsTestFunc(allowedIPs)
+	p.state.testAllowedIP.Store(&f)
+}
+
+// SetPresharedKey sets the optional WireGuard pre-shared key for this peer.
+// The zero value disables the pre-shared-key layer.
+func (p *Peer) SetPresharedKey(psk NoisePresharedKey) {
+	p.handshake.mutex.Lock()
+	defer p.handshake.mutex.Unlock()
+	p.handshake.presharedKey = psk
+}
+
 // SendBuffers sends buffers to peer. WireGuard packet data in each element of
 // buffers must be preceded by MessageEncapsulatingTransportSize number of
 // bytes.
@@ -147,6 +234,12 @@ func (peer *Peer) SendBuffers(buffers [][]byte) error {
 	return err
 }
 
+// writeTUN writes bufs to the TUN queue this peer is pinned to, keeping the
+// peer's decrypted packets in order.
+func (peer *Peer) writeTUN(bufs [][]byte, offset int) (int, error) {
+	return peer.device.tun.writeTo(int(peer.flowID), bufs, offset)
+}
+
 func (peer *Peer) String() string {
 	// The awful goo that follows is identical to:
 	//
@@ -183,16 +276,20 @@ func (peer *Peer) Start() {
 	peer.state.Lock()
 	defer peer.state.Unlock()
 
-	if peer.isRunning.Load() {
+	if peer.runningState.isRunning.Load() {
 		return
 	}
 
 	device := peer.device
 	device.log.Verbosef("%v - Starting", peer)
 
+	// init inbound & outbound packet queues
+	peer.queue.outbound = make(chan *QueueOutboundElementsContainer, device.config.queueOutboundSize)
+	peer.queue.inbound = make(chan *QueueInboundElementsContainer, device.config.queueInboundSize)
+
 	// reset routine state
-	peer.stopping.Wait()
-	peer.stopping.Add(2)
+	peer.runningState.queueReaders.Wait()
+	peer.runningState.queueReaders.Add(2)
 
 	peer.handshake.mutex.Lock()
 	peer.handshake.lastSentHandshake = time.Now().Add(-(RekeyTimeout + time.Second))
@@ -202,20 +299,28 @@ func (peer *Peer) Start() {
 
 	peer.timersStart()
 
-	device.flushInboundQueue(peer.queue.inbound)
-	device.flushOutboundQueue(peer.queue.outbound)
-
 	// Use the device batch size, not the bind batch size, as the device size is
 	// the size of the batch pools.
 	batchSize := peer.device.BatchSize()
 	go peer.RoutineSequentialSender(batchSize)
 	go peer.RoutineSequentialReceiver(batchSize)
 
-	peer.isRunning.Store(true)
+	peer.runningState.isRunning.Store(true)
+
+	// A lazily-created peer that never completes a handshake otherwise never
+	// arms its reaping timer. Arm it here, while running under state.Lock, so
+	// it's reclaimed after RejectAfterTime*3 of no session and is guaranteed to
+	// be torn down by a matching Stop. A completed handshake re-Mods it.
+	if peer.deleteOnIdle {
+		peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3)
+	}
 }
 
 func (peer *Peer) ZeroAndFlushAll() {
 	device := peer.device
+	if peer.timers.sessionExpired != nil {
+		peer.timers.sessionExpired.Del()
+	}
 
 	// clear key pairs
 
@@ -238,6 +343,11 @@ func (peer *Peer) ZeroAndFlushAll() {
 	handshake.mutex.Unlock()
 
 	peer.FlushStagedPackets()
+
+	peer.sessionState.Lock()
+	peer.sessionState.sessionExpires = time.Time{}
+	peer.noteSessionStateLocked(PeerSessionNone)
+	peer.sessionState.Unlock()
 }
 
 func (peer *Peer) ExpireCurrentKeypairs() {
@@ -257,36 +367,181 @@ func (peer *Peer) ExpireCurrentKeypairs() {
 		next.sendNonce.Store(RejectAfterMessages)
 	}
 	keypairs.Unlock()
+
+	peer.sessionState.Lock()
+	peer.sessionState.sessionExpires = time.Time{}
+	peer.noteSessionStateLocked(PeerSessionExpired)
+	peer.sessionState.Unlock()
+}
+
+// queueOutboundIfRunning queues elems for outbound encryption & transmission to
+// peer if peer is running, otherwise elems are returned to their respective
+// pools for later re-use.
+func (peer *Peer) queueOutboundIfRunning(elems *QueueOutboundElementsContainer) (queued bool) {
+	if queued = peer.doIfRunning(func() {
+		elems.filling.Add(1)
+		peer.queue.outbound <- elems
+		peer.device.queue.encryption.c <- elems
+	}); !queued {
+		for _, elem := range elems.elems {
+			peer.device.PutOutboundElement(elem)
+		}
+		peer.device.PutOutboundElementsContainer(elems)
+	}
+	return queued
+}
+
+// queueInboundIfRunning queues elems for inbound decryption from peer if peer
+// is running, otherwise elems are returned to their respective pools for later
+// re-use.
+func (peer *Peer) queueInboundIfRunning(elems *QueueInboundElementsContainer) (queued bool) {
+	if queued = peer.doIfRunning(func() {
+		elems.filling.Add(1)
+		peer.queue.inbound <- elems
+		peer.device.queue.decryption.c <- elems
+	}); !queued {
+		for _, elem := range elems.elems {
+			peer.device.PutInboundElement(elem)
+		}
+		peer.device.PutInboundElementsContainer(elems)
+	}
+	return queued
+}
+
+// doIfRunning executes doQueueCrit if the [Peer] is running, i.e. the [Peer] is
+// not stopped or stopping. It returns true if doQueueCrit ran, otherwise it
+// returns false.
+//
+// doIfRunning exists to enable safe, bare (no select{}) write operations against
+// [Peer.queue].* channels.
+//
+// doIfRunning makes no guarantee about the running generation of the [Peer], i.e.
+// a [Peer] may stop then start all while a goroutine is within doIfRunning, but
+// before doQueueCrit has run.
+//
+// doIfRunning is built on the assumption that the supplied critical sections are
+// relatively cheap, as the mutually excluded peer-stopping side
+// ([Peer.waitQueueActors]) busy-waits for actors to exit.
+func (peer *Peer) doIfRunning(doQueueCrit func()) (ranCrit bool) {
+	if !peer.runningState.isRunning.Load() {
+		return false
+	}
+	peer.runningState.queueWriters.Add(1)
+	defer peer.runningState.queueWriters.Add(-1)
+	if peer.runningState.isRunning.Load() {
+		doQueueCrit()
+		return true
+	}
+	return false
+}
+
+// waitQueueActors waits for [Peer.runningState.queueWriters] and [Peer.runningState.queueReaders]
+// to reach zero. It should be called after [Peer.runningState.isRunning] is
+// swapped false.
+func (peer *Peer) waitQueueActors() {
+	drainInboundOutbound := func() {
+		for {
+			select {
+			case container := <-peer.queue.inbound:
+				container.filling.Wait()
+				for _, elem := range container.elems {
+					peer.device.PutInboundElement(elem)
+				}
+				peer.device.PutInboundElementsContainer(container)
+			case container := <-peer.queue.outbound:
+				container.filling.Wait()
+				for _, elem := range container.elems {
+					peer.device.PutOutboundElement(elem)
+				}
+				peer.device.PutOutboundElementsContainer(container)
+			default:
+				return
+			}
+		}
+	}
+
+	// Busy-wait for queue writers to return.
+	for peer.runningState.queueWriters.Load() != 0 {
+		// We drain at every iteration to give our busy-wait some meaningful
+		// work that can unblock actors performing channel writes. The goal at
+		// this point is to get them to return as soon as possible. It's ok if
+		// there are outstanding elements on packet queues once queueWriters
+		// reaches zero, our reader shutdown below will handle those stragglers.
+		drainInboundOutbound()
+		// We don't want our busy-wait starving actors, so explicitly yield back
+		// to the runtime, to bias towards actor goroutines getting CPU time.
+		runtime.Gosched()
+	}
+
+	// Signal to RoutineSequentialSender and RoutineSequentialReceiver to
+	// exit. Let them deal with any outstanding elements on the associated
+	// channels.
+	close(peer.queue.inbound)
+	close(peer.queue.outbound)
+	peer.runningState.queueReaders.Wait()
 }
 
 func (peer *Peer) Stop() {
 	peer.state.Lock()
 	defer peer.state.Unlock()
 
-	if !peer.isRunning.Swap(false) {
+	if !peer.runningState.isRunning.Swap(false) {
 		return
 	}
 
+	peer.waitQueueActors()
+
 	peer.device.log.Verbosef("%v - Stopping", peer)
 
 	peer.timersStop()
-	// Signal that RoutineSequentialSender and RoutineSequentialReceiver should exit.
-	peer.queue.inbound.c <- nil
-	peer.queue.outbound.c <- nil
-	peer.stopping.Wait()
 	peer.device.queue.encryption.wg.Done() // no more writes to encryption queue from us
 
 	peer.ZeroAndFlushAll()
 }
 
-func (peer *Peer) SetEndpointFromPacket(endpoint conn.Endpoint) {
-	peer.endpoint.Lock()
-	defer peer.endpoint.Unlock()
-	if peer.endpoint.disableRoaming {
+func (peer *Peer) noteSessionState(state PeerSessionState) {
+	peer.sessionState.Lock()
+	defer peer.sessionState.Unlock()
+	peer.noteSessionStateLocked(state)
+}
+
+// noteSessionStateLocked records a session state transition and delivers the
+// callback. The caller must hold peer.sessionState.Mutex during the
+// state determination and transition.
+func (peer *Peer) noteSessionStateLocked(state PeerSessionState) {
+	if peer.sessionState.current == state {
 		return
 	}
-	peer.endpoint.clearSrcOnTx = false
-	peer.endpoint.val = endpoint
+	peer.sessionState.current = state
+	if f := peer.device.peerStateFn.Load(); f != nil {
+		(*f)(peer.handshake.remoteStatic, state)
+	}
+}
+
+func (peer *Peer) noteSessionHandshakeStarted() {
+	peer.sessionState.Lock()
+	defer peer.sessionState.Unlock()
+	if peer.sessionState.current == PeerSessionEstablished {
+		return
+	}
+	peer.noteSessionStateLocked(PeerSessionHandshake)
+}
+
+func (peer *Peer) noteSessionHandshakeStopped() {
+	peer.sessionState.Lock()
+	defer peer.sessionState.Unlock()
+	state := PeerSessionNone
+	if peer.hasKeyMaterial() {
+		state = PeerSessionExpired
+	}
+	peer.noteSessionStateLocked(state)
+}
+
+func (peer *Peer) hasKeyMaterial() bool {
+	keypairs := &peer.keypairs
+	keypairs.RLock()
+	defer keypairs.RUnlock()
+	return keypairs.previous != nil || keypairs.current != nil || keypairs.next.Load() != nil
 }
 
 func (peer *Peer) markEndpointSrcForClearing() {

vendor/github.com/tailscale/wireguard-go/device/pools.go

@@ -7,6 +7,8 @@ package device
 
 import (
 	"sync"
+
+	"github.com/tailscale/wireguard-go/tun"
 )
 
 type WaitPool struct {
@@ -46,29 +48,77 @@ func (p *WaitPool) Put(x any) {
 	p.cond.Signal()
 }
 
+const (
+	// smallPacketBufSize is the size to be used for [packetBuf.slab]'s pooled in
+	// [Device.pool.smallPacketBufs]. It's large enough to fit WireGuard keepalive
+	// messages, WireGuard-encrypted [MaxPriorityMessageContentSize], and
+	// WireGuard-encrypted internet MTU messages.
+	smallPacketBufSize = 2048
+
+	internetMTU = 1500
+	// assert that [smallPacketBufSize] is sufficient to hold [internetMTU]-sized
+	// plaintext read via [tun.Device.Read], for eventual encryption and
+	// transmission as a WireGuard transport message.
+	_ uint = smallPacketBufSize -
+		tun.ReadPacketSpacing -
+		internetMTU -
+		outboundPlaintextTailroom
+	// assert that [smallPacketBufSize] is sufficient to hold [MaxPriorityMessageContentSize]-sized
+	// plaintext for eventual encryption and transmission as a WireGuard transport
+	// message. This assertion also covers keepalive messages, which only have
+	// the auth tag, accounted for in [outboundPlaintextTailroom], following
+	// the WireGuard header.
+	_ uint = smallPacketBufSize -
+		outboundPlaintextHeadroom -
+		MaxPriorityMessageContentSize -
+		outboundPlaintextTailroom
+
+	// minPacketBufSizeForDistinctSmallPool represents the minimum [packetBuf.slab]
+	// size at which a distinct ([Device.pool.smallPacketBufs]) is created.
+	minPacketBufSizeForDistinctSmallPool = 2 * smallPacketBufSize
+)
+
 func (device *Device) PopulatePools() {
-	device.pool.inboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any {
+	device.pool.inboundElementsContainer = NewWaitPool(device.config.preallocatedBuffersPerPool, func() any {
 		s := make([]*QueueInboundElement, 0, device.BatchSize())
 		return &QueueInboundElementsContainer{elems: s}
 	})
-	device.pool.outboundElementsContainer = NewWaitPool(PreallocatedBuffersPerPool, func() any {
+	device.pool.outboundElementsContainer = NewWaitPool(device.config.preallocatedBuffersPerPool, func() any {
 		s := make([]*QueueOutboundElement, 0, device.BatchSize())
 		return &QueueOutboundElementsContainer{elems: s}
 	})
-	device.pool.messageBuffers = NewWaitPool(PreallocatedBuffersPerPool, func() any {
-		return new([MaxMessageSize]byte)
-	})
-	device.pool.inboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any {
+	device.pool.inboundElements = NewWaitPool(device.config.preallocatedBuffersPerPool, func() any {
 		return new(QueueInboundElement)
 	})
-	device.pool.outboundElements = NewWaitPool(PreallocatedBuffersPerPool, func() any {
+	device.pool.outboundElements = NewWaitPool(device.config.preallocatedBuffersPerPool, func() any {
 		return new(QueueOutboundElement)
 	})
+	packetBufSize := singlePacketSlabSize
+	if device.BatchSize() > 1 {
+		packetBufSize = batchingSlabSize
+	}
+	device.pool.packetBufs = NewWaitPool(device.config.preallocatedBuffersPerPool, func() any {
+		return newPacketBuf(packetBufSize, func(buf *packetBuf) {
+			device.pool.packetBufs.Put(buf)
+		})
+	})
+	// A distinct smallPacketBufs pool makes TUN readers copy packets that fit
+	// into [smallPacketBufSize] in the interest of memory savings. Alias
+	// smallPacketBufs to the "full-sized" pool if there is little to be gained
+	// in memory savings, or if the device is configured to limit outstanding
+	// packet memory pool items, which we don't want to artificially inflate.
+	device.pool.smallPacketBufs = device.pool.packetBufs
+	if packetBufSize >= minPacketBufSizeForDistinctSmallPool && device.config.preallocatedBuffersPerPool == 0 {
+		device.pool.smallPacketBufs = NewWaitPool(device.config.preallocatedBuffersPerPool, func() any {
+			return newPacketBuf(smallPacketBufSize, func(buf *packetBuf) {
+				device.pool.smallPacketBufs.Put(buf)
+			})
+		})
+	}
 }
 
 func (device *Device) GetInboundElementsContainer() *QueueInboundElementsContainer {
 	c := device.pool.inboundElementsContainer.Get().(*QueueInboundElementsContainer)
-	c.Mutex = sync.Mutex{}
 	return c
 }
 
@@ -82,7 +132,6 @@ func (device *Device) PutInboundElementsContainer(c *QueueInboundElementsContain
 
 func (device *Device) GetOutboundElementsContainer() *QueueOutboundElementsContainer {
 	c := device.pool.outboundElementsContainer.Get().(*QueueOutboundElementsContainer)
-	c.Mutex = sync.Mutex{}
 	return c
 }
 
@@ -94,12 +143,25 @@ func (device *Device) PutOutboundElementsContainer(c *QueueOutboundElementsConta
 	device.pool.outboundElementsContainer.Put(c)
 }
 
-func (device *Device) GetMessageBuffer() *[MaxMessageSize]byte {
-	return device.pool.messageBuffers.Get().(*[MaxMessageSize]byte)
+// hasDistinctSmallPacketBufPool returns true if [Device.getSmallPacketBuf]
+// will fetch from a pool independent of [Device.getPacketBuf].
+func (device *Device) hasDistinctSmallPacketBufPool() bool {
+	return device.pool.smallPacketBufs != device.pool.packetBufs
 }
 
-func (device *Device) PutMessageBuffer(msg *[MaxMessageSize]byte) {
-	device.pool.messageBuffers.Put(msg)
+// getSmallPacketBuf returns a [*packetBuf] smaller or equal in size to what
+// [Device.getPacketBuf] returns depending on [Device] batching and [WaitPool]
+// configuration.
+func (device *Device) getSmallPacketBuf() *packetBuf {
+	b := device.pool.smallPacketBufs.Get().(*packetBuf)
+	b.incRef()
+	return b
+}
+
+func (device *Device) getPacketBuf() *packetBuf {
+	b := device.pool.packetBufs.Get().(*packetBuf)
+	b.incRef()
+	return b
 }
 
 func (device *Device) GetInboundElement() *QueueInboundElement {
@@ -107,15 +169,25 @@ func (device *Device) GetInboundElement() *QueueInboundElement {
 }
 
 func (device *Device) PutInboundElement(elem *QueueInboundElement) {
+	elem.buffer.decRef()
 	elem.clearPointers()
 	device.pool.inboundElements.Put(elem)
 }
 
+// GetOutboundElement returns a [*QueueOutboundElement] with all its fields
+// set to their respective zero values.
 func (device *Device) GetOutboundElement() *QueueOutboundElement {
-	return device.pool.outboundElements.Get().(*QueueOutboundElement)
+	elem := device.pool.outboundElements.Get().(*QueueOutboundElement)
+	elem.plaintextOffset = 0
+	elem.nonce = 0
+	// buffer, packet, keypair, and peer were cleared (if necessary) by [QueueOutboundElement.clearPointers].
+	return elem
 }
 
 func (device *Device) PutOutboundElement(elem *QueueOutboundElement) {
+	if elem.buffer != nil {
+		elem.buffer.decRef()
+	}
 	elem.clearPointers()
 	device.pool.outboundElements.Put(elem)
 }

vendor/github.com/tailscale/wireguard-go/device/queueconstants_android.go

@@ -10,10 +10,10 @@ import "github.com/tailscale/wireguard-go/conn"
 /* Reduce memory consumption for Android */
 
 const (
-	QueueStagedSize            = conn.IdealBatchSize
-	QueueOutboundSize          = 1024
-	QueueInboundSize           = 1024
-	QueueHandshakeSize         = 1024
-	MaxSegmentSize             = 2200
-	PreallocatedBuffersPerPool = 4096
+	DefaultQueueStagedSize            = conn.IdealBatchSize
+	DefaultQueueOutboundSize          = 1024
+	DefaultQueueInboundSize           = 1024
+	DefaultQueueHandshakeSize         = 1024
+	MaxSegmentSize                    = 2200
+	DefaultPreallocatedBuffersPerPool = 4096
 )

vendor/github.com/tailscale/wireguard-go/device/queueconstants_default.go

@@ -10,10 +10,10 @@ package device
 import "github.com/tailscale/wireguard-go/conn"
 
 const (
-	QueueStagedSize            = conn.IdealBatchSize
-	QueueOutboundSize          = 1024
-	QueueInboundSize           = 1024
-	QueueHandshakeSize         = 1024
-	MaxSegmentSize             = (1 << 16) - 1 // largest possible UDP datagram
-	PreallocatedBuffersPerPool = 0             // Disable and allow for infinite memory growth
+	DefaultQueueStagedSize            = conn.IdealBatchSize
+	DefaultQueueOutboundSize          = 1024
+	DefaultQueueInboundSize           = 1024
+	DefaultQueueHandshakeSize         = 1024
+	MaxSegmentSize                    = (1 << 16) - 1 // largest possible UDP datagram
+	DefaultPreallocatedBuffersPerPool = 0             // Disable and allow for infinite memory growth
 )

vendor/github.com/tailscale/wireguard-go/device/queueconstants_ios.go

@@ -7,15 +7,15 @@
 
 package device
 
-// Fit within memory limits for iOS's Network Extension API, which has stricter requirements.
-// These are vars instead of consts, because heavier network extensions might want to reduce
-// them further.
-var (
-	QueueStagedSize                   = 128
-	QueueOutboundSize                 = 1024
-	QueueInboundSize                  = 1024
-	QueueHandshakeSize                = 1024
-	PreallocatedBuffersPerPool uint32 = 1024
+// Fit within memory limits for iOS's Network Extension API, which has stricter
+// requirements. Heavier network extensions can reduce these further using
+// [Device.Option]'s.
+const (
+	DefaultQueueStagedSize            = 128
+	DefaultQueueOutboundSize          = 1024
+	DefaultQueueInboundSize           = 1024
+	DefaultQueueHandshakeSize         = 1024
+	DefaultPreallocatedBuffersPerPool = 1024
 )
 
 const MaxSegmentSize = 1700

vendor/github.com/tailscale/wireguard-go/device/queueconstants_windows.go

@@ -6,10 +6,10 @@
 package device
 
 const (
-	QueueStagedSize            = 128
-	QueueOutboundSize          = 1024
-	QueueInboundSize           = 1024
-	QueueHandshakeSize         = 1024
-	MaxSegmentSize             = 2048 - 32 // largest possible UDP datagram
-	PreallocatedBuffersPerPool = 0         // Disable and allow for infinite memory growth
+	DefaultQueueStagedSize            = 128
+	DefaultQueueOutboundSize          = 1024
+	DefaultQueueInboundSize           = 1024
+	DefaultQueueHandshakeSize         = 1024
+	MaxSegmentSize                    = 2048 - 32 // largest possible UDP datagram
+	DefaultPreallocatedBuffersPerPool = 0         // Disable and allow for infinite memory growth
 )

vendor/github.com/tailscale/wireguard-go/device/receive.go

@@ -8,7 +8,9 @@ package device
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"net"
+	"net/netip"
 	"sync"
 	"time"
 
@@ -19,23 +21,28 @@ import (
 )
 
 type QueueHandshakeElement struct {
-	msgType  uint32
-	packet   []byte
-	endpoint conn.Endpoint
-	buffer   *[MaxMessageSize]byte
+	msgType    uint32
+	packet     []byte
+	packetMeta conn.ReceivedPacket
+	buffer     *packetBuf
 }
 
 type QueueInboundElement struct {
-	buffer   *[MaxMessageSize]byte
-	packet   []byte
-	counter  uint64
-	keypair  *Keypair
-	endpoint conn.Endpoint
+	buffer     *packetBuf
+	packet     []byte
+	packetMeta conn.ReceivedPacket
+	counter    uint64
+	keypair    *Keypair
 }
 
 type QueueInboundElementsContainer struct {
-	sync.Mutex
-	elems []*QueueInboundElement
+	// filling is a one-shot barrier signaling decryption→receive
+	// handoff. RoutineReceiveIncoming calls Add(1) before sending the
+	// container down the decryption and inbound queues; RoutineDecryption
+	// calls Done after decrypting; RoutineSequentialReceiver calls Wait
+	// before reading the decrypted packets.
+	filling sync.WaitGroup
+	elems   []*QueueInboundElement
 }
 
 // clearPointers clears elem fields that contain pointers.
@@ -45,8 +52,8 @@ type QueueInboundElementsContainer struct {
 func (elem *QueueInboundElement) clearPointers() {
 	elem.buffer = nil
 	elem.packet = nil
+	elem.packetMeta = conn.ReceivedPacket{}
 	elem.keypair = nil
-	elem.endpoint = nil
 }
 
 /* Called when a new authenticated message has been received
@@ -83,31 +90,29 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive
 	// receive datagrams until conn is closed
 
 	var (
-		bufsArrs    = make([]*[MaxMessageSize]byte, maxBatchSize)
-		bufs        = make([][]byte, maxBatchSize)
+		packets = make([]conn.ReceivedPacket, maxBatchSize)
+		buf     = device.getPacketBuf()
+		// bufDirty tracks whether the current buf has been pushed downstream to
+		// crypto and per-peer/handshake functions, or can be re-used across the
+		// next read cycle
+		bufDirty    bool
 		err         error
-		sizes       = make([]int, maxBatchSize)
 		count       int
-		endpoints   = make([]conn.Endpoint, maxBatchSize)
 		deathSpiral int
 		elemsByPeer = make(map[*Peer]*QueueInboundElementsContainer, maxBatchSize)
 	)
 
-	for i := range bufsArrs {
-		bufsArrs[i] = device.GetMessageBuffer()
-		bufs[i] = bufsArrs[i][:]
-	}
-
 	defer func() {
-		for i := 0; i < maxBatchSize; i++ {
-			if bufsArrs[i] != nil {
-				device.PutMessageBuffer(bufsArrs[i])
-			}
-		}
+		buf.decRef()
 	}()
 
 	for {
-		count, err = recv(bufs, sizes, endpoints)
+		if bufDirty {
+			buf.decRef()
+			buf = device.getPacketBuf()
+			bufDirty = false
+		}
+		count, err = recv(buf.slab, packets)
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
 				return
@@ -126,14 +131,14 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive
 		deathSpiral = 0
 
 		// handle each packet in the batch
-		for i, size := range sizes[:count] {
-			if size < MinMessageSize {
+		for _, meta := range packets[:count] {
+			if meta.Size < MinMessageSize {
 				continue
 			}
 
 			// check size of packet
 
-			packet := bufsArrs[i][:size]
+			packet := meta.Bytes(buf.slab)
 			msgType := binary.LittleEndian.Uint32(packet[:4])
 
 			switch msgType {
@@ -168,21 +173,20 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive
 				// create work element
 				peer := value.peer
 				elem := device.GetInboundElement()
+				bufDirty = true
+				buf.incRef()
+				elem.buffer = buf
 				elem.packet = packet
-				elem.buffer = bufsArrs[i]
+				elem.packetMeta = meta
 				elem.keypair = keypair
-				elem.endpoint = endpoints[i]
 				elem.counter = 0
 
 				elemsForPeer, ok := elemsByPeer[peer]
 				if !ok {
 					elemsForPeer = device.GetInboundElementsContainer()
-					elemsForPeer.Lock()
 					elemsByPeer[peer] = elemsForPeer
 				}
 				elemsForPeer.elems = append(elemsForPeer.elems, elem)
-				bufsArrs[i] = device.GetMessageBuffer()
-				bufs[i] = bufsArrs[i][:]
 				continue
 
 			// otherwise it is a fixed size & handshake related packet
@@ -207,29 +211,21 @@ func (device *Device) RoutineReceiveIncoming(maxBatchSize int, recv conn.Receive
 				continue
 			}
 
+			buf.incRef()
 			select {
 			case device.queue.handshake.c <- QueueHandshakeElement{
-				msgType:  msgType,
-				buffer:   bufsArrs[i],
-				packet:   packet,
-				endpoint: endpoints[i],
+				msgType:    msgType,
+				buffer:     buf,
+				packet:     packet,
+				packetMeta: meta,
 			}:
-				bufsArrs[i] = device.GetMessageBuffer()
-				bufs[i] = bufsArrs[i][:]
+				bufDirty = true
 			default:
+				buf.decRef()
 			}
 		}
 		for peer, elemsContainer := range elemsByPeer {
-			if peer.isRunning.Load() {
-				peer.queue.inbound.c <- elemsContainer
-				device.queue.decryption.c <- elemsContainer
-			} else {
-				for _, elem := range elemsContainer.elems {
-					device.PutMessageBuffer(elem.buffer)
-					device.PutInboundElement(elem)
-				}
-				device.PutInboundElementsContainer(elemsContainer)
-			}
+			peer.queueInboundIfRunning(elemsContainer)
 			delete(elemsByPeer, peer)
 		}
 	}
@@ -262,7 +258,7 @@ func (device *Device) RoutineDecryption(id int) {
 				elem.packet = nil
 			}
 		}
-		elemsContainer.Unlock()
+		elemsContainer.filling.Done()
 	}
 }
 
@@ -302,8 +298,8 @@ func (device *Device) RoutineHandshake(id int) {
 
 			// consume reply
 
-			if peer := entry.peer; peer.isRunning.Load() {
-				device.log.Verbosef("Receiving cookie response from %s", elem.endpoint.DstToString())
+			if peer := entry.peer; peer.runningState.isRunning.Load() {
+				device.log.Verbosef("Receiving cookie response from %s", elem.packetMeta.Endpoint.DstToString())
 				if !peer.cookieGenerator.ConsumeReply(&reply) {
 					device.log.Verbosef("Could not decrypt invalid cookie response")
 				}
@@ -326,14 +322,14 @@ func (device *Device) RoutineHandshake(id int) {
 
 				// verify MAC2 field
 
-				if !device.cookieChecker.CheckMAC2(elem.packet, elem.endpoint.DstToBytes()) {
+				if !device.cookieChecker.CheckMAC2(elem.packet, elem.packetMeta.Endpoint.DstToBytes()) {
 					device.SendHandshakeCookie(&elem)
 					goto skip
 				}
 
 				// check ratelimiter
 
-				if !device.rate.limiter.Allow(elem.endpoint.DstIP()) {
+				if !device.rate.limiter.Allow(elem.packetMeta.Endpoint.DstIP()) {
 					goto skip
 				}
 			}
@@ -359,9 +355,9 @@ func (device *Device) RoutineHandshake(id int) {
 
 			// consume initiation
 
-			peer := device.ConsumeMessageInitiation(&msg, elem.endpoint)
+			peer := device.ConsumeMessageInitiation(&msg, elem.packetMeta.Endpoint)
 			if peer == nil {
-				device.log.Verbosef("Received invalid initiation message from %s", elem.endpoint.DstToString())
+				device.log.Verbosef("Received invalid initiation message from %s", elem.packetMeta.Endpoint.DstToString())
 				goto skip
 			}
 
@@ -370,9 +366,6 @@ func (device *Device) RoutineHandshake(id int) {
 			peer.timersAnyAuthenticatedPacketTraversal()
 			peer.timersAnyAuthenticatedPacketReceived()
 
-			// update endpoint
-			peer.SetEndpointFromPacket(elem.endpoint)
-
 			device.log.Verbosef("%v - Received handshake initiation", peer)
 			peer.rxBytes.Add(uint64(len(elem.packet)))
 
@@ -393,13 +386,10 @@ func (device *Device) RoutineHandshake(id int) {
 
 			peer := device.ConsumeMessageResponse(&msg)
 			if peer == nil {
-				device.log.Verbosef("Received invalid response message from %s", elem.endpoint.DstToString())
+				device.log.Verbosef("Received invalid response message from %s", elem.packetMeta.Endpoint.DstToString())
 				goto skip
 			}
 
-			// update endpoint
-			peer.SetEndpointFromPacket(elem.endpoint)
-
 			device.log.Verbosef("%v - Received handshake response", peer)
 			peer.rxBytes.Add(uint64(len(elem.packet)))
 
@@ -417,12 +407,14 @@ func (device *Device) RoutineHandshake(id int) {
 				goto skip
 			}
 
+			device.config.metrics.HandshakeInitiatorCompleted.Add(1)
 			peer.timersSessionDerived()
 			peer.timersHandshakeComplete()
+			peer.SendPriorityMessage()
 			peer.SendKeepalive()
 		}
 	skip:
-		device.PutMessageBuffer(elem.buffer)
+		elem.buffer.decRef()
 	}
 }
 
@@ -430,110 +422,136 @@ func (peer *Peer) RoutineSequentialReceiver(maxBatchSize int) {
 	device := peer.device
 	defer func() {
 		device.log.Verbosef("%v - Routine: sequential receiver - stopped", peer)
-		peer.stopping.Done()
+		peer.runningState.queueReaders.Done()
 	}()
 	device.log.Verbosef("%v - Routine: sequential receiver - started", peer)
 
 	bufs := make([][]byte, 0, maxBatchSize)
 
-	for elemsContainer := range peer.queue.inbound.c {
-		if elemsContainer == nil {
-			return
+	for elemsContainer := range peer.queue.inbound {
+		peer.processInboundContainer(elemsContainer, bufs[:0])
+	}
+}
+
+// processInboundContainer waits for the decryption routine to finish
+// filling elemsContainer, then writes the valid packets to the TUN
+// device and returns the container to the pool.
+//
+// scratch is a length-0 slice used to assemble the per-packet buffers
+// passed to tun.device.Write; its backing array is reused across calls.
+func (peer *Peer) processInboundContainer(elemsContainer *QueueInboundElementsContainer, scratch [][]byte) {
+	// Invariants from RoutineSequentialReceiver; all should be unreachable.
+	if len(scratch) != 0 || cap(scratch) == 0 {
+		panic(fmt.Sprintf("processInboundContainer: scratch must be empty with non-zero cap; got len=%d cap=%d",
+			len(scratch), cap(scratch)))
+	}
+	if cap(scratch) < len(elemsContainer.elems) {
+		panic(fmt.Sprintf("processInboundContainer: scratch cap %d < elems %d",
+			cap(scratch), len(elemsContainer.elems)))
+	}
+
+	device := peer.device
+	defer device.PutInboundElementsContainer(elemsContainer)
+
+	// Wait for RoutineDecryption to finish filling the container. After
+	// Wait returns we have happens-before with that goroutine and are the
+	// sole owner of the container until Put hands it back to the pool.
+	elemsContainer.filling.Wait()
+	elems := elemsContainer.elems
+
+	validTailPacket := -1
+	dataPacketReceived := false
+	rxBytesLen := uint64(0)
+	for i, elem := range elems {
+		if elem.packet == nil {
+			// decryption failed
+			continue
 		}
-		elemsContainer.Lock()
-		validTailPacket := -1
-		dataPacketReceived := false
-		rxBytesLen := uint64(0)
-		for i, elem := range elemsContainer.elems {
-			if elem.packet == nil {
-				// decryption failed
+
+		if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) {
+			device.config.metrics.MessageTransportRXDroppedReplay.Add(1)
+			continue
+		}
+
+		validTailPacket = i
+		if peer.ReceivedWithKeypair(elem.keypair) {
+			peer.device.config.metrics.HandshakeResponderCompleted.Add(1)
+			peer.timersHandshakeComplete()
+			peer.SendPriorityMessage()
+			peer.SendStagedPackets()
+		}
+		if ep, ok := elem.packetMeta.Endpoint.(conn.PeerAwareEndpoint); ok {
+			ep.FromPeer(peer.handshake.remoteStatic)
+		}
+		rxBytesLen += uint64(len(elem.packet) + MinMessageSize)
+
+		if len(elem.packet) == 0 {
+			device.log.Verbosef("%v - Receiving keepalive packet", peer)
+			continue
+		}
+		dataPacketReceived = true
+
+		switch elem.packet[0] >> 4 {
+		case 4:
+			if len(elem.packet) < ipv4.HeaderLen {
+				continue
+			}
+			field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2]
+			length := binary.BigEndian.Uint16(field)
+			if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen {
 				continue
 			}
-
-			if !elem.keypair.replayFilter.ValidateCounter(elem.counter, RejectAfterMessages) {
+			elem.packet = elem.packet[:length]
+			src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]
+			srcAddr, _ := netip.AddrFromSlice(src)
+			if !peer.AllowedPeerSourceIP(srcAddr) {
+				device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer)
 				continue
 			}
 
-			validTailPacket = i
-			if peer.ReceivedWithKeypair(elem.keypair) {
-				peer.SetEndpointFromPacket(elem.endpoint)
-				peer.timersHandshakeComplete()
-				peer.SendStagedPackets()
-			}
-			if ep, ok := elem.endpoint.(conn.PeerAwareEndpoint); ok {
-				ep.FromPeer(peer.handshake.remoteStatic)
+		case 6:
+			if len(elem.packet) < ipv6.HeaderLen {
+				continue
 			}
-			rxBytesLen += uint64(len(elem.packet) + MinMessageSize)
-
-			if len(elem.packet) == 0 {
-				device.log.Verbosef("%v - Receiving keepalive packet", peer)
+			field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2]
+			length := binary.BigEndian.Uint16(field)
+			length += ipv6.HeaderLen
+			if int(length) > len(elem.packet) {
 				continue
 			}
-			dataPacketReceived = true
-
-			switch elem.packet[0] >> 4 {
-			case 4:
-				if len(elem.packet) < ipv4.HeaderLen {
-					continue
-				}
-				field := elem.packet[IPv4offsetTotalLength : IPv4offsetTotalLength+2]
-				length := binary.BigEndian.Uint16(field)
-				if int(length) > len(elem.packet) || int(length) < ipv4.HeaderLen {
-					continue
-				}
-				elem.packet = elem.packet[:length]
-				src := elem.packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]
-				if device.allowedips.Lookup(src) != peer {
-					device.log.Verbosef("IPv4 packet with disallowed source address from %v", peer)
-					continue
-				}
-
-			case 6:
-				if len(elem.packet) < ipv6.HeaderLen {
-					continue
-				}
-				field := elem.packet[IPv6offsetPayloadLength : IPv6offsetPayloadLength+2]
-				length := binary.BigEndian.Uint16(field)
-				length += ipv6.HeaderLen
-				if int(length) > len(elem.packet) {
-					continue
-				}
-				elem.packet = elem.packet[:length]
-				src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]
-				if device.allowedips.Lookup(src) != peer {
-					device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer)
-					continue
-				}
-
-			default:
-				device.log.Verbosef("Packet with invalid IP version from %v", peer)
+			elem.packet = elem.packet[:length]
+			src := elem.packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]
+			srcAddr, _ := netip.AddrFromSlice(src)
+			if !peer.AllowedPeerSourceIP(srcAddr) {
+				device.log.Verbosef("IPv6 packet with disallowed source address from %v", peer)
 				continue
 			}
 
-			bufs = append(bufs, elem.buffer[:MessageTransportOffsetContent+len(elem.packet)])
+		default:
+			device.log.Verbosef("Packet with invalid IP version from %v", peer)
+			continue
 		}
+
+		elem.packetMeta.Size = MessageTransportOffsetContent + len(elem.packet)
+		scratch = append(scratch, elem.packetMeta.Bytes(elem.buffer.slab))
+	}
 
-		peer.rxBytes.Add(rxBytesLen)
-		if validTailPacket >= 0 {
-			peer.SetEndpointFromPacket(elemsContainer.elems[validTailPacket].endpoint)
-			peer.keepKeyFreshReceiving()
-			peer.timersAnyAuthenticatedPacketTraversal()
-			peer.timersAnyAuthenticatedPacketReceived()
-		}
-		if dataPacketReceived {
-			peer.timersDataReceived()
-		}
-		if len(bufs) > 0 {
-			_, err := device.tun.device.Write(bufs, MessageTransportOffsetContent)
-			if err != nil && !device.isClosed() {
-				device.log.Errorf("Failed to write packets to TUN device: %v", err)
-			}
-		}
-		for _, elem := range elemsContainer.elems {
-			device.PutMessageBuffer(elem.buffer)
-			device.PutInboundElement(elem)
+	peer.rxBytes.Add(rxBytesLen)
+	if validTailPacket >= 0 {
+		peer.keepKeyFreshReceiving()
+		peer.timersAnyAuthenticatedPacketTraversal()
+		peer.timersAnyAuthenticatedPacketReceived()
+	}
+	if dataPacketReceived {
+		peer.timersDataReceived()
+	}
+	if len(scratch) > 0 {
+		_, err := peer.writeTUN(scratch, MessageTransportOffsetContent)
+		if err != nil && !device.isClosed() {
+			device.log.Errorf("Failed to write packets to TUN device: %v", err)
 		}
-		bufs = bufs[:0]
-		device.PutInboundElementsContainer(elemsContainer)
+	}
+	for _, elem := range elems {
+		device.PutInboundElement(elem)
 	}
 }

vendor/github.com/tailscale/wireguard-go/device/send.go

@@ -8,7 +8,9 @@ package device
 import (
 	"encoding/binary"
 	"errors"
+	"fmt"
 	"net"
+	"net/netip"
 	"os"
 	"sync"
 	"time"
@@ -45,28 +47,26 @@ import (
  */
 
 type QueueOutboundElement struct {
-	buffer *[MaxMessageSize]byte // slice holding the packet data
-	// packet is always a slice of "buffer". The starting offset in buffer
-	// is either:
-	//  a) MessageEncapsulatingTransportSize+MessageTransportHeaderSize (plaintext)
-	//  b) 0 (post-encryption)
-	packet  []byte
-	nonce   uint64   // nonce for encryption
-	keypair *Keypair // keypair for encryption
-	peer    *Peer    // related peer
+	buffer *packetBuf // shared packet buffer that contains packet
+	packet []byte     // packet is always a slice of buffer.slab
+	// plaintextOffset describes the pre-encryption offset of packet within
+	// buffer.slab. The encryption process grows packet on both ends to make
+	// room for any potential encapsulating transport, WireGuard header, and
+	// tail padding.
+	plaintextOffset int
+	nonce           uint64   // nonce for encryption
+	keypair         *Keypair // keypair for encryption
+	peer            *Peer    // related peer
 }
 
 type QueueOutboundElementsContainer struct {
-	sync.Mutex
-	elems []*QueueOutboundElement
-}
-
-func (device *Device) NewOutboundElement() *QueueOutboundElement {
-	elem := device.GetOutboundElement()
-	elem.buffer = device.GetMessageBuffer()
-	elem.nonce = 0
-	// keypair and peer were cleared (if necessary) by clearPointers.
-	return elem
+	// filling is a one-shot barrier signaling encryption→send handoff.
+	// SendStagedPackets calls Add(1) before sending the container down
+	// the encryption and outbound queues; RoutineEncryption calls Done
+	// after encrypting; RoutineSequentialSender calls Wait before
+	// reading the encrypted packets.
+	filling sync.WaitGroup
+	elems   []*QueueOutboundElement
 }
 
 // clearPointers clears elem fields that contain pointers.
@@ -80,18 +80,24 @@ func (elem *QueueOutboundElement) clearPointers() {
 	elem.peer = nil
 }
 
-/* Queues a keepalive if no packets are queued for peer
- */
+// SendKeepalive queues a keepalive if no packets are queued for
+// peer.
 func (peer *Peer) SendKeepalive() {
-	if len(peer.queue.staged) == 0 && peer.isRunning.Load() {
-		elem := peer.device.NewOutboundElement()
+	if len(peer.queue.staged) == 0 {
+		elem := peer.device.GetOutboundElement()
+		elem.buffer = peer.device.getSmallPacketBuf()
+		elem.plaintextOffset = MessageEncapsulatingTransportSize + MessageTransportHeaderSize
 		elemsContainer := peer.device.GetOutboundElementsContainer()
 		elemsContainer.elems = append(elemsContainer.elems, elem)
-		select {
-		case peer.queue.staged <- elemsContainer:
-			peer.device.log.Verbosef("%v - Sending keepalive packet", peer)
-		default:
-			peer.device.PutMessageBuffer(elem.buffer)
+		if queued := peer.doIfRunning(func() {
+			select {
+			case peer.queue.staged <- elemsContainer:
+				peer.device.log.Verbosef("%v - Sending keepalive packet", peer)
+			default:
+				peer.device.PutOutboundElement(elem)
+				peer.device.PutOutboundElementsContainer(elemsContainer)
+			}
+		}); !queued {
 			peer.device.PutOutboundElement(elem)
 			peer.device.PutOutboundElementsContainer(elemsContainer)
 		}
@@ -99,6 +105,62 @@ func (peer *Peer) SendKeepalive() {
 	peer.SendStagedPackets()
 }
 
+// SendPriorityMessage invokes the [PeerPriorityMessageFunc] callback if one is
+// set, and queues the returned message for encryption and transmission if the
+// current keypair is valid.
+func (peer *Peer) SendPriorityMessage() {
+	f := peer.device.priorityMsgFn.Load()
+	if f == nil {
+		return
+	}
+	keypair := peer.keypairs.Current()
+	if keypair == nil || keypair.sendNonce.Load() >= RejectAfterMessages || time.Since(keypair.created) >= RejectAfterTime {
+		// SendStagedPackets initializes a handshake when the keypair is invalid,
+		// but we explicitly avoid that here. A priority message is only intended
+		// to flow around symmetric session establishment, but it should never
+		// trigger a new session. Reaching this branch due to nonce exhaustion
+		// or keypair expiration is highly unlikely considering where
+		// SendPriorityMessage is called (at current keypair establishment).
+		return
+	}
+
+	// get plaintext message to send
+	msg := (*f)(peer.handshake.remoteStatic)
+	if len(msg) == 0 {
+		return
+	}
+	if len(msg) > MaxPriorityMessageContentSize {
+		peer.device.log.Verbosef("%v - Failed to queue priority message due to size", peer)
+		return
+	}
+
+	// get pooled elements
+	elem := peer.device.GetOutboundElement()
+	elemsContainer := peer.device.GetOutboundElementsContainer()
+	elemsContainer.elems = append(elemsContainer.elems, elem)
+	buf := peer.device.getSmallPacketBuf()
+
+	// initialize outbound element
+	const offset = MessageEncapsulatingTransportSize + MessageTransportHeaderSize
+
+	n := copy(buf.slab[offset:], msg)
+	elem.buffer = buf
+	elem.packet = buf.slab[offset : offset+n]
+	elem.plaintextOffset = offset
+	elem.peer = peer
+	elem.nonce = keypair.sendNonce.Add(1) - 1
+	if elem.nonce >= RejectAfterMessages {
+		keypair.sendNonce.Store(RejectAfterMessages)
+		peer.device.PutOutboundElement(elem)
+		peer.device.PutOutboundElementsContainer(elemsContainer)
+		return
+	}
+	elem.keypair = keypair
+
+	// add to parallel and sequential queue
+	peer.queueOutboundIfRunning(elemsContainer)
+}
+
 func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 	if !isRetry {
 		peer.timers.handshakeAttempts.Store(0)
@@ -116,6 +178,10 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 		peer.handshake.mutex.Unlock()
 		return nil
 	}
+	// handshakeOnUserSend is flipped false after lastSentHandshake checks,
+	// enabling eventual transmission at a future call of this method, while
+	// still honoring [RekeyTimeout].
+	peer.handshakeOnUserSend.Store(false)
 	peer.handshake.lastSentHandshake = time.Now()
 	peer.handshake.mutex.Unlock()
 
@@ -135,6 +201,11 @@ func (peer *Peer) SendHandshakeInitiation(isRetry bool) error {
 	peer.timersAnyAuthenticatedPacketTraversal()
 	peer.timersAnyAuthenticatedPacketSent()
 
+	if isRetry {
+		peer.device.config.metrics.MessageInitiationTXAttemptRetry.Add(1)
+	} else {
+		peer.device.config.metrics.MessageInitiationTXAttemptInitial.Add(1)
+	}
 	err = peer.SendBuffers([][]byte{buf})
 	if err != nil {
 		peer.device.log.Errorf("%v - Failed to send handshake initiation: %v", peer, err)
@@ -172,6 +243,7 @@ func (peer *Peer) SendHandshakeResponse() error {
 	peer.timersAnyAuthenticatedPacketTraversal()
 	peer.timersAnyAuthenticatedPacketSent()
 
+	peer.device.config.metrics.MessageResponseTXAttempt.Add(1)
 	// TODO: allocation could be avoided
 	err = peer.SendBuffers([][]byte{buf})
 	if err != nil {
@@ -181,10 +253,10 @@ func (peer *Peer) SendHandshakeResponse() error {
 }
 
 func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement) error {
-	device.log.Verbosef("Sending cookie response for denied handshake message for %v", initiatingElem.endpoint.DstToString())
+	device.log.Verbosef("Sending cookie response for denied handshake message for %v", initiatingElem.packetMeta.Endpoint.DstToString())
 
 	sender := binary.LittleEndian.Uint32(initiatingElem.packet[4:8])
-	reply, err := device.cookieChecker.CreateReply(initiatingElem.packet, sender, initiatingElem.endpoint.DstToBytes())
+	reply, err := device.cookieChecker.CreateReply(initiatingElem.packet, sender, initiatingElem.packetMeta.Endpoint.DstToBytes())
 	if err != nil {
 		device.log.Errorf("Failed to create cookie reply: %v", err)
 		return err
@@ -193,8 +265,9 @@ func (device *Device) SendHandshakeCookie(initiatingElem *QueueHandshakeElement)
 	buf := make([]byte, MessageEncapsulatingTransportSize+MessageCookieReplySize)
 	packet := buf[MessageEncapsulatingTransportSize:]
 	_ = reply.marshal(packet)
+	device.config.metrics.MessageCookieReplyTXAttempt.Add(1)
 	// TODO: allocation could be avoided
-	device.net.bind.Send([][]byte{buf}, initiatingElem.endpoint, MessageEncapsulatingTransportSize)
+	device.net.bind.Send([][]byte{buf}, initiatingElem.packetMeta.Endpoint, MessageEncapsulatingTransportSize)
 
 	return nil
 }
@@ -204,73 +277,132 @@ func (peer *Peer) keepKeyFreshSending() {
 	if keypair == nil {
 		return
 	}
+	txHandshake := peer.handshakeOnUserSend.Load()
 	nonce := keypair.sendNonce.Load()
-	if nonce > RekeyAfterMessages || (keypair.isInitiator && time.Since(keypair.created) > RekeyAfterTime) {
+	if txHandshake || nonce > RekeyAfterMessages || (keypair.isInitiator && time.Since(keypair.created) > RekeyAfterTime) {
 		peer.SendHandshakeInitiation(false)
 	}
 }
 
-func (device *Device) RoutineReadFromTUN() {
+// copyPacketBufIfFits copies the prefix of src.slab ending at the final packet
+// into dst.slab if that prefix+[outboundPlaintextTailroom] fits. It reports
+// whether the copy occurred. Packet descriptors must be in offset ascending order.
+func copyPacketBufIfFits(dst, src *packetBuf, packets []tun.ReadPacket) bool {
+	if dst == nil || len(packets) < 1 {
+		return false
+	}
+	// validate tail descriptor as it relates to src
+	tailPkt := packets[len(packets)-1]
+	if tailPkt.Offset < 0 || tailPkt.Size < 1 {
+		return false
+	}
+	end := tailPkt.Offset + tailPkt.Size
+	if end < tailPkt.Offset || end > len(src.slab) ||
+		outboundPlaintextTailroom > len(src.slab)-end {
+		return false
+	}
+	// validate src fits in dst
+	if end+outboundPlaintextTailroom > len(dst.slab) {
+		return false
+	}
+	copy(dst.slab, src.slab[:end])
+	return true
+}
+
+func (device *Device) RoutineReadFromTUN(id int, queue tun.Reader) {
 	defer func() {
-		device.log.Verbosef("Routine: TUN reader - stopped")
+		device.log.Verbosef("Routine: TUN reader %d - stopped", id)
 		device.state.stopping.Done()
 		device.queue.encryption.wg.Done()
 	}()
 
-	device.log.Verbosef("Routine: TUN reader - started")
+	device.log.Verbosef("Routine: TUN reader %d - started", id)
 
 	var (
 		batchSize   = device.BatchSize()
+		packets     = make([]tun.ReadPacket, batchSize)
+		fullSizeBuf = device.getPacketBuf()
+		smallBuf    *packetBuf
 		readErr     error
 		elems       = make([]*QueueOutboundElement, batchSize)
-		bufs        = make([][]byte, batchSize)
 		elemsByPeer = make(map[*Peer]*QueueOutboundElementsContainer, batchSize)
 		count       = 0
-		sizes       = make([]int, batchSize)
-		offset      = MessageEncapsulatingTransportSize + MessageTransportHeaderSize
 	)
 
+	if device.hasDistinctSmallPacketBufPool() {
+		smallBuf = device.getSmallPacketBuf()
+	}
+
+	defer func() {
+		fullSizeBuf.decRef()
+		if smallBuf != nil {
+			smallBuf.decRef()
+		}
+	}()
+
 	for i := range elems {
-		elems[i] = device.NewOutboundElement()
-		bufs[i] = elems[i].buffer[:]
+		elems[i] = device.GetOutboundElement()
 	}
 
 	defer func() {
 		for _, elem := range elems {
 			if elem != nil {
-				device.PutMessageBuffer(elem.buffer)
 				device.PutOutboundElement(elem)
 			}
 		}
 	}()
 
+	var (
+		buf          *packetBuf // either fullSizeBuf or smallBuf
+		bufHandedOff bool
+	)
+
 	for {
+		if bufHandedOff {
+			if buf == smallBuf {
+				smallBuf.decRef()
+				smallBuf = device.getSmallPacketBuf()
+			} else {
+				fullSizeBuf.decRef()
+				fullSizeBuf = device.getPacketBuf()
+			}
+		}
+		buf = fullSizeBuf
+		bufHandedOff = false
+
 		// read packets
-		count, readErr = device.tun.device.Read(bufs, sizes, offset)
-		for i := 0; i < count; i++ {
-			if sizes[i] < 1 {
+		count, readErr = queue.Read(fullSizeBuf.slab, packets)
+		if copyPacketBufIfFits(smallBuf, fullSizeBuf, packets[:count]) {
+			buf = smallBuf
+		}
+
+		for i, meta := range packets[:count] {
+			if meta.Size < 1 || meta.Size > MaxContentSize {
 				continue
 			}
 
-			elem := elems[i]
-			elem.packet = bufs[i][offset : offset+sizes[i]]
+			// explicitly pin slice capacity, so downstream expansion of length
+			// never leak into an adjacent packet
+			packet := buf.slab[meta.Offset : meta.Offset+meta.Size : meta.Offset+meta.Size+outboundPlaintextTailroom]
 
 			// lookup peer
 			var peer *Peer
-			switch elem.packet[0] >> 4 {
+			switch packet[0] >> 4 {
 			case 4:
-				if len(elem.packet) < ipv4.HeaderLen {
+				if len(packet) < ipv4.HeaderLen {
 					continue
 				}
-				dst := elem.packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len]
-				peer = device.allowedips.Lookup(dst)
+				src := netip.AddrFrom4([4]byte(packet[IPv4offsetSrc : IPv4offsetSrc+net.IPv4len]))
+				dst := netip.AddrFrom4([4]byte(packet[IPv4offsetDst : IPv4offsetDst+net.IPv4len]))
+				peer = device.allowedips.LookupFromPacket(src, dst, packet)
 
 			case 6:
-				if len(elem.packet) < ipv6.HeaderLen {
+				if len(packet) < ipv6.HeaderLen {
 					continue
 				}
-				dst := elem.packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len]
-				peer = device.allowedips.Lookup(dst)
+				src := netip.AddrFrom16([16]byte(packet[IPv6offsetSrc : IPv6offsetSrc+net.IPv6len]))
+				dst := netip.AddrFrom16([16]byte(packet[IPv6offsetDst : IPv6offsetDst+net.IPv6len]))
+				peer = device.allowedips.LookupFromPacket(src, dst, packet)
 
 			default:
 				device.log.Verbosef("Received packet with unknown IP version")
@@ -279,27 +411,26 @@ func (device *Device) RoutineReadFromTUN() {
 			if peer == nil {
 				continue
 			}
+
+			elem := elems[i]
+			buf.incRef()
+			bufHandedOff = true
+			elem.buffer = buf
+			elem.packet = packet
+			elem.plaintextOffset = meta.Offset
+
 			elemsForPeer, ok := elemsByPeer[peer]
 			if !ok {
 				elemsForPeer = device.GetOutboundElementsContainer()
 				elemsByPeer[peer] = elemsForPeer
 			}
 			elemsForPeer.elems = append(elemsForPeer.elems, elem)
-			elems[i] = device.NewOutboundElement()
-			bufs[i] = elems[i].buffer[:]
+			elems[i] = device.GetOutboundElement()
 		}
 
 		for peer, elemsForPeer := range elemsByPeer {
-			if peer.isRunning.Load() {
-				peer.StagePackets(elemsForPeer)
-				peer.SendStagedPackets()
-			} else {
-				for _, elem := range elemsForPeer.elems {
-					device.PutMessageBuffer(elem.buffer)
-					device.PutOutboundElement(elem)
-				}
-				device.PutOutboundElementsContainer(elemsForPeer)
-			}
+			peer.StagePackets(elemsForPeer)
+			peer.SendStagedPackets()
 			delete(elemsByPeer, peer)
 		}
 
@@ -323,24 +454,31 @@ func (device *Device) RoutineReadFromTUN() {
 }
 
 func (peer *Peer) StagePackets(elems *QueueOutboundElementsContainer) {
-	for {
-		select {
-		case peer.queue.staged <- elems:
-			return
-		default:
-		}
-		select {
-		case tooOld := <-peer.queue.staged:
-			for _, elem := range tooOld.elems {
-				peer.device.PutMessageBuffer(elem.buffer)
-				peer.device.PutOutboundElement(elem)
+	if running := peer.doIfRunning(func() {
+		for {
+			select {
+			case peer.queue.staged <- elems:
+				return
+			default:
+			}
+			select {
+			case tooOld := <-peer.queue.staged:
+				for _, elem := range tooOld.elems {
+					peer.device.PutOutboundElement(elem)
+				}
+				peer.device.PutOutboundElementsContainer(tooOld)
+			default:
 			}
-			peer.device.PutOutboundElementsContainer(tooOld)
-		default:
 		}
+	}); !running {
+		for _, elem := range elems.elems {
+			peer.device.PutOutboundElement(elem)
+		}
+		peer.device.PutOutboundElementsContainer(elems)
 	}
 }
 
+// SendStagedPackets sends any staged packets to Peer.
 func (peer *Peer) SendStagedPackets() {
 top:
 	if len(peer.queue.staged) == 0 || !peer.device.isUp() {
@@ -375,7 +513,6 @@ top:
 
 				elem.keypair = keypair
 			}
-			elemsContainer.Lock()
 			elemsContainer.elems = elemsContainer.elems[:i]
 
 			if elemsContainerOOO != nil {
@@ -387,17 +524,7 @@ top:
 				goto top
 			}
 
-			// add to parallel and sequential queue
-			if peer.isRunning.Load() {
-				peer.queue.outbound.c <- elemsContainer
-				peer.device.queue.encryption.c <- elemsContainer
-			} else {
-				for _, elem := range elemsContainer.elems {
-					peer.device.PutMessageBuffer(elem.buffer)
-					peer.device.PutOutboundElement(elem)
-				}
-				peer.device.PutOutboundElementsContainer(elemsContainer)
-			}
+			peer.queueOutboundIfRunning(elemsContainer)
 
 			if elemsContainerOOO != nil {
 				goto top
@@ -413,7 +540,6 @@ func (peer *Peer) FlushStagedPackets() {
 		select {
 		case elemsContainer := <-peer.queue.staged:
 			for _, elem := range elemsContainer.elems {
-				peer.device.PutMessageBuffer(elem.buffer)
 				peer.device.PutOutboundElement(elem)
 			}
 			peer.device.PutOutboundElementsContainer(elemsContainer)
@@ -453,7 +579,7 @@ func (device *Device) RoutineEncryption(id int) {
 	for elemsContainer := range device.queue.encryption.c {
 		for _, elem := range elemsContainer.elems {
 			// populate header fields
-			header := elem.buffer[MessageEncapsulatingTransportSize : MessageEncapsulatingTransportSize+MessageTransportHeaderSize]
+			header := elem.buffer.slab[elem.plaintextOffset-MessageTransportHeaderSize : elem.plaintextOffset]
 
 			fieldType := header[0:4]
 			fieldReceiver := header[4:8]
@@ -465,6 +591,7 @@ func (device *Device) RoutineEncryption(id int) {
 
 			// pad content to multiple of 16
 			paddingSize := calculatePaddingSize(len(elem.packet), int(device.tun.mtu.Load()))
+
 			elem.packet = append(elem.packet, paddingZeros[:paddingSize]...)
 
 			// encrypt content and release to consumer
@@ -478,9 +605,11 @@ func (device *Device) RoutineEncryption(id int) {
 			)
 
 			// re-slice packet to include encapsulating transport space
-			elem.packet = elem.buffer[:MessageEncapsulatingTransportSize+len(elem.packet)]
+			start := elem.plaintextOffset - MessageTransportHeaderSize - MessageEncapsulatingTransportSize
+			end := elem.plaintextOffset - MessageTransportHeaderSize + len(elem.packet)
+			elem.packet = elem.buffer.slab[start:end]
 		}
-		elemsContainer.Unlock()
+		elemsContainer.filling.Done()
 	}
 }
 
@@ -488,65 +617,84 @@ func (peer *Peer) RoutineSequentialSender(maxBatchSize int) {
 	device := peer.device
 	defer func() {
 		defer device.log.Verbosef("%v - Routine: sequential sender - stopped", peer)
-		peer.stopping.Done()
+		peer.runningState.queueReaders.Done()
 	}()
 	device.log.Verbosef("%v - Routine: sequential sender - started", peer)
 
 	bufs := make([][]byte, 0, maxBatchSize)
 
-	for elemsContainer := range peer.queue.outbound.c {
-		bufs = bufs[:0]
-		if elemsContainer == nil {
-			return
-		}
-		if !peer.isRunning.Load() {
-			// peer has been stopped; return re-usable elems to the shared pool.
-			// This is an optimization only. It is possible for the peer to be stopped
-			// immediately after this check, in which case, elem will get processed.
-			// The timers and SendBuffers code are resilient to a few stragglers.
-			// TODO: rework peer shutdown order to ensure
-			// that we never accidentally keep timers alive longer than necessary.
-			elemsContainer.Lock()
-			for _, elem := range elemsContainer.elems {
-				device.PutMessageBuffer(elem.buffer)
-				device.PutOutboundElement(elem)
-			}
-			device.PutOutboundElementsContainer(elemsContainer)
-			continue
-		}
-		dataSent := false
-		elemsContainer.Lock()
-		for _, elem := range elemsContainer.elems {
-			if len(elem.packet[MessageEncapsulatingTransportSize:]) != MessageKeepaliveSize {
-				dataSent = true
-			}
-			bufs = append(bufs, elem.packet)
-		}
-
-		peer.timersAnyAuthenticatedPacketTraversal()
-		peer.timersAnyAuthenticatedPacketSent()
-
-		err := peer.SendBuffers(bufs)
-		if dataSent {
-			peer.timersDataSent()
-		}
-		for _, elem := range elemsContainer.elems {
-			device.PutMessageBuffer(elem.buffer)
-			device.PutOutboundElement(elem)
-		}
-		device.PutOutboundElementsContainer(elemsContainer)
-		if err != nil {
-			var errGSO conn.ErrUDPGSODisabled
-			if errors.As(err, &errGSO) {
-				device.log.Verbosef(err.Error())
-				err = errGSO.RetryErr
-			}
-		}
-		if err != nil {
-			device.log.Errorf("%v - Failed to send data packets: %v", peer, err)
-			continue
-		}
-
-		peer.keepKeyFreshSending()
+	for elemsContainer := range peer.queue.outbound {
+		peer.processOutboundContainer(elemsContainer, bufs[:0])
 	}
 }
+
+// processOutboundContainer waits for the encryption routine to finish
+// filling elemsContainer, then sends the batch (or drops it, if the peer
+// has been stopped) and returns the container to the pool.
+//
+// scratch is a length-0 slice used to assemble the per-packet buffers
+// passed to SendBuffers; its backing array is reused across calls.
+func (peer *Peer) processOutboundContainer(elemsContainer *QueueOutboundElementsContainer, scratch [][]byte) {
+	// Invariants from RoutineSequentialSender; all should be unreachable.
+	if len(scratch) != 0 || cap(scratch) == 0 {
+		panic(fmt.Sprintf("processOutboundContainer: scratch must be empty with non-zero cap; got len=%d cap=%d",
+			len(scratch), cap(scratch)))
+	}
+	if cap(scratch) < len(elemsContainer.elems) {
+		panic(fmt.Sprintf("processOutboundContainer: scratch cap %d < elems %d",
+			cap(scratch), len(elemsContainer.elems)))
+	}
+
+	device := peer.device
+	defer device.PutOutboundElementsContainer(elemsContainer)
+
+	// Wait for RoutineEncryption to finish filling the container. After
+	// Wait returns we have happens-before with that goroutine and are the
+	// sole owner of the container until Put hands it back to the pool.
+	elemsContainer.filling.Wait()
+
+	if !peer.runningState.isRunning.Load() {
+		// peer has been stopped; return re-usable elems to the shared pool.
+		// This is an optimization only. It is possible for the peer to be stopped
+		// immediately after this check, in which case, elem will get processed.
+		// The timers and SendBuffers code are resilient to a few stragglers.
+		// TODO: rework peer shutdown order to ensure
+		// that we never accidentally keep timers alive longer than necessary.
+		for _, elem := range elemsContainer.elems {
+			device.PutOutboundElement(elem)
+		}
+		return
+	}
+
+	dataSent := false
+	for _, elem := range elemsContainer.elems {
+		if len(elem.packet[MessageEncapsulatingTransportSize:]) != MessageKeepaliveSize {
+			dataSent = true
+		}
+		scratch = append(scratch, elem.packet)
+	}
+
+	peer.timersAnyAuthenticatedPacketTraversal()
+	peer.timersAnyAuthenticatedPacketSent()
+
+	err := peer.SendBuffers(scratch)
+	if dataSent {
+		peer.timersDataSent()
+	}
+	for _, elem := range elemsContainer.elems {
+		device.PutOutboundElement(elem)
+	}
+	if err != nil {
+		var errGSO conn.ErrUDPGSODisabled
+		if errors.As(err, &errGSO) {
+			device.log.Verbosef(err.Error())
+			err = errGSO.RetryErr
+		}
+	}
+	if err != nil {
+		device.log.Errorf("%v - Failed to send data packets: %v", peer, err)
+		return
+	}
+
+	peer.keepKeyFreshSending()
+}

vendor/github.com/tailscale/wireguard-go/device/timers.go

@@ -73,7 +73,7 @@ func (timer *Timer) IsPending() bool {
 }
 
 func (peer *Peer) timersActive() bool {
-	return peer.isRunning.Load() && peer.device != nil && peer.device.isUp()
+	return peer.runningState.isRunning.Load() && peer.device != nil && peer.device.isUp()
 }
 
 func expiredRetransmitHandshake(peer *Peer) {
@@ -95,6 +95,7 @@ func expiredRetransmitHandshake(peer *Peer) {
 		if peer.timersActive() && !peer.timers.zeroKeyMaterial.IsPending() {
 			peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3)
 		}
+		peer.noteSessionHandshakeStopped()
 	} else {
 		peer.timers.handshakeAttempts.Add(1)
 		peer.device.log.Verbosef("%s - Handshake did not complete after %d seconds, retrying (try %d)", peer, int(RekeyTimeout.Seconds()), peer.timers.handshakeAttempts.Load()+1)
@@ -126,6 +127,24 @@ func expiredNewHandshake(peer *Peer) {
 func expiredZeroKeyMaterial(peer *Peer) {
 	peer.device.log.Verbosef("%s - Removing all keys, since we haven't received a new one in %d seconds", peer, int((RejectAfterTime * 3).Seconds()))
 	peer.ZeroAndFlushAll()
+	if peer.deleteOnIdle {
+		peer.device.log.Verbosef("%s - Removing idle lazy peer", peer)
+		// Remove the peer from the device in a new goroutine as we're currently
+		// holding timer locks which RemovePeer also needs. This is TOCTOU, but
+		// acceptable since the worst case is we remove the peer and the lazy
+		// peerfunc created it again after. We might lose some packets.
+		go peer.device.RemovePeer(peer.handshake.remoteStatic)
+	}
+}
+
+func expiredSession(peer *Peer) {
+	peer.sessionState.Lock()
+	defer peer.sessionState.Unlock()
+	if peer.sessionState.sessionExpires.IsZero() || time.Now().Before(peer.sessionState.sessionExpires) {
+		return
+	}
+	peer.device.log.Verbosef("%s - Session expired after %d seconds", peer, int(RejectAfterTime.Seconds()))
+	peer.noteSessionStateLocked(PeerSessionExpired)
 }
 
 func expiredPersistentKeepalive(peer *Peer) {
@@ -171,6 +190,7 @@ func (peer *Peer) timersHandshakeInitiated() {
 	if peer.timersActive() {
 		peer.timers.retransmitHandshake.Mod(RekeyTimeout + time.Millisecond*time.Duration(fastrandn(RekeyTimeoutJitterMaxMs)))
 	}
+	peer.noteSessionHandshakeStarted()
 }
 
 /* Should be called after a handshake response message is received and processed or when getting key confirmation via the first data message. */
@@ -186,7 +206,14 @@ func (peer *Peer) timersHandshakeComplete() {
 /* Should be called after an ephemeral key is created, which is before sending a handshake response or after receiving a handshake response. */
 func (peer *Peer) timersSessionDerived() {
 	if peer.timersActive() {
+		peer.sessionState.Lock()
+		peer.sessionState.sessionExpires = time.Now().Add(RejectAfterTime)
+		peer.noteSessionStateLocked(PeerSessionEstablished)
+		peer.sessionState.Unlock()
+		peer.timers.sessionExpired.Mod(RejectAfterTime)
 		peer.timers.zeroKeyMaterial.Mod(RejectAfterTime * 3)
+	} else {
+		peer.noteSessionState(PeerSessionEstablished)
 	}
 }
 
@@ -202,6 +229,7 @@ func (peer *Peer) timersInit() {
 	peer.timers.retransmitHandshake = peer.NewTimer(expiredRetransmitHandshake)
 	peer.timers.sendKeepalive = peer.NewTimer(expiredSendKeepalive)
 	peer.timers.newHandshake = peer.NewTimer(expiredNewHandshake)
+	peer.timers.sessionExpired = peer.NewTimer(expiredSession)
 	peer.timers.zeroKeyMaterial = peer.NewTimer(expiredZeroKeyMaterial)
 	peer.timers.persistentKeepalive = peer.NewTimer(expiredPersistentKeepalive)
 }
@@ -216,6 +244,7 @@ func (peer *Peer) timersStop() {
 	peer.timers.retransmitHandshake.DelSync()
 	peer.timers.sendKeepalive.DelSync()
 	peer.timers.newHandshake.DelSync()
+	peer.timers.sessionExpired.DelSync()
 	peer.timers.zeroKeyMaterial.DelSync()
 	peer.timers.persistentKeepalive.DelSync()
 }

vendor/github.com/tailscale/wireguard-go/device/uapi.go

@@ -259,9 +259,6 @@ func (peer *ipcSetPeer) handlePostConfig() {
 	if peer.Peer == nil || peer.dummy {
 		return
 	}
-	if peer.created {
-		peer.endpoint.disableRoaming = peer.device.net.brokenRoaming && peer.endpoint.val != nil
-	}
 	if peer.device.isUp() {
 		peer.Start()
 		if peer.pkaOn {

vendor/github.com/tailscale/wireguard-go/tsasm/amd64/chacha20poly1305/chacha20poly1305.go

@@ -0,0 +1,162 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build amd64 && gc && !purego
+
+/*
+Package chacha20poly1305 provides a ChaCha20-Poly1305 AEAD backed by the SSE assembly kernel golang.org/x/crypto deleted in commit 7ee5970 ("chacha20poly1305: drop pre-AVX assembly impl", v0.52.0). It is used on amd64 CPUs with SSSE3 that cannot use x/crypto's AVX2 kernel.
+*/
+package chacha20poly1305
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+	"unsafe"
+
+	"golang.org/x/sys/cpu"
+
+	xchacha20poly1305 "golang.org/x/crypto/chacha20poly1305"
+)
+
+const (
+	KeySize   = 32
+	NonceSize = 12
+	Overhead  = 16
+)
+
+// Available reports whether this CPU can run the kernel, which requires SSSE3.
+func Available() bool { return cpu.X86.HasSSSE3 }
+
+//go:noescape
+func chacha20Poly1305Open(dst []byte, key []uint32, src []byte, ad []byte) bool
+
+//go:noescape
+func chacha20Poly1305Seal(dst []byte, key []uint32, src []byte, ad []byte)
+
+type aead struct {
+	key [KeySize]byte
+}
+
+// New returns a ChaCha20-Poly1305 AEAD using the assembly kernel.
+func New(key []byte) (cipher.AEAD, error) {
+	// Delegate to x/crypto rather than checking the key length here: its New also
+	// refuses ChaCha20-Poly1305 under fips140=only, and this package is reached
+	// instead of x/crypto on the CPUs it covers.
+	if _, err := xchacha20poly1305.New(key); err != nil {
+		return nil, err
+	}
+	if !Available() {
+		return nil, errors.New("chacha20poly1305: CPU lacks SSSE3")
+	}
+	a := &aead{}
+	copy(a.key[:], key)
+	return a, nil
+}
+
+func (a *aead) NonceSize() int { return NonceSize }
+func (a *aead) Overhead() int  { return Overhead }
+
+// setupState writes a ChaCha20 input matrix to state, per RFC 8439 §2.3.
+// Copied from x/crypto's chacha20poly1305_amd64.go.
+func setupState(state *[16]uint32, key *[KeySize]byte, nonce []byte) {
+	state[0] = 0x61707865
+	state[1] = 0x3320646e
+	state[2] = 0x79622d32
+	state[3] = 0x6b206574
+
+	state[4] = binary.LittleEndian.Uint32(key[0:4])
+	state[5] = binary.LittleEndian.Uint32(key[4:8])
+	state[6] = binary.LittleEndian.Uint32(key[8:12])
+	state[7] = binary.LittleEndian.Uint32(key[12:16])
+	state[8] = binary.LittleEndian.Uint32(key[16:20])
+	state[9] = binary.LittleEndian.Uint32(key[20:24])
+	state[10] = binary.LittleEndian.Uint32(key[24:28])
+	state[11] = binary.LittleEndian.Uint32(key[28:32])
+
+	state[12] = 0
+	state[13] = binary.LittleEndian.Uint32(nonce[0:4])
+	state[14] = binary.LittleEndian.Uint32(nonce[4:8])
+	state[15] = binary.LittleEndian.Uint32(nonce[8:12])
+}
+
+func (a *aead) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if len(nonce) != NonceSize {
+		panic("chacha20poly1305: bad nonce length passed to Seal")
+	}
+	if uint64(len(plaintext)) > (1<<38)-64 {
+		panic("chacha20poly1305: plaintext too large")
+	}
+
+	var state [16]uint32
+	setupState(&state, &a.key, nonce)
+
+	ret, out := sliceForAppend(dst, len(plaintext)+Overhead)
+	if inexactOverlap(out, plaintext) {
+		panic("chacha20poly1305: invalid buffer overlap of output and input")
+	}
+	if anyOverlap(out, additionalData) {
+		panic("chacha20poly1305: invalid buffer overlap of output and additional data")
+	}
+	chacha20Poly1305Seal(out[:], state[:], plaintext, additionalData)
+	return ret
+}
+
+var errOpen = errors.New("chacha20poly1305: message authentication failed")
+
+func (a *aead) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	if len(nonce) != NonceSize {
+		// Matches x/crypto: a wrong-size nonce is API misuse, not a decrypt failure.
+		panic("chacha20poly1305: bad nonce length passed to Open")
+	}
+	if len(ciphertext) < Overhead {
+		return nil, errOpen
+	}
+	if uint64(len(ciphertext)) > (1<<38)-48 {
+		panic("chacha20poly1305: ciphertext too large")
+	}
+
+	var state [16]uint32
+	setupState(&state, &a.key, nonce)
+
+	ciphertext = ciphertext[:len(ciphertext)-Overhead]
+	ret, out := sliceForAppend(dst, len(ciphertext))
+	if inexactOverlap(out, ciphertext) {
+		panic("chacha20poly1305: invalid buffer overlap of output and input")
+	}
+	if anyOverlap(out, additionalData) {
+		panic("chacha20poly1305: invalid buffer overlap of output and additional data")
+	}
+	if !chacha20Poly1305Open(out, state[:], ciphertext, additionalData) {
+		for i := range out {
+			out[i] = 0
+		}
+		return nil, errOpen
+	}
+	return ret, nil
+}
+
+func sliceForAppend(dst []byte, n int) (head, tail []byte) {
+	if total := len(dst) + n; cap(dst) >= total {
+		head = dst[:total]
+	} else {
+		head = make([]byte, total)
+		copy(head, dst)
+	}
+	tail = head[len(dst):]
+	return
+}
+
+// anyOverlap and inexactOverlap reproduce x/crypto/internal/alias, which cannot be
+// imported from outside x/crypto: exact aliasing is fine, partial overlap is not.
+func anyOverlap(x, y []byte) bool {
+	return len(x) > 0 && len(y) > 0 &&
+		uintptr(unsafe.Pointer(&x[0])) <= uintptr(unsafe.Pointer(&y[len(y)-1])) &&
+		uintptr(unsafe.Pointer(&y[0])) <= uintptr(unsafe.Pointer(&x[len(x)-1]))
+}
+
+func inexactOverlap(x, y []byte) bool {
+	if len(x) == 0 || len(y) == 0 || &x[0] == &y[0] {
+		return false
+	}
+	return anyOverlap(x, y)
+}

vendor/github.com/tailscale/wireguard-go/tsasm/arm/chacha20/chacha20_arm.go

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Generated from CryptoGAMS chacha-armv4.pl by plan9-xlate.pl. The
+// generated chacha20_arm.s carries the upstream BSD-3-Clause notice
+// in its file header.
+
+//go:build arm
+
+//go:generate sh ../regen/regen.sh
+
+// Package chacha20 implements the ChaCha20 stream cipher with the
+// CryptoGAMS ARMv4 assembly inner loop, regenerated by
+// tsasm/arm/regen/plan9-xlate.pl. It only builds on GOARCH=arm;
+// other architectures should use golang.org/x/crypto/chacha20.
+package chacha20
+
+import (
+	"encoding/binary"
+	"unsafe"
+
+	"golang.org/x/sys/cpu"
+)
+
+const (
+	KeySize   = 32
+	NonceSize = 12
+	BlockSize = 64
+)
+
+// ChaCha20_ctr32 is the scalar upstream entry point: it XORs a
+// ChaCha20 keystream against `length` bytes from inp into out, using
+// key (32 bytes) and ctr_nonce (16 bytes: counter as little-endian
+// u32 followed by the 12-byte nonce).
+//
+//go:noescape
+func ChaCha20_ctr32(out, inp unsafe.Pointer, length uintptr, key, ctr_nonce unsafe.Pointer)
+
+// ChaCha20_neon is the NEON entry point. The CryptoGAMS NEON loop's
+// .Loop_neon_outer head does `cmp r11,#128; bls .Lbreak_neon` and
+// then branches cross-function into ChaCha20_ctr32's main loop --
+// which we rewrite to UDF, since Plan 9 has no way to express that
+// cross-function branch. So `length` must be either 0..256, an exact
+// multiple of 256, or 256*K + R with R in 129..255. XORKeyStream
+// trims to a safe length and sends any leftover 1..128-byte tail
+// through ChaCha20_ctr32.
+//
+//go:noescape
+func ChaCha20_neon(out, inp unsafe.Pointer, length uintptr, key, ctr_nonce unsafe.Pointer)
+
+// XORKeyStream XORs a ChaCha20 keystream into in, writing to out, using
+// the given 32-byte key, 12-byte IETF nonce, and starting 32-bit block
+// counter. out and in must have the same length and may overlap exactly
+// or not at all. It returns the next block counter
+// (counter + ceil(len/64)).
+func XORKeyStream(out, in []byte, key *[KeySize]byte, nonce *[NonceSize]byte, counter uint32) uint32 {
+	if len(out) < len(in) {
+		panic("chacha20: output buffer shorter than input")
+	}
+	if len(in) == 0 {
+		return counter
+	}
+
+	// CryptoGAMS expects the counter+nonce as a single 16-byte block:
+	// 4 little-endian words.
+	var ctrNonce [16]byte
+	binary.LittleEndian.PutUint32(ctrNonce[0:4], counter)
+	copy(ctrNonce[4:], nonce[:])
+
+	// On NEON-capable hardware, hand a "good" prefix length to
+	// ChaCha20_neon. The CryptoGAMS NEON loop processes 256 bytes per
+	// outer iteration and only re-checks for break-out into the scalar
+	// path when entering .Loop_neon_outer; on the first pass through
+	// .Loop_neon_enter that check doesn't fire, so any length up to
+	// 256 bytes is safe. After that, the inner test `cmp r11,#128;
+	// bls .Lbreak_neon` jumps cross-function into ChaCha20_ctr32 --
+	// which we replaced with UDF -- whenever the post-iteration
+	// remainder falls in 1..128 bytes. We avoid that by trimming to a
+	// length whose mod-256 tail is either 0 or > 128, sending the
+	// trimmed remainder (the small 1..128 byte tail when it shows up)
+	// to the scalar path.
+	pos := 0
+	if cpu.ARM.HasNEON {
+		n := len(in)
+		neonLen := n
+		if n > 256 {
+			if rem := n % 256; rem != 0 && rem <= 128 {
+				neonLen = n - rem
+			}
+		}
+		if neonLen > 0 {
+			ChaCha20_neon(
+				unsafe.Pointer(&out[0]),
+				unsafe.Pointer(&in[0]),
+				uintptr(neonLen),
+				unsafe.Pointer(&key[0]),
+				unsafe.Pointer(&ctrNonce[0]),
+			)
+			pos = neonLen
+			binary.LittleEndian.PutUint32(ctrNonce[0:4], counter+uint32(neonLen>>6))
+		}
+	}
+
+	if pos < len(in) {
+		ChaCha20_ctr32(
+			unsafe.Pointer(&out[pos]),
+			unsafe.Pointer(&in[pos]),
+			uintptr(len(in)-pos),
+			unsafe.Pointer(&key[0]),
+			unsafe.Pointer(&ctrNonce[0]),
+		)
+	}
+
+	return counter + uint32((len(in)+63)>>6)
+}

vendor/github.com/tailscale/wireguard-go/tsasm/arm/chacha20poly1305/chacha20poly1305.go

@@ -0,0 +1,136 @@
+// SPDX-License-Identifier: BSD-3-Clause
+
+//go:build arm
+
+// Package chacha20poly1305 provides a ChaCha20-Poly1305 AEAD built
+// from the hand-ported ChaCha20 and Poly1305 kernels in
+// tsasm/arm/chacha20 and tsasm/arm/poly1305. It only builds on
+// GOARCH=arm; other architectures should use
+// golang.org/x/crypto/chacha20poly1305.
+package chacha20poly1305
+
+import (
+	"crypto/cipher"
+	"crypto/subtle"
+	"encoding/binary"
+	"errors"
+
+	chacha "github.com/tailscale/wireguard-go/tsasm/arm/chacha20"
+	"github.com/tailscale/wireguard-go/tsasm/arm/poly1305"
+)
+
+const (
+	KeySize   = 32
+	NonceSize = 12
+	Overhead  = 16
+)
+
+type aead struct {
+	key [KeySize]byte
+}
+
+// New returns a chacha20poly1305 AEAD with the given 32-byte key.
+func New(key []byte) (cipher.AEAD, error) {
+	if len(key) != KeySize {
+		return nil, errors.New("chacha20poly1305: bad key length")
+	}
+	a := &aead{}
+	copy(a.key[:], key)
+	return a, nil
+}
+
+func (a *aead) NonceSize() int { return NonceSize }
+func (a *aead) Overhead() int  { return Overhead }
+
+func (a *aead) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if len(nonce) != NonceSize {
+		panic("chacha20poly1305: bad nonce length")
+	}
+	ret, out := sliceForAppend(dst, len(plaintext)+Overhead)
+
+	var nonceArr [chacha.NonceSize]byte
+	copy(nonceArr[:], nonce)
+
+	// Derive the Poly1305 one-time key from the first 32 bytes of the
+	// ChaCha20 keystream at counter 0, then encrypt at counter 1.
+	var (
+		zeros   [64]byte
+		polyBuf [64]byte
+		polyKey [32]byte
+	)
+	chacha.XORKeyStream(polyBuf[:], zeros[:], &a.key, &nonceArr, 0)
+	copy(polyKey[:], polyBuf[:32])
+	chacha.XORKeyStream(out[:len(plaintext)], plaintext, &a.key, &nonceArr, 1)
+
+	tag := computeTag(additionalData, out[:len(plaintext)], &polyKey)
+	copy(out[len(plaintext):], tag[:])
+	return ret
+}
+
+func (a *aead) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	if len(nonce) != NonceSize {
+		return nil, errors.New("chacha20poly1305: bad nonce length")
+	}
+	if len(ciphertext) < Overhead {
+		return nil, errors.New("chacha20poly1305: ciphertext too short")
+	}
+	ctLen := len(ciphertext) - Overhead
+	ct := ciphertext[:ctLen]
+	receivedTag := ciphertext[ctLen:]
+
+	var nonceArr [chacha.NonceSize]byte
+	copy(nonceArr[:], nonce)
+
+	var (
+		zeros   [64]byte
+		polyBuf [64]byte
+		polyKey [32]byte
+	)
+	chacha.XORKeyStream(polyBuf[:], zeros[:], &a.key, &nonceArr, 0)
+	copy(polyKey[:], polyBuf[:32])
+
+	expectedTag := computeTag(additionalData, ct, &polyKey)
+	if subtle.ConstantTimeCompare(receivedTag, expectedTag[:]) != 1 {
+		return nil, errors.New("chacha20poly1305: message authentication failed")
+	}
+
+	ret, out := sliceForAppend(dst, ctLen)
+	chacha.XORKeyStream(out, ct, &a.key, &nonceArr, 1)
+	return ret, nil
+}
+
+var zeros16 [16]byte
+
+// computeTag streams (aad || pad16 || ct || pad16 || u64le(|aad|) ||
+// u64le(|ct|)) through a stack-allocated Poly1305 MAC.
+func computeTag(aad, ct []byte, polyKey *[32]byte) [16]byte {
+	var mac poly1305.MAC
+	mac.Init(polyKey)
+	mac.Write(aad)
+	if rem := len(aad) % 16; rem != 0 {
+		mac.Write(zeros16[:16-rem])
+	}
+	mac.Write(ct)
+	if rem := len(ct) % 16; rem != 0 {
+		mac.Write(zeros16[:16-rem])
+	}
+	var lenBuf [16]byte
+	binary.LittleEndian.PutUint64(lenBuf[0:8], uint64(len(aad)))
+	binary.LittleEndian.PutUint64(lenBuf[8:16], uint64(len(ct)))
+	mac.Write(lenBuf[:])
+
+	var tag [16]byte
+	mac.Sum(&tag)
+	return tag
+}
+
+func sliceForAppend(dst []byte, n int) (head, tail []byte) {
+	if total := len(dst) + n; cap(dst) >= total {
+		head = dst[:total]
+	} else {
+		head = make([]byte, total)
+		copy(head, dst)
+	}
+	tail = head[len(dst):]
+	return
+}

vendor/github.com/tailscale/wireguard-go/tsasm/arm/poly1305/poly1305_arm.go

@@ -0,0 +1,107 @@
+// SPDX-License-Identifier: BSD-3-Clause
+//
+// Generated from CryptoGAMS poly1305-armv4.pl by plan9-xlate.pl. The
+// generated poly1305_arm.s carries the upstream BSD-3-Clause notice
+// in its file header.
+
+//go:build arm
+
+//go:generate sh ../regen/regen.sh
+
+// Package poly1305 implements Poly1305 with the CryptoGAMS ARMv4
+// assembly inner loop, regenerated by tsasm/arm/regen/plan9-xlate.pl.
+// It only builds on GOARCH=arm; other architectures should use
+// golang.org/x/crypto/poly1305.
+package poly1305
+
+import "unsafe"
+
+const (
+	KeySize = 32
+	TagSize = 16
+)
+
+// ctxSize is the byte size of the upstream poly1305 state struct.
+// Scalar paths use the first 40 bytes (5 base-2^32 h limbs at 0..19,
+// 4 r limbs at 20..35, an is_base2_26 flag at 36). The NEON paths
+// use more; we size generously.
+const ctxSize = 192
+
+//go:noescape
+func poly1305_init(ctx unsafe.Pointer, key unsafe.Pointer)
+
+//go:noescape
+func poly1305_blocks(ctx unsafe.Pointer, inp unsafe.Pointer, length uintptr, padbit uintptr)
+
+//go:noescape
+func poly1305_emit(ctx unsafe.Pointer, mac unsafe.Pointer, nonce unsafe.Pointer)
+
+// Sum computes a Poly1305 MAC of m under key, writing the 16-byte tag
+// to out.
+func Sum(out *[TagSize]byte, m []byte, key *[KeySize]byte) {
+	var mac MAC
+	mac.Init(key)
+	mac.Write(m)
+	mac.Sum(out)
+}
+
+// MAC is an incremental Poly1305 message-authentication code computer.
+// Single-use; not safe for concurrent use. Zero-value is not usable --
+// call Init first.
+type MAC struct {
+	ctx   [ctxSize]byte
+	nonce [16]byte
+	buf   [16]byte
+	n     int
+}
+
+// New returns a fresh MAC keyed with key. Prefer (*MAC).Init for hot
+// paths; this constructor heap-allocates.
+func New(key *[KeySize]byte) *MAC {
+	m := new(MAC)
+	m.Init(key)
+	return m
+}
+
+// Init keys an existing (typically stack-allocated) MAC with key.
+func (m *MAC) Init(key *[KeySize]byte) {
+	*m = MAC{}
+	poly1305_init(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&key[0]))
+	copy(m.nonce[:], key[16:])
+}
+
+// Write feeds bytes into the MAC. Returns len(p), nil.
+func (m *MAC) Write(p []byte) (int, error) {
+	n := len(p)
+	if m.n > 0 {
+		k := copy(m.buf[m.n:], p)
+		m.n += k
+		p = p[k:]
+		if m.n < 16 {
+			return n, nil
+		}
+		poly1305_blocks(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&m.buf[0]), 16, 1)
+		m.n = 0
+	}
+	if blocks := len(p) >> 4; blocks > 0 {
+		poly1305_blocks(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&p[0]), uintptr(blocks*16), 1)
+		p = p[blocks*16:]
+	}
+	if len(p) > 0 {
+		copy(m.buf[:], p)
+		m.n = len(p)
+	}
+	return n, nil
+}
+
+// Sum finalizes the MAC and writes the 16-byte tag to out.
+func (m *MAC) Sum(out *[TagSize]byte) {
+	if m.n > 0 {
+		var pad [16]byte
+		copy(pad[:], m.buf[:m.n])
+		pad[m.n] = 1
+		poly1305_blocks(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&pad[0]), 16, 0)
+		m.n = 0
+	}
+	poly1305_emit(unsafe.Pointer(&m.ctx[0]), unsafe.Pointer(&out[0]), unsafe.Pointer(&m.nonce[0]))
+}

vendor/github.com/tailscale/wireguard-go/tsasm/arm/poly1305/poly1305_arm.s

@@ -0,0 +1,908 @@
+// Code generated by plan9-xlate.pl from CryptoGAMS Perl. DO NOT EDIT.
+//
+// Run `go generate ./tsasm/arm/...` (or tsasm/arm/regen/regen.sh) to
+// regenerate. The upstream chacha-armv4.pl / poly1305-armv4.pl are
+// fetched at a pinned commit by regen.sh; see that script for the
+// upstream URL and SHA. Translation is by tsasm/arm/regen/plan9-xlate.pl.
+//
+// The original CryptoGAMS code this was derived from is dual-licensed
+// under the BSD-3-Clause and OpenSSL licenses, at the holder's option.
+// The CryptoGAMS BSD-3-Clause text is reproduced below; that license
+// requires the copyright notice to travel with the source. Files in
+// tsasm/arm/regen/ that are not derived from CryptoGAMS (plan9-xlate.pl
+// and friends) are BSD-3-Clause without the dual-license option.
+//
+// ----------------------------------------------------------------
+// Copyright (c) 2006, CRYPTOGAMS by <appro@openssl.org>
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+//
+//       * Redistributions of source code must retain copyright
+//         notices, this list of conditions and the following
+//         disclaimer.
+//
+//       * Redistributions in binary form must reproduce the above
+//         copyright notice, this list of conditions and the following
+//         disclaimer in the documentation and/or other materials
+//         provided with the distribution.
+//
+//       * Neither the name of the CRYPTOGAMS nor the names of its
+//         copyright holder and contributors may be used to endorse or
+//         promote products derived from this software without specific
+//         prior written permission.
+//
+// ALTERNATIVELY, provided that this notice is retained in full, this
+// product may be distributed under the terms of the GNU General Public
+// License (GPL), in which case the provisions of the GPL apply INSTEAD
+// OF those given above.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDER AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+// FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+// COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+// INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+// BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+// LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
+// ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+// ----------------------------------------------------------------
+
+#include "textflag.h"
+#include "funcdata.h"
+
+
+// func poly1305_init(ctx uintptr, key uintptr)
+TEXT ·poly1305_init(SB), NOSPLIT, $36-8
+	PCDATA	$PCDATA_UnsafePoint, $-2
+	NO_LOCAL_POINTERS
+	MOVW	ctx+0(FP), R0
+	MOVW	key+4(FP), R1
+	MOVW	R4, 4(R13)
+	MOVW	R5, 8(R13)
+	MOVW	R6, 12(R13)
+	MOVW	R7, 16(R13)
+	MOVW	R8, 20(R13)
+	MOVW	R9, 24(R13)
+	MOVW	R11, 32(R13)
+lb_Lpoly1305_init:
+	EOR	R3, R3, R3
+	CMP	$0, R1
+	MOVW	R3, 0(R0)	// zero hash value
+	MOVW	R3, 4(R0)
+	MOVW	R3, 8(R0)
+	MOVW	R3, 12(R0)
+	MOVW	R3, 16(R0)
+	MOVW	R3, 36(R0)	// clear is_base2_26
+	ADD	$20, R0, R0
+	MOVW.EQ	$0, R0
+	B.EQ	lb_Lno_key
+	MOVBU	0(R1), R4
+	MOVW	R14, 36(R13)
+	MOVW	$0x0fffffff, R14
+	MOVW	R14, 28(R13)
+	MOVBU	1(R1), R5
+	BIC	$3, R14, R3	// 0x0ffffffc
+	MOVBU	2(R1), R6
+	MOVBU	3(R1), R7
+	ORR	R5<<8, R4, R4
+	MOVBU	4(R1), R5
+	ORR	R6<<16, R4, R4
+	MOVBU	5(R1), R6
+	ORR	R7<<24, R4, R4
+	MOVBU	6(R1), R7
+	AND	R14, R4, R4
+	MOVBU	7(R1), R8
+	ORR	R6<<8, R5, R5
+	MOVBU	8(R1), R6
+	ORR	R7<<16, R5, R5
+	MOVBU	9(R1), R7
+	ORR	R8<<24, R5, R5
+	MOVBU	10(R1), R8
+	AND	R3, R5, R5
+	MOVBU	11(R1), R9
+	ORR	R7<<8, R6, R6
+	MOVBU	12(R1), R7
+	ORR	R8<<16, R6, R6
+	MOVBU	13(R1), R8
+	ORR	R9<<24, R6, R6
+	MOVBU	14(R1), R9
+	AND	R3, R6, R6
+	MOVBU	15(R1), R14
+	MOVW	R14, 28(R13)
+	ORR	R8<<8, R7, R7
+	MOVW	R4, 0(R0)
+	ORR	R9<<16, R7, R7
+	MOVW	R5, 4(R0)
+	ORR	R14<<24, R7, R7
+	MOVW	R6, 8(R0)
+	AND	R3, R7, R7
+	MOVW	R7, 12(R0)
+	MOVW	$0, R0
+	MOVW	36(R13), R14
+lb_Lno_key:
+	RET
+
+// func poly1305_blocks(ctx uintptr, inp uintptr, len uintptr, padbit uintptr)
+TEXT ·poly1305_blocks(SB), NOSPLIT, $72-16
+	PCDATA	$PCDATA_UnsafePoint, $-2
+	NO_LOCAL_POINTERS
+	MOVW	ctx+0(FP), R0
+	MOVW	inp+4(FP), R1
+	MOVW	len+8(FP), R2
+	MOVW	padbit+12(FP), R3
+	MOVW	R3, 36(R13)
+	MOVW	R4, 40(R13)
+	MOVW	R5, 44(R13)
+	MOVW	R6, 48(R13)
+	MOVW	R7, 52(R13)
+	MOVW	R8, 56(R13)
+	MOVW	R9, 60(R13)
+	MOVW	R11, 68(R13)
+lb_Lpoly1305_blocks:
+	BIC.S	$15, R2, R2
+	B.EQ	lb_Lno_data
+	ADD	R1, R2, R2	// end pointer
+	MOVW	R14, 72(R13)
+	MOVW	0(R0), R4
+	MOVW	4(R0), R5
+	MOVW	8(R0), R6
+	MOVW	12(R0), R7
+	MOVW	16(R0), R8
+	MOVW	20(R0), R9
+	MOVW	24(R0), R14
+	MOVW	R14, 64(R13)
+	MOVW	28(R0), R11
+	MOVW	32(R0), R12
+	MOVW	72(R13), R14
+	ADD	$20, R0, R0
+	MOVW	R2, 20(R13)	// offload stuff
+	MOVW	R0, 16(R13)
+	MOVW	R1, R14
+	CMP	$0, R3
+	MOVW	R14, 72(R13)
+	MOVW	64(R13), R14
+	MOVW	R14, 24(R13)
+	MOVW	R11, 28(R13)
+	MOVW	R12, 32(R13)
+	MOVW	72(R13), R14
+	B	lb_Loop
+lb_Loop:
+	MOVBU.P	16(R14), R0	// load input
+	ADD.HI	$1, R8, R8	// 1<<128
+	MOVBU	-15(R14), R1
+	MOVBU	-14(R14), R2
+	MOVBU	-13(R14), R3
+	ORR	R1<<8, R0, R1
+	MOVBU	-12(R14), R0
+	ORR	R2<<16, R1, R2
+	MOVBU	-11(R14), R1
+	ORR	R3<<24, R2, R3
+	MOVBU	-10(R14), R2
+	ADD.S	R3, R4, R4	// accumulate input
+	MOVBU	-9(R14), R3
+	ORR	R1<<8, R0, R1
+	MOVBU	-8(R14), R0
+	ORR	R2<<16, R1, R2
+	MOVBU	-7(R14), R1
+	ORR	R3<<24, R2, R3
+	MOVBU	-6(R14), R2
+	ADC.S	R3, R5, R5
+	MOVBU	-5(R14), R3
+	ORR	R1<<8, R0, R1
+	MOVBU	-4(R14), R0
+	ORR	R2<<16, R1, R2
+	MOVBU	-3(R14), R1
+	ORR	R3<<24, R2, R3
+	MOVBU	-2(R14), R2
+	ADC.S	R3, R6, R6
+	MOVBU	-1(R14), R3
+	ORR	R1<<8, R0, R1
+	MOVW	R14, 12(R13)	// offload input pointer
+	ORR	R2<<16, R1, R2
+	MOVW	R14, 72(R13)
+	MOVW	64(R13), R14
+	ADD	R14>>2, R14, R14
+	MOVW	R14, 64(R13)
+	ORR	R3<<24, R2, R3
+	ADD	R11>>2, R11, R11
+	ADC.S	R3, R7, R7
+	ADD	R12>>2, R12, R12
+	MULLU	R5, R9, (R3, R2)
+	ADC	$0, R8, R8
+	MULLU	R4, R9, (R1, R0)
+	MULALU	R8, R14, (R3, R2)
+	MULALU	R7, R14, (R1, R0)
+	MOVW	24(R13), R14	// reload r10
+	MOVW	R14, 64(R13)
+	MULALU	R6, R12, (R3, R2)
+	MULALU	R5, R12, (R1, R0)
+	MULALU	R7, R11, (R3, R2)
+	MULALU	R6, R11, (R1, R0)
+	MULALU	R4, R14, (R3, R2)
+	MOVW	R0, 4(R13)	// future r4
+	MUL	R8, R11, R0
+	MOVW	28(R13), R11	// reload r11
+	ADD.S	R1, R2, R2	// d1+=d0>>32
+	EOR	R1, R1, R1
+	MOVW	72(R13), R14
+	ADC	$0, R3, R14	// future r6
+	MOVW	R2, 8(R13)	// future r5
+	MUL	R8, R12, R2
+	EOR	R3, R3, R3
+	MULALU	R7, R12, (R1, R0)
+	MOVW	32(R13), R12	// reload r12
+	MULALU	R7, R9, (R3, R2)
+	MULALU	R6, R9, (R1, R0)
+	MOVW	R14, 72(R13)
+	MOVW	64(R13), R14
+	MULALU	R6, R14, (R3, R2)
+	MULALU	R5, R14, (R1, R0)
+	MULALU	R5, R11, (R3, R2)
+	MULALU	R4, R11, (R1, R0)
+	MULALU	R4, R12, (R3, R2)
+	MOVW	4(R13), R4
+	MUL	R8, R9, R8
+	MOVW	8(R13), R5
+	MOVW	72(R13), R14
+	ADD.S	R0, R14, R6	// d2+=d1>>32
+	MOVW	12(R13), R14	// reload input pointer
+	ADC	$0, R1, R1
+	ADD.S	R1, R2, R7	// d3+=d2>>32
+	MOVW	20(R13), R0	// reload end pointer
+	ADC	$0, R3, R3
+	ADD	R3, R8, R8	// h4+=d3>>32
+	BIC	$3, R8, R1
+	AND	$3, R8, R8
+	ADD	R1>>2, R1, R1	// *=5
+	ADD.S	R1, R4, R4
+	ADC.S	$0, R5, R5
+	ADC.S	$0, R6, R6
+	ADC.S	$0, R7, R7
+	ADC	$0, R8, R8
+	CMP	R14, R0	// done yet?
+	B.HI	lb_Loop
+	MOVW	16(R13), R0
+	MOVM.DB	[R4, R5, R6, R7, R8], (R0)	// store the result
+lb_Lno_data:
+	RET
+
+// func poly1305_emit(ctx uintptr, mac uintptr, nonce uintptr)
+TEXT ·poly1305_emit(SB), NOSPLIT, $36-12
+	PCDATA	$PCDATA_UnsafePoint, $-2
+	NO_LOCAL_POINTERS
+	MOVW	ctx+0(FP), R0
+	MOVW	mac+4(FP), R1
+	MOVW	nonce+8(FP), R2
+	MOVW	R4, 4(R13)
+	MOVW	R5, 8(R13)
+	MOVW	R6, 12(R13)
+	MOVW	R7, 16(R13)
+	MOVW	R8, 20(R13)
+	MOVW	R9, 24(R13)
+	MOVW	R11, 32(R13)
+lb_Lpoly1305_emit:
+	MOVM.IA	(R0), [R3, R4, R5, R6, R7]
+	ADD.S	$5, R3, R8	// compare to modulus
+	ADC.S	$0, R4, R9
+	MOVW	R14, 36(R13)
+	ADC.S	$0, R5, R14
+	MOVW	R14, 28(R13)
+	ADC.S	$0, R6, R11
+	ADC	$0, R7, R0
+	TST	$4, R0	// did it carry/borrow?
+	MOVW.NE	R8, R3
+	MOVW	0(R2), R8
+	MOVW.NE	R9, R4
+	MOVW	4(R2), R9
+	MOVW.NE	R14, R5
+	MOVW	8(R2), R14
+	MOVW	R14, 28(R13)
+	MOVW.NE	R11, R6
+	MOVW	12(R2), R11
+	ADD.S	R8, R3, R3
+	ADC.S	R9, R4, R4
+	ADC.S	R14, R5, R5
+	ADC	R11, R6, R6
+	MOVB	R3, 0(R1)
+	MOVW	R3>>8, R3
+	MOVB	R4, 4(R1)
+	MOVW	R4>>8, R4
+	MOVB	R5, 8(R1)
+	MOVW	R5>>8, R5
+	MOVB	R6, 12(R1)
+	MOVW	R6>>8, R6
+	MOVB	R3, 1(R1)
+	MOVW	R3>>8, R3
+	MOVB	R4, 5(R1)
+	MOVW	R4>>8, R4
+	MOVB	R5, 9(R1)
+	MOVW	R5>>8, R5
+	MOVB	R6, 13(R1)
+	MOVW	R6>>8, R6
+	MOVB	R3, 2(R1)
+	MOVW	R3>>8, R3
+	MOVB	R4, 6(R1)
+	MOVW	R4>>8, R4
+	MOVB	R5, 10(R1)
+	MOVW	R5>>8, R5
+	MOVB	R6, 14(R1)
+	MOVW	R6>>8, R6
+	MOVB	R3, 3(R1)
+	MOVB	R4, 7(R1)
+	MOVB	R5, 11(R1)
+	MOVB	R6, 15(R1)
+	RET
+TEXT ·poly1305_init_neon(SB), NOSPLIT, $0-8
+	PCDATA	$PCDATA_UnsafePoint, $-2
+	NO_LOCAL_POINTERS
+	MOVW	ctx+0(FP), R0
+	MOVW	key+4(FP), R1
+lb_Lpoly1305_init_neon:
+	MOVW	48(R0), R3	// first table element
+	CMP	$-1, R3	// is value impossible?
+	B.NE	lb_Lno_init_neon
+	MOVW	20(R0), R4	// load key base 2^32
+	MOVW	24(R0), R5
+	MOVW	28(R0), R6
+	MOVW	32(R0), R7
+	AND	$0x03ffffff, R4, R2	// base 2^32 -> base 2^26
+	MOVW	R4>>26, R3
+	MOVW	R5>>20, R4
+	ORR	R5<<6, R3, R3
+	MOVW	R6>>14, R5
+	ORR	R6<<12, R4, R4
+	MOVW	R7>>8, R6
+	ORR	R7<<18, R5, R5
+	AND	$0x03ffffff, R3, R3
+	AND	$0x03ffffff, R4, R4
+	AND	$0x03ffffff, R5, R5
+	WORD	$0xee802b10	// vdup.32 d0,r2
+	ADD	R3<<2, R3, R2	// *5
+	WORD	$0xee813b10	// vdup.32 d1,r3
+	ADD	R4<<2, R4, R3
+	WORD	$0xee822b10	// vdup.32 d2,r2
+	WORD	$0xee834b10	// vdup.32 d3,r4
+	ADD	R5<<2, R5, R4
+	WORD	$0xee843b10	// vdup.32 d4,r3
+	WORD	$0xee855b10	// vdup.32 d5,r5
+	ADD	R6<<2, R6, R5
+	WORD	$0xee864b10	// vdup.32 d6,r4
+	WORD	$0xee876b10	// vdup.32 d7,r6
+	WORD	$0xee885b10	// vdup.32 d8,r5
+	MOVW	$2, R5	// counter
+lb_Lsquare_neon:
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+	// d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
+	// d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
+	// d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
+	// d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
+	WORD	$0xf3a0aa60	// vmull.u32 q5,d0,d0[1]
+	WORD	$0xf3a1ca60	// vmull.u32 q6,d1,d0[1]
+	WORD	$0xf3a3ea60	// vmull.u32 q7,d3,d0[1]
+	WORD	$0xf3e50a60	// vmull.u32 q8,d5,d0[1]
+	WORD	$0xf3e72a60	// vmull.u32 q9,d7,d0[1]
+	WORD	$0xf3a7a262	// vmlal.u32 q5,d7,d2[1]
+	WORD	$0xf3a0c261	// vmlal.u32 q6,d0,d1[1]
+	WORD	$0xf3a1e261	// vmlal.u32 q7,d1,d1[1]
+	WORD	$0xf3e30261	// vmlal.u32 q8,d3,d1[1]
+	WORD	$0xf3e52261	// vmlal.u32 q9,d5,d1[1]
+	WORD	$0xf3a5a264	// vmlal.u32 q5,d5,d4[1]
+	WORD	$0xf3a7c264	// vmlal.u32 q6,d7,d4[1]
+	WORD	$0xf3e10263	// vmlal.u32 q8,d1,d3[1]
+	WORD	$0xf3a0e263	// vmlal.u32 q7,d0,d3[1]
+	WORD	$0xf3e32263	// vmlal.u32 q9,d3,d3[1]
+	WORD	$0xf3a3a266	// vmlal.u32 q5,d3,d6[1]
+	WORD	$0xf3e00265	// vmlal.u32 q8,d0,d5[1]
+	WORD	$0xf3a5c266	// vmlal.u32 q6,d5,d6[1]
+	WORD	$0xf3a7e266	// vmlal.u32 q7,d7,d6[1]
+	WORD	$0xf3e12265	// vmlal.u32 q9,d1,d5[1]
+	WORD	$0xf3e70268	// vmlal.u32 q8,d7,d8[1]
+	WORD	$0xf3a1a268	// vmlal.u32 q5,d1,d8[1]
+	WORD	$0xf3a3c268	// vmlal.u32 q6,d3,d8[1]
+	WORD	$0xf3a5e268	// vmlal.u32 q7,d5,d8[1]
+	WORD	$0xf3e02267	// vmlal.u32 q9,d0,d7[1]
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
+	// and P. Schwabe
+	// H0>>+H1>>+H2>>+H3>>+H4
+	// H3>>+H4>>*5+H0>>+H1
+	// Trivia.
+	// Result of multiplication of n-bit number by m-bit number is
+	// n+m bits wide. However! Even though 2^n is a n+1-bit number,
+	// m-bit number multiplied by 2^n is still n+m bits wide.
+	// Sum of two n-bit numbers is n+1 bits wide, sum of three - n+2,
+	// and so is sum of four. Sum of 2^m n-m-bit numbers and n-bit
+	// one is n+1 bits wide.
+	// >>+ denotes Hnext += Hn>>26, Hn &= 0x3ffffff. This means that
+	// H0, H2, H3 are guaranteed to be 26 bits wide, while H1 and H4
+	// can be 27. However! In cases when their width exceeds 26 bits
+	// they are limited by 2^26+2^6. This in turn means that *sum*
+	// of the products with these values can still be viewed as sum
+	// of 52-bit numbers as long as the amount of addends is not a
+	// power of 2. For example,
+	// H4 = H4*R0 + H3*R1 + H2*R2 + H1*R3 + H0 * R4,
+	// which can't be larger than 5 * (2^26 + 2^6) * (2^26 + 2^6), or
+	// 5 * (2^52 + 2*2^32 + 2^12), which in turn is smaller than
+	// 8 * (2^52) or 2^55. However, the value is then multiplied by
+	// by 5, so we should be looking at 5 * 5 * (2^52 + 2^33 + 2^12),
+	// which is less than 32 * (2^52) or 2^57. And when processing
+	// data we are looking at triple as many addends...
+	// In key setup procedure pre-reduced H0 is limited by 5*4+1 and
+	// 5*H4 - by 5*5 52-bit addends, or 57 bits. But when hashing the
+	// input H0 is limited by (5*4+1)*3 addends, or 58 bits, while
+	// 5*H4 by 5*5*3, or 59[!] bits. How is this relevant? vmlal.u32
+	// instruction accepts 2x32-bit input and writes 2x64-bit result.
+	// This means that result of reduction have to be compressed upon
+	// loop wrap-around. This can be done in the process of reduction
+	// to minimize amount of instructions [as well as amount of
+	// 128-bit instructions, which benefits low-end processors], but
+	// one has to watch for H2 (which is narrower than H0) and 5*H4
+	// not being wider than 58 bits, so that result of right shift
+	// by 26 bits fits in 32 bits. This is also useful on x86,
+	// because it allows to use paddd in place for paddq, which
+	// benefits Atom, where paddq is ridiculously slow.
+	WORD	$0xf3e6e0f0	// vshr.u64 q15,q8,#26
+	WORD	$0xf3fa0220	// vmovn.i64 d16,q8
+	WORD	$0xf3a680da	// vshr.u64 q4,q5,#26
+	WORD	$0xf3baa20a	// vmovn.i64 d10,q5
+	WORD	$0xf27228ee	// vadd.i64 q9,q9,q15
+	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
+	WORD	$0xf23cc848	// vadd.i64 q6,q6,q4
+	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
+	WORD	$0xf2e6e832	// vshrn.u64 d30,q9,#26
+	WORD	$0xf3fa2222	// vmovn.i64 d18,q9
+	WORD	$0xf3a680dc	// vshr.u64 q4,q6,#26
+	WORD	$0xf3bac20c	// vmovn.i64 d12,q6
+	WORD	$0xf23ee848	// vadd.i64 q7,q7,q4
+	WORD	$0xf3c7273c	// vbic.i32 d18,#0xfc000000
+	WORD	$0xf387c73c	// vbic.i32 d12,#0xfc000000
+	WORD	$0xf22aa82e	// vadd.i32 d10,d10,d30
+	WORD	$0xf2e2e53e	// vshl.u32 d30,d30,#2
+	WORD	$0xf2a6881e	// vshrn.u64 d8,q7,#26
+	WORD	$0xf3bae20e	// vmovn.i64 d14,q7
+	WORD	$0xf22aa82e	// vadd.i32 d10,d10,d30
+	WORD	$0xf2600888	// vadd.i32 d16,d16,d8
+	WORD	$0xf387e73c	// vbic.i32 d14,#0xfc000000
+	WORD	$0xf3e6e01a	// vshr.u32 d30,d10,#26
+	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
+	WORD	$0xf3a68030	// vshr.u32 d8,d16,#26
+	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
+	WORD	$0xf22cc82e	// vadd.i32 d12,d12,d30
+	WORD	$0xf2622888	// vadd.i32 d18,d18,d8
+	SUB.S	$1, R5, R5
+	B.EQ	lb_Lsquare_break_neon
+	ADD	$(48+0*9*4), R0, R6
+	ADD	$(48+1*9*4), R0, R7
+	WORD	$0xf3ba008a	// vtrn.32 d0,d10
+	WORD	$0xf3ba308e	// vtrn.32 d3,d14
+	WORD	$0xf3ba50a0	// vtrn.32 d5,d16
+	WORD	$0xf3ba108c	// vtrn.32 d1,d12
+	WORD	$0xf3ba70a2	// vtrn.32 d7,d18
+	WORD	$0xf2a24513	// vshl.u32 d4,d3,#2
+	WORD	$0xf2a26515	// vshl.u32 d6,d5,#2
+	WORD	$0xf2a22511	// vshl.u32 d2,d1,#2
+	WORD	$0xf2a28517	// vshl.u32 d8,d7,#2
+	WORD	$0xf2244803	// vadd.i32 d4,d4,d3
+	WORD	$0xf2222801	// vadd.i32 d2,d2,d1
+	WORD	$0xf2266805	// vadd.i32 d6,d6,d5
+	WORD	$0xf2288807	// vadd.i32 d8,d8,d7
+	WORD	$0xf4860b0d	// vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
+	WORD	$0xf4870b8d	// vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
+	WORD	$0xf4864b0d	// vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
+	WORD	$0xf4874b8d	// vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
+	WORD	$0xf486883f	// vst1.32 {d8[0]},[r6,:32]
+	WORD	$0xf48788bf	// vst1.32 {d8[1]},[r7,:32]
+	B	lb_Lsquare_neon
+lb_Lsquare_break_neon:
+	ADD	$(48+2*4*9), R0, R6
+	ADD	$(48+3*4*9), R0, R7
+	WORD	$0xf22a011a	// vmov d0,d10
+	WORD	$0xf2a2251c	// vshl.u32 d2,d12,#2
+	WORD	$0xf22c111c	// vmov d1,d12
+	WORD	$0xf2a2451e	// vshl.u32 d4,d14,#2
+	WORD	$0xf22e311e	// vmov d3,d14
+	WORD	$0xf2a26530	// vshl.u32 d6,d16,#2
+	WORD	$0xf22051b0	// vmov d5,d16
+	WORD	$0xf2a28532	// vshl.u32 d8,d18,#2
+	WORD	$0xf22271b2	// vmov d7,d18
+	WORD	$0xf222280c	// vadd.i32 d2,d2,d12
+	WORD	$0xf224480e	// vadd.i32 d4,d4,d14
+	WORD	$0xf2266820	// vadd.i32 d6,d6,d16
+	WORD	$0xf2288822	// vadd.i32 d8,d8,d18
+	WORD	$0xf4860b0d	// vst4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
+	WORD	$0xf4870b8d	// vst4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
+	WORD	$0xf4864b0d	// vst4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
+	WORD	$0xf4874b8d	// vst4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
+	WORD	$0xf486880f	// vst1.32 {d8[0]},[r6]
+	WORD	$0xf487888f	// vst1.32 {d8[1]},[r7]
+lb_Lno_init_neon:
+	RET
+
+// func poly1305_blocks_neon(ctx uintptr, inp uintptr, len uintptr, padbit uintptr)
+TEXT ·poly1305_blocks_neon(SB), NOSPLIT, $96-16
+	PCDATA	$PCDATA_UnsafePoint, $-2
+	NO_LOCAL_POINTERS
+	MOVW	ctx+0(FP), R0
+	MOVW	inp+4(FP), R1
+	MOVW	len+8(FP), R2
+	MOVW	padbit+12(FP), R3
+	MOVW	R4, 84(R13)
+	MOVW	R5, 88(R13)
+	MOVW	R6, 92(R13)
+	MOVW	R7, 96(R13)
+lb_Lpoly1305_blocks_neon:
+	MOVW	36(R0), R12	// is_base2_26
+	CMP	$64, R2
+	TST	R12, R12	// is_base2_26?
+	B.NE	lb_Lbase2_26_neon
+	MOVW	R1, 4(R13)
+	MOVW	R2, 8(R13)
+	MOVW	R3, 12(R13)
+	BL	·poly1305_init_neon(SB)
+	MOVW	0(R0), R4	// load hash value base 2^32
+	MOVW	4(R0), R5
+	MOVW	8(R0), R6
+	MOVW	12(R0), R7
+	MOVW	16(R0), R12
+	AND	$0x03ffffff, R4, R2	// base 2^32 -> base 2^26
+	MOVW	R4>>26, R3
+	WORD	$0xf30aa11a	// veor d10,d10,d10
+	MOVW	R5>>20, R4
+	ORR	R5<<6, R3, R3
+	WORD	$0xf30cc11c	// veor d12,d12,d12
+	MOVW	R6>>14, R5
+	ORR	R6<<12, R4, R4
+	WORD	$0xf30ee11e	// veor d14,d14,d14
+	MOVW	R7>>8, R6
+	ORR	R7<<18, R5, R5
+	WORD	$0xf34001b0	// veor d16,d16,d16
+	AND	$0x03ffffff, R3, R3
+	ORR	R12<<24, R6, R6
+	WORD	$0xf34221b2	// veor d18,d18,d18
+	AND	$0x03ffffff, R4, R4
+	MOVW	$1, R1
+	AND	$0x03ffffff, R5, R5
+	MOVW	R1, 36(R0)	// set is_base2_26
+	WORD	$0xee0a2b10	// vmov.32 d10[0],r2
+	WORD	$0xee0c3b10	// vmov.32 d12[0],r3
+	WORD	$0xee0e4b10	// vmov.32 d14[0],r4
+	WORD	$0xee005b90	// vmov.32 d16[0],r5
+	WORD	$0xee026b90	// vmov.32 d18[0],r6
+	MOVW	$zeros<>(SB), R5
+	MOVM.IA.W	(R13), [R1, R2, R3, R14]
+	B	lb_Lhash_loaded
+lb_Lbase2_26_neon:
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// load hash value
+	WORD	$0xf30aa11a	// veor d10,d10,d10
+	WORD	$0xf30cc11c	// veor d12,d12,d12
+	WORD	$0xf30ee11e	// veor d14,d14,d14
+	WORD	$0xf34001b0	// veor d16,d16,d16
+	WORD	$0xf34221b2	// veor d18,d18,d18
+	WORD	$0xf4a0ab4d	// vld4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]!
+	MOVW	$zeros<>(SB), R5
+	WORD	$0xf4e0280f	// vld1.32 {d18[0]},[r0]
+	SUB	$16, R0, R0	// rewind
+lb_Lhash_loaded:
+	ADD	$32, R1, R4
+	MOVW	R3<<24, R3
+	TST	$31, R2
+	B.EQ	lb_Leven
+	WORD	$0xf4e14b4d	// vld4.32 {d20[0],d22[0],d24[0],d26[0]},[r1]!
+	WORD	$0xee0c3b90	// vmov.32 d28[0],r3
+	SUB	$16, R2, R2
+	ADD	$32, R1, R4
+	WORD	$0xf3f8c43a	// vsri.u32 d28,d26,#8
+	WORD	$0xf2f2a53a	// vshl.u32 d26,d26,#18
+	WORD	$0xf3f2a438	// vsri.u32 d26,d24,#14
+	WORD	$0xf2ec8538	// vshl.u32 d24,d24,#12
+	WORD	$0xf26cd8a2	// vadd.i32 d29,d28,d18
+	WORD	$0xf3c7a73c	// vbic.i32 d26,#0xfc000000
+	WORD	$0xf3ec8436	// vsri.u32 d24,d22,#20
+	WORD	$0xf2e66536	// vshl.u32 d22,d22,#6
+	WORD	$0xf3c7873c	// vbic.i32 d24,#0xfc000000
+	WORD	$0xf3e66434	// vsri.u32 d22,d20,#26
+	WORD	$0xf26ab8a0	// vadd.i32 d27,d26,d16
+	WORD	$0xf3c7473c	// vbic.i32 d20,#0xfc000000
+	WORD	$0xf3c7673c	// vbic.i32 d22,#0xfc000000
+	WORD	$0xf268988e	// vadd.i32 d25,d24,d14
+	WORD	$0xf264588a	// vadd.i32 d21,d20,d10
+	WORD	$0xf266788c	// vadd.i32 d23,d22,d12
+	MOVW	R5, R7
+	ADD	$48, R0, R6
+	CMP	R2, R2
+	B	lb_Long_tail
+lb_Leven:
+	SUB.S	$64, R2, R2
+	MOVW.LO	R5, R4
+	WORD	$0xf2c0c651	// vmov.i32 q14,#1<<24
+	WORD	$0xf461418f	// vld4.32 {d20,d22,d24,d26},[r1]
+	ADD	$64, R1, R1
+	WORD	$0xf464518f	// vld4.32 {d21,d23,d25,d27},[r4]
+	ADD	$64, R4, R4
+	ADD.HI	$(48+1*9*4), R0, R7
+	ADD.HI	$(48+3*9*4), R0, R6
+	WORD	$0xf3f8c47a	// vsri.u32 q14,q13,#8
+	WORD	$0xf2f2a57a	// vshl.u32 q13,q13,#18
+	WORD	$0xf3f2a478	// vsri.u32 q13,q12,#14
+	WORD	$0xf2ec8578	// vshl.u32 q12,q12,#12
+	WORD	$0xf3c7a77c	// vbic.i32 q13,#0xfc000000
+	WORD	$0xf3ec8476	// vsri.u32 q12,q11,#20
+	WORD	$0xf2e66576	// vshl.u32 q11,q11,#6
+	WORD	$0xf3c7877c	// vbic.i32 q12,#0xfc000000
+	WORD	$0xf3e66474	// vsri.u32 q11,q10,#26
+	WORD	$0xf3c7477c	// vbic.i32 q10,#0xfc000000
+	WORD	$0xf3c7677c	// vbic.i32 q11,#0xfc000000
+	B.LS	lb_Lskip_loop
+	WORD	$0xf4a70b8d	// vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
+	WORD	$0xf4a60b0d	// vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
+	WORD	$0xf4a74b8d	// vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
+	WORD	$0xf4a64b0d	// vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
+	B	lb_Loop_neon
+lb_Loop_neon:
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
+	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
+	// ___________________/
+	// ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
+	// ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
+	// ___________________/ ____________________/
+	// Note that we start with inp[2:3]*r^2. This is because it
+	// doesn't depend on reduction in previous iteration.
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
+	// d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
+	// d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
+	// d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
+	// d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// inp[2:3]*r^2
+	WORD	$0xf268888e	// vadd.i32 d24,d24,d14
+	WORD	$0xf3a9eae0	// vmull.u32 q7,d25,d0[1]
+	WORD	$0xf264488a	// vadd.i32 d20,d20,d10
+	WORD	$0xf3a5aae0	// vmull.u32 q5,d21,d0[1]
+	WORD	$0xf26aa8a0	// vadd.i32 d26,d26,d16
+	WORD	$0xf3eb0ae0	// vmull.u32 q8,d27,d0[1]
+	WORD	$0xf3a7e2e1	// vmlal.u32 q7,d23,d1[1]
+	WORD	$0xf266688c	// vadd.i32 d22,d22,d12
+	WORD	$0xf3a7cae0	// vmull.u32 q6,d23,d0[1]
+	WORD	$0xf26cc8a2	// vadd.i32 d28,d28,d18
+	WORD	$0xf3ed2ae0	// vmull.u32 q9,d29,d0[1]
+	SUB.S	$64, R2, R2
+	WORD	$0xf3ada2e2	// vmlal.u32 q5,d29,d2[1]
+	MOVW.LO	R5, R4
+	WORD	$0xf3e902e1	// vmlal.u32 q8,d25,d1[1]
+	WORD	$0xf4a788bf	// vld1.32 d8[1],[r7,:32]
+	WORD	$0xf3a5c2e1	// vmlal.u32 q6,d21,d1[1]
+	WORD	$0xf3eb22e1	// vmlal.u32 q9,d27,d1[1]
+	WORD	$0xf3aba2e4	// vmlal.u32 q5,d27,d4[1]
+	WORD	$0xf3e702e3	// vmlal.u32 q8,d23,d3[1]
+	WORD	$0xf3e922e3	// vmlal.u32 q9,d25,d3[1]
+	WORD	$0xf3adc2e4	// vmlal.u32 q6,d29,d4[1]
+	WORD	$0xf3a5e2e3	// vmlal.u32 q7,d21,d3[1]
+	WORD	$0xf3e502e5	// vmlal.u32 q8,d21,d5[1]
+	WORD	$0xf3a9a2e6	// vmlal.u32 q5,d25,d6[1]
+	WORD	$0xf3e722e5	// vmlal.u32 q9,d23,d5[1]
+	WORD	$0xf3abc2e6	// vmlal.u32 q6,d27,d6[1]
+	WORD	$0xf3ade2e6	// vmlal.u32 q7,d29,d6[1]
+	WORD	$0xf3ed02e8	// vmlal.u32 q8,d29,d8[1]
+	WORD	$0xf3a7a2e8	// vmlal.u32 q5,d23,d8[1]
+	WORD	$0xf3e522e7	// vmlal.u32 q9,d21,d7[1]
+	WORD	$0xf3a9c2e8	// vmlal.u32 q6,d25,d8[1]
+	WORD	$0xf3abe2e8	// vmlal.u32 q7,d27,d8[1]
+	WORD	$0xf464518f	// vld4.32 {d21,d23,d25,d27},[r4]
+	ADD	$64, R4, R4
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// (hash+inp[0:1])*r^4 and accumulate
+	WORD	$0xf3ea02c0	// vmlal.u32 q8,d26,d0[0]
+	WORD	$0xf3a4a2c0	// vmlal.u32 q5,d20,d0[0]
+	WORD	$0xf3ec22c0	// vmlal.u32 q9,d28,d0[0]
+	WORD	$0xf3a6c2c0	// vmlal.u32 q6,d22,d0[0]
+	WORD	$0xf3a8e2c0	// vmlal.u32 q7,d24,d0[0]
+	WORD	$0xf4a6883f	// vld1.32 d8[0],[r6,:32]
+	WORD	$0xf3e802c1	// vmlal.u32 q8,d24,d1[0]
+	WORD	$0xf3aca2c2	// vmlal.u32 q5,d28,d2[0]
+	WORD	$0xf3ea22c1	// vmlal.u32 q9,d26,d1[0]
+	WORD	$0xf3a4c2c1	// vmlal.u32 q6,d20,d1[0]
+	WORD	$0xf3a6e2c1	// vmlal.u32 q7,d22,d1[0]
+	WORD	$0xf3e602c3	// vmlal.u32 q8,d22,d3[0]
+	WORD	$0xf3aaa2c4	// vmlal.u32 q5,d26,d4[0]
+	WORD	$0xf3e822c3	// vmlal.u32 q9,d24,d3[0]
+	WORD	$0xf3acc2c4	// vmlal.u32 q6,d28,d4[0]
+	WORD	$0xf3a4e2c3	// vmlal.u32 q7,d20,d3[0]
+	WORD	$0xf3e402c5	// vmlal.u32 q8,d20,d5[0]
+	WORD	$0xf3a8a2c6	// vmlal.u32 q5,d24,d6[0]
+	WORD	$0xf3e622c5	// vmlal.u32 q9,d22,d5[0]
+	WORD	$0xf3aac2c6	// vmlal.u32 q6,d26,d6[0]
+	WORD	$0xf3ec02c8	// vmlal.u32 q8,d28,d8[0]
+	WORD	$0xf3ace2c6	// vmlal.u32 q7,d28,d6[0]
+	WORD	$0xf3a6a2c8	// vmlal.u32 q5,d22,d8[0]
+	WORD	$0xf3e422c7	// vmlal.u32 q9,d20,d7[0]
+	WORD	$0xf2c0c651	// vmov.i32 q14,#1<<24
+	WORD	$0xf3a8c2c8	// vmlal.u32 q6,d24,d8[0]
+	WORD	$0xf3aae2c8	// vmlal.u32 q7,d26,d8[0]
+	WORD	$0xf461418f	// vld4.32 {d20,d22,d24,d26},[r1]
+	ADD	$64, R1, R1
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// lazy reduction interleaved with base 2^32 -> base 2^26 of
+	// inp[0:3] previously loaded to q10-q13 and smashed to q10-q14.
+	WORD	$0xf3e6e0f0	// vshr.u64 q15,q8,#26
+	WORD	$0xf3fa0220	// vmovn.i64 d16,q8
+	WORD	$0xf3a680da	// vshr.u64 q4,q5,#26
+	WORD	$0xf3baa20a	// vmovn.i64 d10,q5
+	WORD	$0xf27228ee	// vadd.i64 q9,q9,q15
+	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
+	WORD	$0xf3f8c47a	// vsri.u32 q14,q13,#8
+	WORD	$0xf23cc848	// vadd.i64 q6,q6,q4
+	WORD	$0xf2f2a57a	// vshl.u32 q13,q13,#18
+	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
+	WORD	$0xf2e6e832	// vshrn.u64 d30,q9,#26
+	WORD	$0xf3fa2222	// vmovn.i64 d18,q9
+	WORD	$0xf3a680dc	// vshr.u64 q4,q6,#26
+	WORD	$0xf3bac20c	// vmovn.i64 d12,q6
+	WORD	$0xf23ee848	// vadd.i64 q7,q7,q4
+	WORD	$0xf3f2a478	// vsri.u32 q13,q12,#14
+	WORD	$0xf3c7273c	// vbic.i32 d18,#0xfc000000
+	WORD	$0xf2ec8578	// vshl.u32 q12,q12,#12
+	WORD	$0xf387c73c	// vbic.i32 d12,#0xfc000000
+	WORD	$0xf22aa82e	// vadd.i32 d10,d10,d30
+	WORD	$0xf2e2e53e	// vshl.u32 d30,d30,#2
+	WORD	$0xf3c7a77c	// vbic.i32 q13,#0xfc000000
+	WORD	$0xf2a6881e	// vshrn.u64 d8,q7,#26
+	WORD	$0xf3bae20e	// vmovn.i64 d14,q7
+	WORD	$0xf3aaa02e	// vaddl.u32 q5,d10,d30
+	WORD	$0xf3ec8476	// vsri.u32 q12,q11,#20
+	WORD	$0xf2600888	// vadd.i32 d16,d16,d8
+	WORD	$0xf2e66576	// vshl.u32 q11,q11,#6
+	WORD	$0xf387e73c	// vbic.i32 d14,#0xfc000000
+	WORD	$0xf3c7877c	// vbic.i32 q12,#0xfc000000
+	WORD	$0xf2e6e81a	// vshrn.u64 d30,q5,#26
+	WORD	$0xf3baa20a	// vmovn.i64 d10,q5
+	WORD	$0xf3e66474	// vsri.u32 q11,q10,#26
+	WORD	$0xf3c7477c	// vbic.i32 q10,#0xfc000000
+	WORD	$0xf3a68030	// vshr.u32 d8,d16,#26
+	WORD	$0xf3c7073c	// vbic.i32 d16,#0xfc000000
+	WORD	$0xf387a73c	// vbic.i32 d10,#0xfc000000
+	WORD	$0xf22cc82e	// vadd.i32 d12,d12,d30
+	WORD	$0xf2622888	// vadd.i32 d18,d18,d8
+	WORD	$0xf3c7677c	// vbic.i32 q11,#0xfc000000
+	B.HI	lb_Loop_neon
+lb_Lskip_loop:
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
+	ADD	$(48+0*9*4), R0, R7
+	ADD	$(48+1*9*4), R0, R6
+	ADD.S	$32, R2, R2
+	MOVW.NE	$0, R2
+	B.NE	lb_Long_tail
+	WORD	$0xf268988e	// vadd.i32 d25,d24,d14
+	WORD	$0xf264588a	// vadd.i32 d21,d20,d10
+	WORD	$0xf26ab8a0	// vadd.i32 d27,d26,d16
+	WORD	$0xf266788c	// vadd.i32 d23,d22,d12
+	WORD	$0xf26cd8a2	// vadd.i32 d29,d28,d18
+lb_Long_tail:
+	WORD	$0xf4a70b8d	// vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
+	WORD	$0xf4a60b0d	// vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
+	WORD	$0xf268888e	// vadd.i32 d24,d24,d14
+	WORD	$0xf3a9ec80	// vmull.u32 q7,d25,d0
+	WORD	$0xf264488a	// vadd.i32 d20,d20,d10
+	WORD	$0xf3a5ac80	// vmull.u32 q5,d21,d0
+	WORD	$0xf26aa8a0	// vadd.i32 d26,d26,d16
+	WORD	$0xf3eb0c80	// vmull.u32 q8,d27,d0
+	WORD	$0xf266688c	// vadd.i32 d22,d22,d12
+	WORD	$0xf3a7cc80	// vmull.u32 q6,d23,d0
+	WORD	$0xf26cc8a2	// vadd.i32 d28,d28,d18
+	WORD	$0xf3ed2c80	// vmull.u32 q9,d29,d0
+	WORD	$0xf3ada882	// vmlal.u32 q5,d29,d2
+	WORD	$0xf4a74b8d	// vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
+	WORD	$0xf3e90881	// vmlal.u32 q8,d25,d1
+	WORD	$0xf4a64b0d	// vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
+	WORD	$0xf3a5c881	// vmlal.u32 q6,d21,d1
+	WORD	$0xf3eb2881	// vmlal.u32 q9,d27,d1
+	WORD	$0xf3a7e881	// vmlal.u32 q7,d23,d1
+	WORD	$0xf3e70883	// vmlal.u32 q8,d23,d3
+	WORD	$0xf4a788bf	// vld1.32 d8[1],[r7,:32]
+	WORD	$0xf3aba884	// vmlal.u32 q5,d27,d4
+	WORD	$0xf4a6883f	// vld1.32 d8[0],[r6,:32]
+	WORD	$0xf3e92883	// vmlal.u32 q9,d25,d3
+	WORD	$0xf3adc884	// vmlal.u32 q6,d29,d4
+	WORD	$0xf3a5e883	// vmlal.u32 q7,d21,d3
+	WORD	$0xf3e50885	// vmlal.u32 q8,d21,d5
+	ADD.NE	$(48+2*9*4), R0, R7
+	WORD	$0xf3a9a886	// vmlal.u32 q5,d25,d6
+	ADD.NE	$(48+3*9*4), R0, R6
+	WORD	$0xf3e72885	// vmlal.u32 q9,d23,d5
+	WORD	$0xf3abc886	// vmlal.u32 q6,d27,d6
+	WORD	$0xf3ade886	// vmlal.u32 q7,d29,d6
+	WORD	$0xf3ed0888	// vmlal.u32 q8,d29,d8
+	WORD	$0xf2300150	// vorn q0,q0,q0
+	WORD	$0xf3a7a888	// vmlal.u32 q5,d23,d8
+	WORD	$0xf39a00d0	// vshr.u64 q0,q0,#38
+	WORD	$0xf3e52887	// vmlal.u32 q9,d21,d7
+	WORD	$0xf3a9c888	// vmlal.u32 q6,d25,d8
+	WORD	$0xf3abe888	// vmlal.u32 q7,d27,d8
+	B.EQ	lb_Lshort_tail
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// (hash+inp[0:1])*r^4:r^3 and accumulate
+	WORD	$0xf4a70b8d	// vld4.32 {d0[1],d1[1],d2[1],d3[1]},[r7]!
+	WORD	$0xf4a60b0d	// vld4.32 {d0[0],d1[0],d2[0],d3[0]},[r6]!
+	WORD	$0xf3a8e880	// vmlal.u32 q7,d24,d0
+	WORD	$0xf3a4a880	// vmlal.u32 q5,d20,d0
+	WORD	$0xf3ea0880	// vmlal.u32 q8,d26,d0
+	WORD	$0xf3a6c880	// vmlal.u32 q6,d22,d0
+	WORD	$0xf3ec2880	// vmlal.u32 q9,d28,d0
+	WORD	$0xf3aca882	// vmlal.u32 q5,d28,d2
+	WORD	$0xf4a74b8d	// vld4.32 {d4[1],d5[1],d6[1],d7[1]},[r7]!
+	WORD	$0xf3e80881	// vmlal.u32 q8,d24,d1
+	WORD	$0xf4a64b0d	// vld4.32 {d4[0],d5[0],d6[0],d7[0]},[r6]!
+	WORD	$0xf3a4c881	// vmlal.u32 q6,d20,d1
+	WORD	$0xf3ea2881	// vmlal.u32 q9,d26,d1
+	WORD	$0xf3a6e881	// vmlal.u32 q7,d22,d1
+	WORD	$0xf3e60883	// vmlal.u32 q8,d22,d3
+	WORD	$0xf4a788bf	// vld1.32 d8[1],[r7,:32]
+	WORD	$0xf3aaa884	// vmlal.u32 q5,d26,d4
+	WORD	$0xf4a6883f	// vld1.32 d8[0],[r6,:32]
+	WORD	$0xf3e82883	// vmlal.u32 q9,d24,d3
+	WORD	$0xf3acc884	// vmlal.u32 q6,d28,d4
+	WORD	$0xf3a4e883	// vmlal.u32 q7,d20,d3
+	WORD	$0xf3e40885	// vmlal.u32 q8,d20,d5
+	WORD	$0xf3a8a886	// vmlal.u32 q5,d24,d6
+	WORD	$0xf3e62885	// vmlal.u32 q9,d22,d5
+	WORD	$0xf3aac886	// vmlal.u32 q6,d26,d6
+	WORD	$0xf3ace886	// vmlal.u32 q7,d28,d6
+	WORD	$0xf3ec0888	// vmlal.u32 q8,d28,d8
+	WORD	$0xf2300150	// vorn q0,q0,q0
+	WORD	$0xf3a6a888	// vmlal.u32 q5,d22,d8
+	WORD	$0xf39a00d0	// vshr.u64 q0,q0,#38
+	WORD	$0xf3e42887	// vmlal.u32 q9,d20,d7
+	WORD	$0xf3a8c888	// vmlal.u32 q6,d24,d8
+	WORD	$0xf3aae888	// vmlal.u32 q7,d26,d8
+lb_Lshort_tail:
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// horizontal addition
+	WORD	$0xf27008a1	// vadd.i64 d16,d16,d17
+	WORD	$0xf23aa80b	// vadd.i64 d10,d10,d11
+	WORD	$0xf27228a3	// vadd.i64 d18,d18,d19
+	WORD	$0xf23cc80d	// vadd.i64 d12,d12,d13
+	WORD	$0xf23ee80f	// vadd.i64 d14,d14,d15
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// lazy reduction, but without narrowing
+	WORD	$0xf3e6e0f0	// vshr.u64 q15,q8,#26
+	WORD	$0xf24001d0	// vand.i64 q8,q8,q0
+	WORD	$0xf3a680da	// vshr.u64 q4,q5,#26
+	WORD	$0xf20aa150	// vand.i64 q5,q5,q0
+	WORD	$0xf27228ee	// vadd.i64 q9,q9,q15
+	WORD	$0xf23cc848	// vadd.i64 q6,q6,q4
+	WORD	$0xf3e6e0f2	// vshr.u64 q15,q9,#26
+	WORD	$0xf24221d0	// vand.i64 q9,q9,q0
+	WORD	$0xf3a680dc	// vshr.u64 q4,q6,#26
+	WORD	$0xf20cc150	// vand.i64 q6,q6,q0
+	WORD	$0xf23ee848	// vadd.i64 q7,q7,q4
+	WORD	$0xf23aa86e	// vadd.i64 q5,q5,q15
+	WORD	$0xf2c2e5fe	// vshl.u64 q15,q15,#2
+	WORD	$0xf3a680de	// vshr.u64 q4,q7,#26
+	WORD	$0xf20ee150	// vand.i64 q7,q7,q0
+	WORD	$0xf23aa86e	// vadd.i64 q5,q5,q15
+	WORD	$0xf27008c8	// vadd.i64 q8,q8,q4
+	WORD	$0xf3e6e0da	// vshr.u64 q15,q5,#26
+	WORD	$0xf20aa150	// vand.i64 q5,q5,q0
+	WORD	$0xf3a680f0	// vshr.u64 q4,q8,#26
+	WORD	$0xf24001d0	// vand.i64 q8,q8,q0
+	WORD	$0xf23cc86e	// vadd.i64 q6,q6,q15
+	WORD	$0xf27228c8	// vadd.i64 q9,q9,q4
+	CMP	$0, R2
+	B.NE	lb_Leven
+	// @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
+	// store hash value
+	WORD	$0xf480ab4d	// vst4.32 {d10[0],d12[0],d14[0],d16[0]},[r0]!
+	WORD	$0xf4c0280f	// vst1.32 {d18[0]},[r0]
+	RET
+lb_LOPENSSL_armcap:
+

vendor/github.com/tailscale/wireguard-go/tun/errors.go

@@ -5,7 +5,7 @@ import (
 )
 
 var (
-	// ErrTooManySegments is returned by Device.Read() when segmentation
+	// ErrTooManySegments is returned by Reader.Read() when segmentation
 	// overflows the length of supplied buffers. This error should not cause
 	// reads to cease.
 	ErrTooManySegments = errors.New("too many segments")

vendor/github.com/tailscale/wireguard-go/tun/offload.go

@@ -2,6 +2,7 @@ package tun
 
 import (
 	"encoding/binary"
+	"errors"
 	"fmt"
 )
 
@@ -73,15 +74,34 @@ const (
 	ipProtoUDP = 17
 )
 
-// GSOSplit splits packets from 'in' into outBufs[<index>][outOffset:], writing
-// the size of each element into sizes. It returns the number of buffers
-// populated, and/or an error. Callers may pass an 'in' slice that overlaps with
-// the first element of outBuffers, i.e. &in[0] may be equal to
-// &outBufs[0][outOffset]. GSONone is a valid options.GSOType regardless of the
-// value of options.NeedsCsum. Length of each outBufs element must be greater
-// than or equal to the length of 'in', otherwise output may be silently
-// truncated.
-func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outOffset int) (int, error) {
+// GSOSplit splits 'in' according to options and writes the resulting segments
+// into slab. It populates packets with each packet's offset and size and returns
+// the number of populated entries.
+//
+// spacing bytes are reserved before the first packet, between adjacent packets,
+// and after the final packet.
+//
+// 'in' may overlap slab only when in begins at slab[spacing]. Overlapping input
+// is modified and must not be used after this call.
+//
+// If packets or slab cannot accommodate every segment, GSOSplit returns the
+// number successfully written together with [ErrTooManySegments]. packets[:n]
+// remains valid.
+func GSOSplit(in []byte, options GSOOptions, slab []byte, packets []ReadPacket, spacing int) (int, error) {
+	if spacing < 0 {
+		return 0, errors.New("spacing must be nonnegative")
+	}
+
+	// return early if instructed to split with no split size
+	if options.GSOType != GSONone && options.GSOSize == 0 {
+		return 0, fmt.Errorf("invalid GSOType (%v) with zero GSOSize", options.GSOType)
+	}
+
+	// return early if we can't fill a single packet
+	if len(packets) == 0 || spacing > len(slab)/2 {
+		return 0, ErrTooManySegments
+	}
+
 	cSumAt := int(options.CsumStart) + int(options.CsumOffset)
 	if cSumAt+1 >= len(in) {
 		return 0, fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(in))
@@ -91,20 +111,34 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 		return 0, fmt.Errorf("length of packet (%d) < GSO HdrLen (%d)", len(in), options.HdrLen)
 	}
 
-	// Handle the conditions where we are copying a single element to outBuffs.
+	// Handle the conditions where we are copying a single element to slab.
 	payloadLen := len(in) - int(options.HdrLen)
 	if options.GSOType == GSONone || payloadLen < int(options.GSOSize) {
-		if len(in) > len(outBufs[0][outOffset:]) {
-			return 0, fmt.Errorf("length of packet (%d) exceeds output element length (%d)", len(in), len(outBufs[0][outOffset:]))
+		// trim spacing on both ends
+		output := slab[spacing : len(slab)-spacing]
+
+		if len(in) > len(output) {
+			return 0, ErrTooManySegments
 		}
+
+		// slice to packet size, so checksum computation below is correct
+		singleOut := output[:len(in)]
+
+		// Copy before checksum computation, so that we don't mutate 'in' if
+		// 'in' and 'slab' happen to be disjoint.
+		copy(singleOut, in)
+		packets[0] = ReadPacket{
+			Offset: spacing,
+			Size:   len(singleOut),
+		}
+
 		if options.NeedsCsum {
 			// The initial value at the checksum offset should be summed with
 			// the checksum we compute. This is typically the pseudo-header sum.
-			initial := binary.BigEndian.Uint16(in[cSumAt:])
-			in[cSumAt], in[cSumAt+1] = 0, 0
-			binary.BigEndian.PutUint16(in[cSumAt:], ^Checksum(in[options.CsumStart:], initial))
+			initial := binary.BigEndian.Uint16(singleOut[cSumAt:])
+			singleOut[cSumAt], singleOut[cSumAt+1] = 0, 0
+			binary.BigEndian.PutUint16(singleOut[cSumAt:], ^Checksum(singleOut[options.CsumStart:], initial))
 		}
-		sizes[0] = copy(outBufs[0][outOffset:], in)
 		return 1, nil
 	}
 
@@ -132,6 +166,62 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 		return 0, fmt.Errorf("invalid ip header version: %d", ipVersion)
 	}
 
+	// Calculate how many segments fit in slab. I found it helpful to mentally
+	// model the slab as:
+	//
+	//	[leading spacing][packet][spacing][packet]...[spacing]
+	//
+	// If every packet were full-sized, after removing the leading spacing each
+	// packet consumes fullPacketSize+spacing bytes.
+	hdrLen := int(options.HdrLen)
+	gsoSize := int(options.GSOSize)
+	// equivalent to ceil(payloadLen / gsoSize), but overflow safe if payloadLen
+	// is close to bounds
+	totalOutSegments := payloadLen / gsoSize
+	if payloadLen%gsoSize != 0 {
+		totalOutSegments++
+	}
+	// how many full gso-sized segments would fit in slab
+	fullPacketSize := hdrLen + gsoSize
+	fullFit := (len(slab) - spacing) / (fullPacketSize + spacing)
+	// this min does not yet account for a smaller tail
+	n := min(totalOutSegments, len(packets), fullFit)
+
+	// Account for the potentially shorter final segment. The exact layout needs
+	// one header per segment, the original payload bytes, and N+1 spacing
+	// regions.
+	requiredBytesLen :=
+		totalOutSegments*hdrLen +
+			payloadLen +
+			(totalOutSegments+1)*spacing
+	if len(packets) >= totalOutSegments && requiredBytesLen <= len(slab) {
+		// n was previously calculated assuming totalOutSegments were all of
+		// gsoSize, but the tail segment may be smaller than gsoSize. If a
+		// smaller than gsoSize tail fits, we can bump n back to totalOutSegments,
+		// assuming packets is long enough.
+		n = totalOutSegments
+	}
+
+	// n is the number of leading segments that fit.
+	//
+	// Since input and output may overlap, split back-to-front. Splitting
+	// front-to-back could overwrite input before it is copied.
+	//
+	// H is the header; A, B, and C are payload segments.
+	//
+	//	Input:
+	//	[ H | A | B | C ]
+	//
+	//	Desired output:
+	//	[ H | A | gap | H | B | gap | H | C ]
+	//
+	//	Front-to-back could overwrite unread B/C:
+	//	[ H | A | gap | H ... ]
+	//	          └── overwrites remaining input ──┘
+	//
+	// For the same reason, copy each segment's payload before its headers.
+
+	// Identify IP addr offsets and length, transport protocol, and transport offsets.
 	iphLen := int(options.CsumStart)
 	srcAddrOffset := ipv6SrcAddrOffset
 	addrLen := 16
@@ -152,22 +242,61 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 	} else {
 		protocol = ipProtoUDP
 	}
-	nextSegmentDataAt := int(options.HdrLen)
-	i := 0
-	for ; nextSegmentDataAt < len(in); i++ {
-		if i == len(outBufs) {
-			return i - 1, ErrTooManySegments
-		}
-		nextSegmentEnd := nextSegmentDataAt + int(options.GSOSize)
-		if nextSegmentEnd > len(in) {
-			nextSegmentEnd = len(in)
-		}
-		segmentDataLen := nextSegmentEnd - nextSegmentDataAt
-		totalLen := int(options.HdrLen) + segmentDataLen
-		sizes[i] = totalLen
-		out := outBufs[i][outOffset:]
 
-		copy(out, in[:iphLen])
+	// The segment-writing loop indexes these fixed IP and transport header
+	// fields within out[:hdrLen]. Validate that each field fits before
+	// modifying slab.
+	csumStart := int(options.CsumStart)
+	csumAt := csumStart + int(options.CsumOffset)
+	if ipVersion == 4 {
+		if hdrLen < 20 {
+			return 0, fmt.Errorf("IPv4 header exceeds GSO header length %d", hdrLen)
+		}
+	} else {
+		if hdrLen < 40 {
+			return 0, fmt.Errorf("IPv6 header exceeds GSO header length %d", hdrLen)
+		}
+	}
+	if protocol == ipProtoTCP {
+		if hdrLen < csumStart+20 {
+			return 0, fmt.Errorf(
+				"TCP header at offset %d exceeds GSO header length %d",
+				csumStart, hdrLen,
+			)
+		}
+	} else {
+		if hdrLen < csumStart+8 {
+			return 0, fmt.Errorf(
+				"UDP header at offset %d exceeds GSO header length %d",
+				csumStart, hdrLen,
+			)
+		}
+	}
+	if hdrLen < csumAt+2 {
+		return 0, fmt.Errorf(
+			"checksum field at offset %d exceeds GSO header length %d",
+			csumAt, hdrLen,
+		)
+	}
+
+	// Split back-to-front
+	for i := n - 1; i >= 0; i-- {
+		// identify input and output offsets
+		payloadStart := hdrLen + (i * gsoSize)
+		payloadEnd := min(payloadStart+gsoSize, len(in))
+		payloadSize := payloadEnd - payloadStart
+		packetSize := hdrLen + payloadSize
+		outOffset := spacing + i*(fullPacketSize+spacing)
+		out := slab[outOffset : outOffset+packetSize]
+		packets[i] = ReadPacket{
+			Offset: outOffset,
+			Size:   packetSize,
+		}
+
+		// Copy payload first because in and out may overlap.
+		copy(out[hdrLen:], in[payloadStart:payloadEnd])
+		copy(out[:hdrLen], in[:hdrLen])
+
 		if ipVersion == 4 {
 			// For IPv4 we are responsible for incrementing the ID field,
 			// updating the total len field, and recalculating the header
@@ -177,44 +306,57 @@ func GSOSplit(in []byte, options GSOOptions, outBufs [][]byte, sizes []int, outO
 				id += uint16(i)
 				binary.BigEndian.PutUint16(out[4:], id)
 			}
-			out[10], out[11] = 0, 0 // clear ipv4 header checksum
-			binary.BigEndian.PutUint16(out[2:], uint16(totalLen))
-			ipv4CSum := ^Checksum(out[:iphLen], 0)
-			binary.BigEndian.PutUint16(out[10:], ipv4CSum)
+			// clear ipv4 header checksum
+			out[10], out[11] = 0, 0
+			// update total len
+			binary.BigEndian.PutUint16(out[2:], uint16(packetSize))
+			// compute & set updated checksum
+			binary.BigEndian.PutUint16(out[10:], ^Checksum(out[:iphLen], 0))
 		} else {
 			// For IPv6 we are responsible for updating the payload length field.
-			binary.BigEndian.PutUint16(out[4:], uint16(totalLen-iphLen))
+			binary.BigEndian.PutUint16(out[4:], uint16(packetSize-iphLen))
 		}
 
-		// copy transport header
-		copy(out[options.CsumStart:options.HdrLen], in[options.CsumStart:options.HdrLen])
-
 		if protocol == ipProtoTCP {
 			// set TCP seq and adjust TCP flags
-			tcpSeq := firstTCPSeqNum + uint32(options.GSOSize*uint16(i))
-			binary.BigEndian.PutUint32(out[options.CsumStart+4:], tcpSeq)
-			if nextSegmentEnd != len(in) {
-				// FIN and PSH should only be set on last segment
+			seq := firstTCPSeqNum + uint32(i*gsoSize)
+			binary.BigEndian.PutUint32(out[options.CsumStart+4:], seq)
+
+			if payloadEnd != len(in) {
+				// FIN and PSH should only be set on last segment. Last
+				// specifically means last input segment, not last output, which
+				// could be last fitting, but not last input.
 				clearFlags := tcpFlagFIN | tcpFlagPSH
 				out[options.CsumStart+tcpFlagsOffset] &^= clearFlags
 			}
 		} else {
 			// set UDP header len
-			binary.BigEndian.PutUint16(out[options.CsumStart+4:], uint16(segmentDataLen)+(options.HdrLen-options.CsumStart))
+			udpLen := uint16(payloadSize) + options.HdrLen - options.CsumStart
+			binary.BigEndian.PutUint16(out[options.CsumStart+4:], udpLen)
 		}
 
-		// payload
-		copy(out[options.HdrLen:], in[nextSegmentDataAt:nextSegmentEnd])
-
 		// transport checksum
 		out[transportCsumAt], out[transportCsumAt+1] = 0, 0 // clear tcp/udp checksum
 		transportHeaderLen := int(options.HdrLen - options.CsumStart)
-		lenForPseudo := uint16(transportHeaderLen + segmentDataLen)
-		transportCSum := PseudoHeaderChecksum(protocol, in[srcAddrOffset:srcAddrOffset+addrLen], in[srcAddrOffset+addrLen:srcAddrOffset+addrLen*2], lenForPseudo)
-		transportCSum = ^Checksum(out[options.CsumStart:totalLen], transportCSum)
-		binary.BigEndian.PutUint16(out[options.CsumStart+options.CsumOffset:], transportCSum)
-
-		nextSegmentDataAt += int(options.GSOSize)
+		pseudoLen := uint16(transportHeaderLen + payloadSize)
+		transportCSum := PseudoHeaderChecksum(
+			protocol,
+			in[srcAddrOffset:srcAddrOffset+addrLen],
+			in[srcAddrOffset+addrLen:srcAddrOffset+2*addrLen],
+			pseudoLen,
+		)
+		transportCSum = ^Checksum(
+			out[options.CsumStart:packetSize],
+			transportCSum,
+		)
+		binary.BigEndian.PutUint16(
+			out[options.CsumStart+options.CsumOffset:],
+			transportCSum,
+		)
 	}
-	return i, nil
+
+	if n < totalOutSegments {
+		return n, ErrTooManySegments
+	}
+	return n, nil
 }

vendor/github.com/tailscale/wireguard-go/tun/offload_linux.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"slices"
 	"unsafe"
 
 	"github.com/tailscale/wireguard-go/conn"
@@ -72,8 +73,76 @@ const (
 	// virtioNetHdrLen is the length in bytes of virtioNetHdr. This matches the
 	// shape of the C ABI for its kernel counterpart -- sizeof(virtio_net_hdr).
 	virtioNetHdrLen = int(unsafe.Sizeof(virtioNetHdr{}))
+
+	// Vector layout: [virtioHdr | headPacket | coalescedPayloadFragments...]
+	iovVirtioNetHdrIdx         = 0
+	iovEmptyVectorLen          = 1
+	iovHeadPacketIdx           = 1
+	iovSinglePacketLen         = 2
+	iovFirstPayloadFragmentIdx = 2
+
+	maxScatterGatherFragments = 1024 // Limited by UIO_MAXIOV of 1024.
 )
 
+// groToWrite holds the write-ordered scatter-gather IO vectors for
+// writev.
+type groToWrite struct {
+	// Each iov is a [][]byte:
+	// - iovs[i][0] is the pre-allocated virtio header, fixed to [virtioNetHdrLen]
+	// - iovs[i][1] is the head packet with transport headers
+	// - iovs[i][2:] are coalesced payload fragments
+	//
+	// Here and elsewhere, "packet" refers to the full IP packet including transport headers,
+	// and "payload fragment" refers to the data without transport headers.
+	//
+	// Empty iov is length 1 (iovEmptyVectorLen), with the first element (iovVirtioNetHdrIdx) zeroed.
+	// Single packet iov is length 2 (iovSinglePacketLen),
+	// with the first element zeroed, and the full packet at index 1 (iovHeadPacketIdx).
+	// Coalesced packet is length >2, with the first element written from [virtioNetHdr],
+	// the second element the head packet,
+	// and the remaining elements (iovFirstPayloadFragmentIdx:) coalesced payload fragments.
+	iovs      [][][]byte
+	allocated int
+}
+
+func newGROToWrite() groToWrite {
+	wi := groToWrite{
+		iovs: make([][][]byte, 0, conn.IdealBatchSize),
+	}
+	for range cap(wi.iovs) {
+		wi.appendIov(nil) // pre-allocate virtio headers
+	}
+	wi.iovs = wi.iovs[:0]
+	return wi
+}
+
+// appendIov extends iovs by one, reusing the pre-allocated backing.
+// Returns the index of the new item's [][]byte.
+func (w *groToWrite) appendIov(pkt []byte) int {
+	n := len(w.iovs)
+	if n < w.allocated {
+		w.iovs = w.iovs[:n+1]
+	} else {
+		iov := make([][]byte, 1, conn.IdealBatchSize)
+		iov[iovVirtioNetHdrIdx] = make([]byte, virtioNetHdrLen)
+		w.iovs = append(w.iovs, iov)
+		w.allocated++
+	}
+	if pkt != nil {
+		// Nil is passed only in [newGROToWrite] and tests.
+		w.iovs[n] = append(w.iovs[n], pkt)
+	}
+	return n
+}
+
+func (w *groToWrite) reset() {
+	for i := range w.iovs {
+		clear(w.iovs[i][iovVirtioNetHdrIdx])
+		w.iovs[i] = w.iovs[i][:iovEmptyVectorLen] // keep virtio header alloc
+	}
+	w.iovs = w.iovs[:0]
+}
+
 // tcpFlowKey represents the key for a TCP flow.
 type tcpFlowKey struct {
 	srcAddr, dstAddr [16]byte
@@ -114,28 +183,30 @@ func newTCPFlowKey(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset int) tcp
 // lookupOrInsert looks up a flow for the provided packet and metadata,
 // returning the packets found for the flow, or inserting a new one if none
 // is found.
-func (t *tcpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex int) ([]tcpGROItem, bool) {
+func (t *tcpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen int, wi *groToWrite) ([]tcpGROItem, bool) {
 	key := newTCPFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset)
 	items, ok := t.itemsByFlow[key]
 	if ok {
 		return items, ok
 	}
 	// TODO: insert() performs another map lookup. This could be rearranged to avoid.
-	t.insert(pkt, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex)
+	t.insert(pkt, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, wi)
 	return nil, false
 }
 
 // insert an item in the table for the provided packet and packet metadata.
-func (t *tcpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen, bufsIndex int) {
+func (t *tcpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, tcphOffset, tcphLen int, wi *groToWrite) {
 	key := newTCPFlowKey(pkt, srcAddrOffset, dstAddrOffset, tcphOffset)
+	idx := wi.appendIov(pkt)
 	item := tcpGROItem{
-		key:       key,
-		bufsIndex: uint16(bufsIndex),
-		gsoSize:   uint16(len(pkt[tcphOffset+tcphLen:])),
-		iphLen:    uint8(tcphOffset),
-		tcphLen:   uint8(tcphLen),
-		sentSeq:   binary.BigEndian.Uint32(pkt[tcphOffset+4:]),
-		pshSet:    pkt[tcphOffset+tcpFlagsOffset]&tcpFlagPSH != 0,
+		key:        key,
+		outputIdx:  uint16(idx),
+		gsoSize:    uint16(len(pkt[tcphOffset+tcphLen:])),
+		iphLen:     uint8(tcphOffset),
+		tcphLen:    uint8(tcphLen),
+		sentSeq:    binary.BigEndian.Uint32(pkt[tcphOffset+4:]),
+		pshSet:     pkt[tcphOffset+tcpFlagsOffset]&tcpFlagPSH != 0,
+		payloadLen: uint16(len(pkt[tcphOffset+tcphLen:])),
 	}
 	items, ok := t.itemsByFlow[key]
 	if !ok {
@@ -159,14 +230,14 @@ func (t *tcpGROTable) deleteAt(key tcpFlowKey, i int) {
 // tcpGROItem represents bookkeeping data for a TCP packet during the lifetime
 // of a GRO evaluation across a vector of packets.
 type tcpGROItem struct {
-	key       tcpFlowKey
-	sentSeq   uint32 // the sequence number
-	bufsIndex uint16 // the index into the original bufs slice
-	numMerged uint16 // the number of packets merged into this item
-	gsoSize   uint16 // payload size
-	iphLen    uint8  // ip header len
-	tcphLen   uint8  // tcp header len
-	pshSet    bool   // psh flag is set
+	key        tcpFlowKey
+	sentSeq    uint32 // the sequence number
+	outputIdx  uint16 // index into groToWrite
+	payloadLen uint16 // accumulated payload bytes
+	gsoSize    uint16 // payload size
+	iphLen     uint8  // ip header len
+	tcphLen    uint8  // tcp header len
+	pshSet     bool   // psh flag is set
 }
 
 func (t *tcpGROTable) newItems() []tcpGROItem {
@@ -221,26 +292,28 @@ func newUDPFlowKey(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int) udp
 // lookupOrInsert looks up a flow for the provided packet and metadata,
 // returning the packets found for the flow, or inserting a new one if none
 // is found.
-func (u *udpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex int) ([]udpGROItem, bool) {
+func (u *udpGROTable) lookupOrInsert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int, wi *groToWrite) ([]udpGROItem, bool) {
 	key := newUDPFlowKey(pkt, srcAddrOffset, dstAddrOffset, udphOffset)
 	items, ok := u.itemsByFlow[key]
 	if ok {
 		return items, ok
 	}
 	// TODO: insert() performs another map lookup. This could be rearranged to avoid.
-	u.insert(pkt, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex, false)
+	u.insert(pkt, srcAddrOffset, dstAddrOffset, udphOffset, wi, false)
 	return nil, false
 }
 
 // insert an item in the table for the provided packet and packet metadata.
-func (u *udpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset, bufsIndex int, cSumKnownInvalid bool) {
+func (u *udpGROTable) insert(pkt []byte, srcAddrOffset, dstAddrOffset, udphOffset int, wi *groToWrite, cSumKnownInvalid bool) {
 	key := newUDPFlowKey(pkt, srcAddrOffset, dstAddrOffset, udphOffset)
+	idx := wi.appendIov(pkt)
 	item := udpGROItem{
 		key:              key,
-		bufsIndex:        uint16(bufsIndex),
+		outputIdx:        uint16(idx),
 		gsoSize:          uint16(len(pkt[udphOffset+udphLen:])),
 		iphLen:           uint8(udphOffset),
 		cSumKnownInvalid: cSumKnownInvalid,
+		payloadLen:       uint16(len(pkt[udphOffset+udphLen:])),
 	}
 	items, ok := u.itemsByFlow[key]
 	if !ok {
@@ -259,8 +332,8 @@ func (u *udpGROTable) updateAt(item udpGROItem, i int) {
 // of a GRO evaluation across a vector of packets.
 type udpGROItem struct {
 	key              udpFlowKey
-	bufsIndex        uint16 // the index into the original bufs slice
-	numMerged        uint16 // the number of packets merged into this item
+	outputIdx        uint16 // index into groToWrite
+	payloadLen       uint16 // accumulated payload bytes
 	gsoSize          uint16 // payload size
 	iphLen           uint8  // ip header len
 	cSumKnownInvalid bool   // UDP header checksum validity; a false value DOES NOT imply valid, just unknown.
@@ -325,15 +398,16 @@ func ipHeadersCanCoalesce(pktA, pktB []byte) bool {
 }
 
 // udpPacketsCanCoalesce evaluates if pkt can be coalesced with the packet
-// described by item. iphLen and gsoSize describe pkt. bufs is the vector of
-// packets involved in the current GRO evaluation. bufsOffset is the offset at
-// which packet data begins within bufs.
-func udpPacketsCanCoalesce(pkt []byte, iphLen uint8, gsoSize uint16, item udpGROItem, bufs [][]byte, bufsOffset int) canCoalesce {
-	pktTarget := bufs[item.bufsIndex][bufsOffset:]
-	if !ipHeadersCanCoalesce(pkt, pktTarget) {
+// described by item. iphLen and gsoSize describe pkt.
+func udpPacketsCanCoalesce(pkt []byte, gsoSize uint16, item udpGROItem, wi *groToWrite) canCoalesce {
+	if len(wi.iovs[item.outputIdx]) >= maxScatterGatherFragments {
 		return coalesceUnavailable
 	}
-	if len(pktTarget[iphLen+udphLen:])%int(item.gsoSize) != 0 {
+	headPacket := wi.iovs[item.outputIdx][iovHeadPacketIdx]
+	if !ipHeadersCanCoalesce(pkt, headPacket) {
+		return coalesceUnavailable
+	}
+	if item.payloadLen%item.gsoSize != 0 {
 		// A smaller than gsoSize packet has been appended previously.
 		// Nothing can come after a smaller packet on the end.
 		return coalesceUnavailable
@@ -342,37 +416,44 @@ func udpPacketsCanCoalesce(pkt []byte, iphLen uint8, gsoSize uint16, item udpGRO
 		// We cannot have a larger packet following a smaller one.
 		return coalesceUnavailable
 	}
+	if int(item.iphLen)+udphLen+int(item.payloadLen)+int(gsoSize) > maxUint16 {
+		return coalesceUnavailable
+	}
 	return coalesceAppend
 }
 
 // tcpPacketsCanCoalesce evaluates if pkt can be coalesced with the packet
 // described by item. This function makes considerations that match the kernel's
 // GRO self tests, which can be found in tools/testing/selftests/net/gro.c.
-func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet bool, gsoSize uint16, item tcpGROItem, bufs [][]byte, bufsOffset int) canCoalesce {
-	pktTarget := bufs[item.bufsIndex][bufsOffset:]
+func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet bool, gsoSize uint16, item tcpGROItem, wi *groToWrite) canCoalesce {
+	if len(wi.iovs[item.outputIdx]) >= maxScatterGatherFragments {
+		return coalesceUnavailable
+	}
+	headPacket := wi.iovs[item.outputIdx][iovHeadPacketIdx]
 	if tcphLen != item.tcphLen {
 		// cannot coalesce with unequal tcp options len
 		return coalesceUnavailable
 	}
 	if tcphLen > 20 {
-		if !bytes.Equal(pkt[iphLen+20:iphLen+tcphLen], pktTarget[item.iphLen+20:iphLen+tcphLen]) {
+		if !bytes.Equal(pkt[iphLen+20:iphLen+tcphLen], headPacket[item.iphLen+20:iphLen+tcphLen]) {
 			// cannot coalesce with unequal tcp options
 			return coalesceUnavailable
 		}
 	}
-	if !ipHeadersCanCoalesce(pkt, pktTarget) {
+	if !ipHeadersCanCoalesce(pkt, headPacket) {
+		return coalesceUnavailable
+	}
+	if int(item.iphLen)+int(item.tcphLen)+int(item.payloadLen)+int(gsoSize) > maxUint16 {
 		return coalesceUnavailable
 	}
 	// seq adjacency
-	lhsLen := item.gsoSize
-	lhsLen += item.numMerged * item.gsoSize
-	if seq == item.sentSeq+uint32(lhsLen) { // pkt aligns following item from a seq num perspective
+	if seq == item.sentSeq+uint32(item.payloadLen) { // pkt aligns following item from a seq num perspective
 		if item.pshSet {
 			// We cannot append to a segment that has the PSH flag set, PSH
 			// can only be set on the final segment in a reassembled group.
 			return coalesceUnavailable
 		}
-		if len(pktTarget[iphLen+tcphLen:])%int(item.gsoSize) != 0 {
+		if item.payloadLen%item.gsoSize != 0 {
 			// A smaller than gsoSize packet has been appended previously.
 			// Nothing can come after a smaller packet on the end.
 			return coalesceUnavailable
@@ -392,7 +473,7 @@ func tcpPacketsCanCoalesce(pkt []byte, iphLen, tcphLen uint8, seq uint32, pshSet
 			// We cannot have a larger packet following a smaller one.
 			return coalesceUnavailable
 		}
-		if gsoSize > item.gsoSize && item.numMerged > 0 {
+		if gsoSize > item.gsoSize && len(wi.iovs[item.outputIdx]) > iovSinglePacketLen {
 			// There's at least one previous merge, and we're larger than all
 			// previous. This would put multiple smaller packets on the end.
 			return coalesceUnavailable
@@ -414,13 +495,11 @@ func checksumValid(pkt []byte, iphLen, proto uint8, isV6 bool) bool {
 	return ^Checksum(pkt[iphLen:], cSum) == 0
 }
 
-// coalesceResult represents the result of attempting to coalesce two TCP
-// packets.
+// coalesceResult represents the result of attempting to coalesce two packets.
 type coalesceResult int
 
 const (
-	coalesceInsufficientCap coalesceResult = iota
-	coalescePSHEnding
+	coalescePSHEnding coalesceResult = iota
 	coalesceItemInvalidCSum
 	coalescePktInvalidCSum
 	coalesceSuccess
@@ -428,54 +507,33 @@ const (
 
 // coalesceUDPPackets attempts to coalesce pkt with the packet described by
 // item, and returns the outcome.
-func coalesceUDPPackets(pkt []byte, item *udpGROItem, bufs [][]byte, bufsOffset int, isV6 bool) coalesceResult {
-	pktHead := bufs[item.bufsIndex][bufsOffset:] // the packet that will end up at the front
-	headersLen := item.iphLen + udphLen
-	coalescedLen := len(bufs[item.bufsIndex][bufsOffset:]) + len(pkt) - int(headersLen)
-
-	if cap(pktHead)-bufsOffset < coalescedLen {
-		// We don't want to allocate a new underlying array if capacity is
-		// too small.
-		return coalesceInsufficientCap
-	}
-	if item.numMerged == 0 {
-		if item.cSumKnownInvalid || !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_UDP, isV6) {
+func coalesceUDPPackets(pkt []byte, item *udpGROItem, wi *groToWrite, isV6 bool) coalesceResult {
+	headersLen := int(item.iphLen) + udphLen
+	iov := &wi.iovs[item.outputIdx]
+	if len(*iov) == iovSinglePacketLen {
+		if item.cSumKnownInvalid || !checksumValid((*iov)[iovHeadPacketIdx], item.iphLen, unix.IPPROTO_UDP, isV6) {
 			return coalesceItemInvalidCSum
 		}
 	}
 	if !checksumValid(pkt, item.iphLen, unix.IPPROTO_UDP, isV6) {
 		return coalescePktInvalidCSum
 	}
-	extendBy := len(pkt) - int(headersLen)
-	bufs[item.bufsIndex] = append(bufs[item.bufsIndex], make([]byte, extendBy)...)
-	copy(bufs[item.bufsIndex][bufsOffset+len(pktHead):], pkt[headersLen:])
-
-	item.numMerged++
+	*iov = append(*iov, pkt[headersLen:])
+	item.payloadLen += uint16(len(pkt) - headersLen)
 	return coalesceSuccess
 }
 
 // coalesceTCPPackets attempts to coalesce pkt with the packet described by
-// item, and returns the outcome. This function may swap bufs elements in the
-// event of a prepend as item's bufs index is already being tracked for writing
-// to a Device.
-func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize uint16, seq uint32, pshSet bool, item *tcpGROItem, bufs [][]byte, bufsOffset int, isV6 bool) coalesceResult {
-	var pktHead []byte // the packet that will end up at the front
-	headersLen := item.iphLen + item.tcphLen
-	coalescedLen := len(bufs[item.bufsIndex][bufsOffset:]) + len(pkt) - int(headersLen)
-
-	// Copy data
+// item, and returns the outcome.
+func coalesceTCPPackets(mode canCoalesce, pkt []byte, gsoSize uint16, seq uint32, pshSet bool, item *tcpGROItem, wi *groToWrite, isV6 bool) coalesceResult {
+	headersLen := int(item.iphLen) + int(item.tcphLen)
+	iov := &wi.iovs[item.outputIdx]
 	if mode == coalescePrepend {
-		pktHead = pkt
-		if cap(pkt)-bufsOffset < coalescedLen {
-			// We don't want to allocate a new underlying array if capacity is
-			// too small.
-			return coalesceInsufficientCap
-		}
 		if pshSet {
 			return coalescePSHEnding
 		}
-		if item.numMerged == 0 {
-			if !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_TCP, isV6) {
+		if len(*iov) == iovSinglePacketLen {
+			if !checksumValid((*iov)[iovHeadPacketIdx], item.iphLen, unix.IPPROTO_TCP, isV6) {
 				return coalesceItemInvalidCSum
 			}
 		}
@@ -483,21 +541,15 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize
 			return coalescePktInvalidCSum
 		}
 		item.sentSeq = seq
-		extendBy := coalescedLen - len(pktHead)
-		bufs[pktBuffsIndex] = append(bufs[pktBuffsIndex], make([]byte, extendBy)...)
-		copy(bufs[pktBuffsIndex][bufsOffset+len(pkt):], bufs[item.bufsIndex][bufsOffset+int(headersLen):])
-		// Flip the slice headers in bufs as part of prepend. The index of item
-		// is already being tracked for writing.
-		bufs[item.bufsIndex], bufs[pktBuffsIndex] = bufs[pktBuffsIndex], bufs[item.bufsIndex]
-	} else {
-		pktHead = bufs[item.bufsIndex][bufsOffset:]
-		if cap(pktHead)-bufsOffset < coalescedLen {
-			// We don't want to allocate a new underlying array if capacity is
-			// too small.
-			return coalesceInsufficientCap
+		oldHead := (*iov)[iovHeadPacketIdx]
+		(*iov)[iovHeadPacketIdx] = pkt
+		oldHeadPayload := oldHead[headersLen:]
+		if len(oldHeadPayload) > 0 {
+			*iov = slices.Insert(*iov, iovFirstPayloadFragmentIdx, oldHeadPayload)
 		}
-		if item.numMerged == 0 {
-			if !checksumValid(bufs[item.bufsIndex][bufsOffset:], item.iphLen, unix.IPPROTO_TCP, isV6) {
+	} else {
+		if len(*iov) == iovSinglePacketLen {
+			if !checksumValid((*iov)[iovHeadPacketIdx], item.iphLen, unix.IPPROTO_TCP, isV6) {
 				return coalesceItemInvalidCSum
 			}
 		}
@@ -507,18 +559,16 @@ func coalesceTCPPackets(mode canCoalesce, pkt []byte, pktBuffsIndex int, gsoSize
 		if pshSet {
 			// We are appending a segment with PSH set.
 			item.pshSet = pshSet
-			pktHead[item.iphLen+tcpFlagsOffset] |= tcpFlagPSH
+			(*iov)[iovHeadPacketIdx][item.iphLen+tcpFlagsOffset] |= tcpFlagPSH
 		}
-		extendBy := len(pkt) - int(headersLen)
-		bufs[item.bufsIndex] = append(bufs[item.bufsIndex], make([]byte, extendBy)...)
-		copy(bufs[item.bufsIndex][bufsOffset+len(pktHead):], pkt[headersLen:])
+		*iov = append(*iov, pkt[headersLen:])
 	}
 
 	if gsoSize > item.gsoSize {
 		item.gsoSize = gsoSize
 	}
 
-	item.numMerged++
+	item.payloadLen += uint16(len(pkt) - headersLen)
 	return coalesceSuccess
 }
 
@@ -538,13 +588,12 @@ const (
 	groResultCoalesced
 )
 
-// tcpGRO evaluates the TCP packet at pktI in bufs for coalescing with
+// tcpGRO evaluates the TCP packet for coalescing with
 // existing packets tracked in table. It returns a groResultNoop when no
 // action was taken, groResultTableInsert when the evaluated packet was
 // inserted into table, and groResultCoalesced when the evaluated packet was
 // coalesced with another packet in table.
-func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool) groResult {
-	pkt := bufs[pktI][offset:]
+func tcpGRO(pkt []byte, table *tcpGROTable, wi *groToWrite, isV6 bool) groResult {
 	if len(pkt) > maxUint16 {
 		// A valid IPv4 or IPv6 packet will never exceed this.
 		return groResultNoop
@@ -599,7 +648,7 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool)
 		srcAddrOffset = ipv6SrcAddrOffset
 		addrLen = 16
 	}
-	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI)
+	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, wi)
 	if !existing {
 		return groResultTableInsert
 	}
@@ -613,9 +662,9 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool)
 		// sequence number perspective, however once an item is inserted into
 		// the table it is never compared across other items later.
 		item := items[i]
-		can := tcpPacketsCanCoalesce(pkt, uint8(iphLen), uint8(tcphLen), seq, pshSet, gsoSize, item, bufs, offset)
+		can := tcpPacketsCanCoalesce(pkt, uint8(iphLen), uint8(tcphLen), seq, pshSet, gsoSize, item, wi)
 		if can != coalesceUnavailable {
-			result := coalesceTCPPackets(can, pkt, pktI, gsoSize, seq, pshSet, &item, bufs, offset, isV6)
+			result := coalesceTCPPackets(can, pkt, gsoSize, seq, pshSet, &item, wi, isV6)
 			switch result {
 			case coalesceSuccess:
 				table.updateAt(item, i)
@@ -631,16 +680,19 @@ func tcpGRO(bufs [][]byte, offset int, pktI int, table *tcpGROTable, isV6 bool)
 		}
 	}
 	// failed to coalesce with any other packets; store the item in the flow
-	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, pktI)
+	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, tcphLen, wi)
 	return groResultTableInsert
 }
 
-// applyTCPCoalesceAccounting updates bufs to account for coalescing based on the
+// applyTCPCoalesceAccounting updates headers to account for coalescing based on the
 // metadata found in table.
-func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) error {
+func applyTCPCoalesceAccounting(wi *groToWrite, table *tcpGROTable) error {
 	for _, items := range table.itemsByFlow {
 		for _, item := range items {
-			if item.numMerged > 0 {
+			iov := wi.iovs[item.outputIdx]
+			pkt := iov[iovHeadPacketIdx]
+			if len(iov) > iovSinglePacketLen {
+				totalLen := uint16(item.iphLen) + uint16(item.tcphLen) + item.payloadLen
 				hdr := virtioNetHdr{
 					flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb
 					hdrLen:     uint16(item.iphLen + item.tcphLen),
@@ -648,21 +700,20 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
 					csumStart:  uint16(item.iphLen),
 					csumOffset: 16,
 				}
-				pkt := bufs[item.bufsIndex][offset:]
 
 				// Recalculate the total len (IPv4) or payload len (IPv6).
 				// Recalculate the (IPv4) header checksum.
 				if item.key.isV6 {
 					hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV6
-					binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len
+					binary.BigEndian.PutUint16(pkt[4:], totalLen-uint16(item.iphLen)) // set new IPv6 header payload len
 				} else {
 					hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_TCPV4
 					pkt[10], pkt[11] = 0, 0
-					binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
-					iphCSum := ^Checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
-					binary.BigEndian.PutUint16(pkt[10:], iphCSum)         // set IPv4 header checksum field
+					binary.BigEndian.PutUint16(pkt[2:], totalLen) // set new total length
+					iphCSum := ^Checksum(pkt[:item.iphLen], 0)    // compute IPv4 header checksum
+					binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field
 				}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
+				err := hdr.encode(iov[iovVirtioNetHdrIdx])
 				if err != nil {
 					return err
 				}
@@ -676,56 +727,51 @@ func applyTCPCoalesceAccounting(bufs [][]byte, offset int, table *tcpGROTable) e
 					addrLen = 16
 					addrOffset = ipv6SrcAddrOffset
 				}
-				srcAddrAt := offset + addrOffset
-				srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
-				dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
-				psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
+				srcAddr := pkt[addrOffset : addrOffset+addrLen]
+				dstAddr := pkt[addrOffset+addrLen : addrOffset+addrLen*2]
+				psum := PseudoHeaderChecksum(unix.IPPROTO_TCP, srcAddr, dstAddr, totalLen-uint16(item.iphLen))
 				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
-			} else {
-				hdr := virtioNetHdr{}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
-				if err != nil {
-					return err
-				}
 			}
 		}
 	}
 	return nil
 }
 
-// applyUDPCoalesceAccounting updates bufs to account for coalescing based on the
+// applyUDPCoalesceAccounting updates headers to account for coalescing based on the
 // metadata found in table.
-func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) error {
+func applyUDPCoalesceAccounting(wi *groToWrite, table *udpGROTable) error {
 	for _, items := range table.itemsByFlow {
 		for _, item := range items {
-			if item.numMerged > 0 {
+			iov := wi.iovs[item.outputIdx]
+			pkt := iov[iovHeadPacketIdx]
+			if len(iov) > iovSinglePacketLen {
+				totalLen := uint16(item.iphLen) + udphLen + item.payloadLen
 				hdr := virtioNetHdr{
 					flags:      unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, // this turns into CHECKSUM_PARTIAL in the skb
-					hdrLen:     uint16(item.iphLen + udphLen),
+					hdrLen:     uint16(item.iphLen) + udphLen,
 					gsoSize:    item.gsoSize,
 					csumStart:  uint16(item.iphLen),
 					csumOffset: 6,
 				}
-				pkt := bufs[item.bufsIndex][offset:]
 
 				// Recalculate the total len (IPv4) or payload len (IPv6).
 				// Recalculate the (IPv4) header checksum.
 				hdr.gsoType = unix.VIRTIO_NET_HDR_GSO_UDP_L4
 				if item.key.isV6 {
-					binary.BigEndian.PutUint16(pkt[4:], uint16(len(pkt))-uint16(item.iphLen)) // set new IPv6 header payload len
+					binary.BigEndian.PutUint16(pkt[4:], totalLen-uint16(item.iphLen)) // set new IPv6 header payload len
 				} else {
 					pkt[10], pkt[11] = 0, 0
-					binary.BigEndian.PutUint16(pkt[2:], uint16(len(pkt))) // set new total length
-					iphCSum := ^Checksum(pkt[:item.iphLen], 0)            // compute IPv4 header checksum
-					binary.BigEndian.PutUint16(pkt[10:], iphCSum)         // set IPv4 header checksum field
+					binary.BigEndian.PutUint16(pkt[2:], totalLen) // set new total length
+					iphCSum := ^Checksum(pkt[:item.iphLen], 0)    // compute IPv4 header checksum
+					binary.BigEndian.PutUint16(pkt[10:], iphCSum) // set IPv4 header checksum field
 				}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
+				err := hdr.encode(iov[iovVirtioNetHdrIdx])
 				if err != nil {
 					return err
 				}
 
 				// Recalculate the UDP len field value
-				binary.BigEndian.PutUint16(pkt[item.iphLen+4:], uint16(len(pkt[item.iphLen:])))
+				binary.BigEndian.PutUint16(pkt[item.iphLen+4:], udphLen+item.payloadLen)
 
 				// Calculate the pseudo header checksum and place it at the UDP
 				// checksum offset. Downstream checksum offloading will combine
@@ -736,17 +782,10 @@ func applyUDPCoalesceAccounting(bufs [][]byte, offset int, table *udpGROTable) e
 					addrLen = 16
 					addrOffset = ipv6SrcAddrOffset
 				}
-				srcAddrAt := offset + addrOffset
-				srcAddr := bufs[item.bufsIndex][srcAddrAt : srcAddrAt+addrLen]
-				dstAddr := bufs[item.bufsIndex][srcAddrAt+addrLen : srcAddrAt+addrLen*2]
-				psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, uint16(len(pkt)-int(item.iphLen)))
+				srcAddr := pkt[addrOffset : addrOffset+addrLen]
+				dstAddr := pkt[addrOffset+addrLen : addrOffset+addrLen*2]
+				psum := PseudoHeaderChecksum(unix.IPPROTO_UDP, srcAddr, dstAddr, totalLen-uint16(item.iphLen))
 				binary.BigEndian.PutUint16(pkt[hdr.csumStart+hdr.csumOffset:], Checksum([]byte{}, psum))
-			} else {
-				hdr := virtioNetHdr{}
-				err := hdr.encode(bufs[item.bufsIndex][offset-virtioNetHdrLen:])
-				if err != nil {
-					return err
-				}
 			}
 		}
 	}
@@ -793,13 +832,12 @@ const (
 	udphLen = 8
 )
 
-// udpGRO evaluates the UDP packet at pktI in bufs for coalescing with
+// udpGRO evaluates the UDP packet for coalescing with
 // existing packets tracked in table. It returns a groResultNoop when no
 // action was taken, groResultTableInsert when the evaluated packet was
 // inserted into table, and groResultCoalesced when the evaluated packet was
 // coalesced with another packet in table.
-func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool) groResult {
-	pkt := bufs[pktI][offset:]
+func udpGRO(pkt []byte, table *udpGROTable, wi *groToWrite, isV6 bool) groResult {
 	if len(pkt) > maxUint16 {
 		// A valid IPv4 or IPv6 packet will never exceed this.
 		return groResultNoop
@@ -840,7 +878,7 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
 		srcAddrOffset = ipv6SrcAddrOffset
 		addrLen = 16
 	}
-	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, pktI)
+	items, existing := table.lookupOrInsert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, wi)
 	if !existing {
 		return groResultTableInsert
 	}
@@ -848,10 +886,10 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
 	// for a given flow. We must also always insert a new item, or successfully
 	// coalesce with an existing item, for the same reason.
 	item := items[len(items)-1]
-	can := udpPacketsCanCoalesce(pkt, uint8(iphLen), gsoSize, item, bufs, offset)
+	can := udpPacketsCanCoalesce(pkt, gsoSize, item, wi)
 	var pktCSumKnownInvalid bool
 	if can == coalesceAppend {
-		result := coalesceUDPPackets(pkt, &item, bufs, offset, isV6)
+		result := coalesceUDPPackets(pkt, &item, wi, isV6)
 		switch result {
 		case coalesceSuccess:
 			table.updateAt(item, len(items)-1)
@@ -868,44 +906,40 @@ func udpGRO(bufs [][]byte, offset int, pktI int, table *udpGROTable, isV6 bool)
 		}
 	}
 	// failed to coalesce with any other packets; store the item in the flow
-	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, pktI, pktCSumKnownInvalid)
+	table.insert(pkt, srcAddrOffset, srcAddrOffset+addrLen, iphLen, wi, pktCSumKnownInvalid)
 	return groResultTableInsert
 }
 
-// handleGRO evaluates bufs for GRO, and writes the indices of the resulting
-// packets into toWrite. toWrite, tcpTable, and udpTable should initially be
+// handleGRO evaluates bufs for GRO, and populates wi with the resulting
+// io vectors for writev. wi, tcpTable, and udpTable should initially be
 // empty (but non-nil), and are passed in to save allocs as the caller may reset
 // and recycle them across vectors of packets. gro indicates if TCP and UDP GRO
 // are supported/enabled.
-func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, toWrite *[]int) error {
+func handleGRO(bufs [][]byte, offset int, tcpTable *tcpGROTable, udpTable *udpGROTable, gro groDisablementFlags, wi *groToWrite) error {
 	for i := range bufs {
 		if offset < virtioNetHdrLen || offset > len(bufs[i])-1 {
 			return errors.New("invalid offset")
 		}
+		pkt := bufs[i][offset:]
 		var result groResult
-		switch packetIsGROCandidate(bufs[i][offset:], gro) {
+		switch packetIsGROCandidate(pkt, gro) {
 		case tcp4GROCandidate:
-			result = tcpGRO(bufs, offset, i, tcpTable, false)
+			result = tcpGRO(pkt, tcpTable, wi, false)
 		case tcp6GROCandidate:
-			result = tcpGRO(bufs, offset, i, tcpTable, true)
+			result = tcpGRO(pkt, tcpTable, wi, true)
 		case udp4GROCandidate:
-			result = udpGRO(bufs, offset, i, udpTable, false)
+			result = udpGRO(pkt, udpTable, wi, false)
 		case udp6GROCandidate:
-			result = udpGRO(bufs, offset, i, udpTable, true)
+			result = udpGRO(pkt, udpTable, wi, true)
 		}
 		switch result {
 		case groResultNoop:
-			hdr := virtioNetHdr{}
-			err := hdr.encode(bufs[i][offset-virtioNetHdrLen:])
-			if err != nil {
-				return err
-			}
-			fallthrough
+			wi.appendIov(pkt)
 		case groResultTableInsert:
-			*toWrite = append(*toWrite, i)
+			// already in wi via table insert
 		}
 	}
-	errTCP := applyTCPCoalesceAccounting(bufs, offset, tcpTable)
-	errUDP := applyUDPCoalesceAccounting(bufs, offset, udpTable)
+	errTCP := applyTCPCoalesceAccounting(wi, tcpTable)
+	errUDP := applyUDPCoalesceAccounting(wi, udpTable)
 	return errors.Join(errTCP, errUDP)
 }

vendor/github.com/tailscale/wireguard-go/tun/tun.go

@@ -17,18 +17,55 @@ const (
 	EventMTUUpdate
 )
 
-type Device interface {
-	// File returns the file descriptor of the device.
+// ReadPacket describes a packet read by [Reader.Read].
+type ReadPacket struct {
+	// Offset is the starting byte offset.
+	Offset int
+	// Size is the size of the packet.
+	Size int
+}
+
+// ReadPacketSpacing is the number of bytes reserved before the first packet,
+// between adjacent packets, and after the final packet filled by [Reader.Read].
+const ReadPacketSpacing = 64
+
+// Reader reads packets from a single queue of a [Device].
+type Reader interface {
+	// Read reads one or more packets from the queue into slab. On return, it
+	// populates packets and returns the number of entries to evaluate. Those
+	// entries are valid even when err is non-nil. Callers must provide at least
+	// [Device.BatchSize] entries.
+	//
+	// Read reserves [ReadPacketSpacing] bytes before the first packet, between
+	// adjacent packets, and after the final packet. The contents of the reserved
+	// space are unspecified; the [Device] may use them while Read is executing.
+	// Callers must provide a slab of at least 2*[ReadPacketSpacing] bytes.
+	//
+	// Read returns [ErrTooManySegments] if packets or slab cannot accommodate
+	// all packets produced by the read.
+	Read(slab []byte, packets []ReadPacket) (n int, err error)
+}
+
+// Queue is a single read queue of a [Device]. Distinct Queues of one Device
+// may be used concurrently.
+//
+// A [Device] implementation may return read-only Queues in addition to those
+// backed by file descriptors. [MultiQueueDevice.WriteTo] is the way to
+// dispatch writes to the queues that accept them.
+type Queue interface {
+	Reader
+
+	// File returns the queue's file descriptor, or nil if the queue is not
+	// backed by one.
 	File() *os.File
+}
 
-	// Read one or more packets from the Device (without any additional headers).
-	// On a successful read it returns the number of packets read, and sets
-	// packet lengths within the sizes slice. len(sizes) must be >= len(bufs).
-	// A nonzero offset can be used to instruct the Device on where to begin
-	// reading into each element of the bufs slice.
-	Read(bufs [][]byte, sizes []int, offset int) (n int, err error)
+type Device interface {
+	// Queue is the Device's first queue. A [MultiQueueDevice] exposes the
+	// rest.
+	Queue
 
-	// Write one or more packets to the device (without any additional headers).
+	// Write one or more packets to the Device.
 	// On a successful write it returns the number of packets written. A nonzero
 	// offset can be used to instruct the Device on where to begin writing from
 	// each packet contained within the bufs slice.
@@ -52,6 +89,82 @@ type Device interface {
 	BatchSize() int
 }
 
+// MultiQueueDevice is a Device that exposes more than one kernel queue, each
+// backed by its own file descriptor. The count is fixed for the Device's
+// lifetime.
+type MultiQueueDevice interface {
+	Device
+
+	// Queues returns one [Queue] per kernel queue with at least one entry.
+	// Entry 0 is equivalent to the Device's own [Reader.Read].
+	Queues() []Queue
+
+	// WriteTo is [Device.Write] directed at one of the Device's write queues.
+	// Data written with the same flow id lands on the same write queue.
+	WriteTo(flow int, bufs [][]byte, offset int) (int, error)
+}
+
+// WriteToOf returns dev's queue-aware write, or [Device.Write] for a Device
+// with one write queue.
+func WriteToOf(dev Device) func(flow int, bufs [][]byte, offset int) (int, error) {
+	if w, ok := dev.(MultiQueueDevice); ok {
+		return w.WriteTo
+	}
+	return func(_ int, bufs [][]byte, offset int) (int, error) {
+		return dev.Write(bufs, offset)
+	}
+}
+
+// QueuesOf returns dev's queues, or dev itself as the single queue. The result
+// always has at least one entry.
+func QueuesOf(dev Device) []Queue {
+	if mq, ok := dev.(MultiQueueDevice); ok {
+		if qs := mq.Queues(); len(qs) > 0 {
+			return qs
+		}
+	}
+	return []Queue{dev}
+}
+
+// An Option configures a [Device] at creation time.
+type Option interface {
+	apply(*config)
+}
+
+type optionFunc func(*config)
+
+func (f optionFunc) apply(config *config) {
+	f(config)
+}
+
+type config struct {
+	queues int // Tun fd's in IFF_MULTI_QUEUE group to open.
+}
+
+func defaultConfig() config {
+	return config{
+		queues: 1,
+	}
+}
+
+// WithQueues requests that the [Device] be created with n queues. n must be at
+// least 1.
+//
+// Only Linux implements multiqueue TUN. Errors are returned if more queues
+// are requested than the Linux kernel supports (at the time of writing,
+// v7.0 has MAX_TAP_QUEUES at 256) or if the per-process limit was reached.
+//
+// Implementations may choose to return more read queues than was requested.
+// Callers must consult [QueuesOf] rather than assume they got n.
+//
+// EXPERIMENTAL: Current implementation introduces nonce reordering with more
+// than one queue.
+func WithQueues(n int) Option {
+	return optionFunc(func(config *config) {
+		config.queues = n
+	})
+}
+
 // GRODevice is a Device extended with methods for disabling GRO. Certain OS
 // versions may have offload bugs. Where these bugs negatively impact throughput
 // or break connectivity entirely we can use these methods to disable the

vendor/github.com/tailscale/wireguard-go/tun/tun_bsd.go

@@ -0,0 +1,23 @@
+//go:build freebsd || openbsd || darwin
+
+/* SPDX-License-Identifier: MIT
+ *
+ * Copyright (C) 2017-2023 WireGuard LLC. All Rights Reserved.
+ */
+
+package tun
+
+import (
+	"fmt"
+	"os"
+)
+
+func createTUNFromFilesNoMQ(files []*os.File, mtu int) (Device, error) {
+	if len(files) != 1 {
+		for _, f := range files {
+			f.Close()
+		}
+		return nil, fmt.Errorf("got %d TUN queue files: multiqueue TUN is only supported on Linux", len(files))
+	}
+	return CreateTUNFromFile(files[0], mtu)
+}

vendor/github.com/tailscale/wireguard-go/tun/tun_darwin.go

@@ -98,7 +98,7 @@ func (tun *NativeTun) routineRouteListener(tunIfindex int) {
 	}
 }
 
-func CreateTUN(name string, mtu int) (Device, error) {
+func CreateTUN(name string, mtu int, _ ...Option) (Device, error) {
 	ifIndex := -1
 	if name != "utun" {
 		_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
@@ -148,6 +148,10 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	return tun, err
 }
 
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	return createTUNFromFilesNoMQ(files, mtu)
+}
+
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 	tun := &NativeTun{
 		tunFile: file,
@@ -217,7 +221,7 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	// TODO: the BSDs look very similar in Read() and Write(). They should be
 	// collapsed, with platform-specific files containing the varying parts of
 	// their implementations.
@@ -225,12 +229,13 @@ func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error)
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		buf := bufs[0][offset-4:]
+		buf := slab[ReadPacketSpacing-4 : len(slab)-ReadPacketSpacing]
 		n, err := tun.tunFile.Read(buf[:])
 		if n < 4 {
 			return 0, err
 		}
-		sizes[0] = n - 4
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n - 4
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_freebsd.go

@@ -159,7 +159,7 @@ func tunDestroy(name string) error {
 	return nil
 }
 
-func CreateTUN(name string, mtu int) (Device, error) {
+func CreateTUN(name string, mtu int, _ ...Option) (Device, error) {
 	if len(name) > unix.IFNAMSIZ-1 {
 		return nil, errors.New("interface name too long")
 	}
@@ -261,6 +261,10 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	return CreateTUNFromFile(tunFile, mtu)
 }
 
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	return createTUNFromFilesNoMQ(files, mtu)
+}
+
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 	tun := &NativeTun{
 		tunFile: file,
@@ -333,17 +337,18 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		buf := bufs[0][offset-4:]
+		buf := slab[ReadPacketSpacing-4 : len(slab)-ReadPacketSpacing]
 		n, err := tun.tunFile.Read(buf[:])
 		if n < 4 {
 			return 0, err
 		}
-		sizes[0] = n - 4
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n - 4
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_linux.go

@@ -13,6 +13,7 @@ import (
 	"fmt"
 	"os"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 	"unsafe"
@@ -28,12 +29,14 @@ const (
 )
 
 type NativeTun struct {
-	tunFile                 *os.File
-	index                   int32      // if index
-	errors                  chan error // async error handling
-	events                  chan Event // device related events
-	netlinkSock             int
-	netlinkCancel           *rwcancel.RWCancel
+	queues []*tunQueue // queues[0] always exists and is used by Read and Write
+
+	index         int32      // if index
+	errors        chan error // async error handling
+	events        chan Event // device related events
+	netlinkSock   int
+	netlinkCancel *rwcancel.RWCancel
+
 	hackListenerClosed      sync.Mutex
 	statusListenersShutdown chan struct{}
 	batchSize               int
@@ -45,14 +48,52 @@ type NativeTun struct {
 	nameCache string    // name of interface
 	nameErr   error
 
-	readOpMu sync.Mutex                    // readOpMu guards readBuff
-	readBuff [virtioNetHdrLen + 65535]byte // if vnetHdr every read() is prefixed by virtioNetHdr
+	gro atomic.Int32 // groDisablementFlags for cross-queue access
+}
+
+// tunQueue is a single kernel queue of a tun device, and is the [Queue]
+// handed out by [NativeTun.Queues].
+// An individual tunQueue is not concurrency-safe, but distinct queues can
+// be read or written concurrently.
+type tunQueue struct {
+	tun     *NativeTun
+	file    *os.File
+	rawConn syscall.RawConn
 
 	writeOpMu   sync.Mutex // writeOpMu guards the following fields
-	toWrite     []int
+	toWrite     groToWrite
 	tcpGROTable *tcpGROTable
 	udpGROTable *udpGROTable
-	gro         groDisablementFlags
+}
+
+// appendQueue wraps file as a new queue of tun. It must be called before tun is
+// published to other goroutines, and only appends on success.
+func appendQueue(tun *NativeTun, file *os.File) error {
+	q := &tunQueue{
+		tun:         tun,
+		file:        file,
+		tcpGROTable: newTCPGROTable(),
+		udpGROTable: newUDPGROTable(),
+		toWrite:     newGROToWrite(),
+	}
+	var err error
+	q.rawConn, err = file.SyscallConn()
+	if err != nil {
+		return err
+	}
+	tun.queues = append(tun.queues, q)
+	return nil
+}
+
+func (q *tunQueue) File() *os.File { return q.file }
+
+// Queues implements [MultiQueueDevice].
+func (tun *NativeTun) Queues() []Queue {
+	qs := make([]Queue, 0, len(tun.queues))
+	for _, q := range tun.queues {
+		qs = append(qs, q)
+	}
+	return qs
 }
 
 type groDisablementFlags int
@@ -62,24 +103,16 @@ const (
 	udpGRODisabled
 )
 
-func (g *groDisablementFlags) disableTCPGRO() {
-	*g |= tcpGRODisabled
-}
-
 func (g *groDisablementFlags) canTCPGRO() bool {
 	return (*g)&tcpGRODisabled == 0
 }
 
-func (g *groDisablementFlags) disableUDPGRO() {
-	*g |= udpGRODisabled
-}
-
 func (g *groDisablementFlags) canUDPGRO() bool {
 	return (*g)&udpGRODisabled == 0
 }
 
 func (tun *NativeTun) File() *os.File {
-	return tun.tunFile
+	return tun.queues[0].file
 }
 
 func (tun *NativeTun) routineHackListener() {
@@ -93,7 +126,7 @@ func (tun *NativeTun) routineHackListener() {
 		down = 2
 	)
 	for {
-		sysconn, err := tun.tunFile.SyscallConn()
+		sysconn, err := tun.File().SyscallConn()
 		if err != nil {
 			return
 		}
@@ -326,9 +359,13 @@ func (tun *NativeTun) initNameCache() {
 }
 
 func (tun *NativeTun) nameSlow() (string, error) {
-	sysconn, err := tun.tunFile.SyscallConn()
+	return tunNameOf(tun.File())
+}
+
+func tunNameOf(file *os.File) (string, error) {
+	sysconn, err := file.SyscallConn()
 	if err != nil {
-		return "", err
+		return "", fmt.Errorf("failed to get name of TUN device: %w", err)
 	}
 	var ifr [ifReqSize]byte
 	var errno syscall.Errno
@@ -350,35 +387,73 @@ func (tun *NativeTun) nameSlow() (string, error) {
 }
 
 func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
-	tun.writeOpMu.Lock()
+	return tun.queues[0].write(bufs, offset)
+}
+
+// WriteTo implements [MultiQueueDevice].
+func (tun *NativeTun) WriteTo(flow int, bufs [][]byte, offset int) (int, error) {
+	return tun.queues[uint(flow)%uint(len(tun.queues))].write(bufs, offset)
+}
+
+func (q *tunQueue) write(bufs [][]byte, offset int) (int, error) {
+	q.writeOpMu.Lock()
 	defer func() {
-		tun.tcpGROTable.reset()
-		tun.udpGROTable.reset()
-		tun.writeOpMu.Unlock()
+		q.tcpGROTable.reset()
+		q.udpGROTable.reset()
+		q.toWrite.reset()
+		q.writeOpMu.Unlock()
 	}()
 	var (
 		errs  error
 		total int
 	)
-	tun.toWrite = tun.toWrite[:0]
-	if tun.vnetHdr {
-		err := handleGRO(bufs, offset, tun.tcpGROTable, tun.udpGROTable, tun.gro, &tun.toWrite)
-		if err != nil {
-			return 0, err
-		}
-		offset -= virtioNetHdrLen
-	} else {
+	if !q.tun.vnetHdr {
 		for i := range bufs {
-			tun.toWrite = append(tun.toWrite, i)
+			n, err := q.file.Write(bufs[i][offset:])
+			if errors.Is(err, syscall.EBADFD) {
+				return total, os.ErrClosed
+			}
+			if err != nil {
+				errs = errors.Join(errs, err)
+			} else {
+				total += n
+			}
 		}
+		return total, errs
 	}
-	for _, bufsI := range tun.toWrite {
-		n, err := tun.tunFile.Write(bufs[bufsI][offset:])
-		if errors.Is(err, syscall.EBADFD) {
+	err := handleGRO(
+		bufs,
+		offset,
+		q.tcpGROTable,
+		q.udpGROTable,
+		groDisablementFlags(q.tun.gro.Load()),
+		&q.toWrite,
+	)
+	if err != nil {
+		return 0, err
+	}
+	for _, nb := range q.toWrite.iovs {
+		var werr error
+		var n int
+		err := q.rawConn.Write(func(fd uintptr) bool {
+			for {
+				n, werr = unix.Writev(int(fd), nb)
+				if errors.Is(werr, syscall.EINTR) {
+					continue // quick retry on interrupt, EINTR is never returned with partial writes
+				}
+				return !errors.Is(werr, syscall.EAGAIN) // poller retry on "would block"
+			}
+		})
+		// err is a poller error (e.g. fd closed before the syscall)
+		// werr is the Writev syscall error itself.
+		if err != nil {
+			return total, err
+		}
+		if errors.Is(werr, syscall.EBADFD) {
 			return total, os.ErrClosed
 		}
-		if err != nil {
-			errs = errors.Join(errs, err)
+		if werr != nil {
+			errs = errors.Join(errs, werr)
 		} else {
 			total += n
 		}
@@ -386,10 +461,11 @@ func (tun *NativeTun) Write(bufs [][]byte, offset int) (int, error) {
 	return total, errs
 }
 
-// handleVirtioRead splits in into bufs, leaving offset bytes at the front of
-// each buffer. It mutates sizes to reflect the size of each element of bufs,
-// and returns the number of packets read.
-func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, error) {
+// handleVirtioRead splits in into slab, leaving [ReadPacketSpacing] bytes at
+// the front of the first packet, between every adjacent packet, and at the end
+// of the last packet. It mutates packets to reflect the size and offset of each
+// packet in slab, and returns the number of packets read.
+func handleVirtioRead(in []byte, slab []byte, packets []ReadPacket) (int, error) {
 	var hdr virtioNetHdr
 	err := hdr.decode(in)
 	if err != nil {
@@ -421,31 +497,39 @@ func handleVirtioRead(in []byte, bufs [][]byte, sizes []int, offset int) (int, e
 		options.HdrLen = options.CsumStart + tcpHLen
 	}
 
-	return GSOSplit(in, options, bufs, sizes, offset)
+	return GSOSplit(in, options, slab, packets, ReadPacketSpacing)
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
-	tun.readOpMu.Lock()
-	defer tun.readOpMu.Unlock()
+// assert that [ReadPacketSpacing] is >= [virtioNetHdrLen], as [NativeTun.Read]
+// assumes so, and uses headroom for [virtioNetHdr].
+const _ = uint(ReadPacketSpacing - virtioNetHdrLen)
+
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
+	return tun.queues[0].Read(slab, packets)
+}
+
+func (q *tunQueue) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
-	case err := <-tun.errors:
+	case err := <-q.tun.errors:
 		return 0, err
 	default:
-		readInto := bufs[0][offset:]
-		if tun.vnetHdr {
-			readInto = tun.readBuff[:]
+		start := ReadPacketSpacing
+		if q.tun.vnetHdr {
+			start -= virtioNetHdrLen
 		}
-		n, err := tun.tunFile.Read(readInto)
+		readInto := slab[start : len(slab)-ReadPacketSpacing]
+		n, err := q.file.Read(readInto)
 		if errors.Is(err, syscall.EBADFD) {
 			err = os.ErrClosed
 		}
 		if err != nil {
 			return 0, err
 		}
-		if tun.vnetHdr {
-			return handleVirtioRead(readInto[:n], bufs, sizes, offset)
+		if q.tun.vnetHdr {
+			return handleVirtioRead(readInto[:n], slab, packets)
 		} else {
-			sizes[0] = n
+			packets[0].Size = n
+			packets[0].Offset = ReadPacketSpacing
 			return 1, nil
 		}
 	}
@@ -466,7 +550,11 @@ func (tun *NativeTun) Close() error {
 		} else if tun.events != nil {
 			close(tun.events)
 		}
-		err2 = tun.tunFile.Close()
+		for _, q := range tun.queues {
+			if cerr := q.file.Close(); cerr != nil && err2 == nil {
+				err2 = cerr
+			}
+		}
 	})
 	if err1 != nil {
 		return err1
@@ -481,17 +569,13 @@ func (tun *NativeTun) BatchSize() int {
 // DisableUDPGRO disables UDP GRO if it is enabled. See the GRODevice interface
 // for cases where it should be called.
 func (tun *NativeTun) DisableUDPGRO() {
-	tun.writeOpMu.Lock()
-	tun.gro.disableUDPGRO()
-	tun.writeOpMu.Unlock()
+	tun.gro.Or(int32(udpGRODisabled))
 }
 
 // DisableTCPGRO disables TCP GRO if it is enabled. See the GRODevice interface
 // for cases where it should be called.
 func (tun *NativeTun) DisableTCPGRO() {
-	tun.writeOpMu.Lock()
-	tun.gro.disableTCPGRO()
-	tun.writeOpMu.Unlock()
+	tun.gro.Or(int32(tcpGRODisabled))
 }
 
 const (
@@ -501,7 +585,7 @@ const (
 )
 
 func (tun *NativeTun) initFromFlags(name string) error {
-	sc, err := tun.tunFile.SyscallConn()
+	sc, err := tun.File().SyscallConn()
 	if err != nil {
 		return err
 	}
@@ -530,7 +614,7 @@ func (tun *NativeTun) initFromFlags(name string) error {
 			// tunUDPOffloads were added in Linux v6.2. We do not return an
 			// error if they are unsupported at runtime.
 			if unix.IoctlSetInt(int(fd), unix.TUNSETOFFLOAD, tunTCPOffloads|tunUDPOffloads) != nil {
-				tun.gro.disableUDPGRO()
+				tun.DisableUDPGRO()
 			}
 		} else {
 			tun.batchSize = 1
@@ -541,8 +625,8 @@ func (tun *NativeTun) initFromFlags(name string) error {
 	return err
 }
 
-// CreateTUN creates a Device with the provided name and MTU.
-func CreateTUN(name string, mtu int) (Device, error) {
+// openTUNFile opens /dev/net/tun and attaches it to name.
+func openTUNFile(name string, flags uint16) (*os.File, error) {
 	nfd, err := unix.Open(cloneDevicePath, unix.O_RDWR|unix.O_CLOEXEC, 0)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -553,13 +637,13 @@ func CreateTUN(name string, mtu int) (Device, error) {
 
 	ifr, err := unix.NewIfreq(name)
 	if err != nil {
+		unix.Close(nfd)
 		return nil, err
 	}
-	// IFF_VNET_HDR enables the "tun status hack" via routineHackListener()
-	// where a null write will return EINVAL indicating the TUN is up.
-	ifr.SetUint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR)
+	ifr.SetUint16(flags)
 	err = unix.IoctlIfreq(nfd, unix.TUNSETIFF, ifr)
 	if err != nil {
+		unix.Close(nfd)
 		return nil, err
 	}
 
@@ -570,46 +654,122 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	}
 
 	// Note that the above -- open,ioctl,nonblock -- must happen prior to handing it to netpoll as below this line.
+	return os.NewFile(uintptr(nfd), cloneDevicePath), nil
+}
 
-	fd := os.NewFile(uintptr(nfd), cloneDevicePath)
-	return CreateTUNFromFile(fd, mtu)
+// CreateTUN creates a Device with the provided name and MTU.
+func CreateTUN(name string, mtu int, opts ...Option) (Device, error) {
+	config := defaultConfig()
+	for _, opt := range opts {
+		opt.apply(&config)
+	}
+	if config.queues < 1 {
+		return nil, fmt.Errorf("%d tun queues requested, must be at least 1", config.queues)
+	}
+	// IFF_VNET_HDR enables the "tun status hack" via routineHackListener()
+	// where a null write will return EINVAL indicating the TUN is up.
+	flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_VNET_HDR)
+	if config.queues > 1 {
+		// all members must declare multiqueue
+		flags |= unix.IFF_MULTI_QUEUE
+	}
+	first, err := openTUNFile(name, flags)
+	if err != nil {
+		return nil, err
+	}
+	files := []*os.File{first}
+	closeAll := func() {
+		for _, f := range files {
+			f.Close()
+		}
+	}
+	if config.queues > 1 {
+		// Resolve name wildcards (%d) to not spawn new groups.
+		resolved, err := tunNameOf(first)
+		if err != nil {
+			closeAll()
+			return nil, err
+		}
+		for i := range config.queues - 1 {
+			f, err := openTUNFile(resolved, flags)
+			if err != nil {
+				closeAll()
+				return nil, fmt.Errorf("attaching extra tun queue %d of %d: %w", i, config.queues-1, err)
+			}
+			files = append(files, f)
+		}
+	}
+	return CreateTUNFromFiles(files, mtu)
 }
 
 // CreateTUNFromFile creates a Device from an os.File with the provided MTU.
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
+	return CreateTUNFromFiles([]*os.File{file}, mtu)
+}
+
+// CreateTUNFromFiles creates a Device with the provided MTU from one file per
+// kernel queue. It satisfies [MultiQueueDevice] when there is more than one.
+// Files must be distinct and ownership transfers.
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	if len(files) == 0 {
+		return nil, errors.New("CreateTUNFromFiles requires at least one file")
+	}
 	tun := &NativeTun{
-		tunFile:                 file,
 		events:                  make(chan Event, 5),
 		errors:                  make(chan error, 5),
 		statusListenersShutdown: make(chan struct{}),
-		tcpGROTable:             newTCPGROTable(),
-		udpGROTable:             newUDPGROTable(),
-		toWrite:                 make([]int, 0, conn.IdealBatchSize),
+	}
+	// Append every queue before spawning routineHackListener et al.
+	for _, file := range files {
+		if err := appendQueue(tun, file); err != nil {
+			for _, f := range files {
+				f.Close()
+			}
+			return nil, err
+		}
 	}
 
 	name, err := tun.Name()
 	if err != nil {
+		tun.Close()
 		return nil, err
 	}
+	// Verify that all fds are attached to the same interface.
+	for i, file := range files[1:] {
+		got, err := tunNameOf(file)
+		if err != nil {
+			tun.Close()
+			return nil, fmt.Errorf("tun queue %d: %w", i+1, err)
+		}
+		if got != name {
+			tun.Close()
+			return nil, fmt.Errorf("tun queue %d is attached to %q, want %q", i+1, got, name)
+		}
+	}
 
-	err = tun.initFromFlags(name)
-	if err != nil {
+	// Offload is negotiated once for the whole device, on queue 0.
+	if err := tun.initFromFlags(name); err != nil {
+		tun.Close()
 		return nil, err
 	}
 
 	// start event listener
 	tun.index, err = getIFIndex(name)
 	if err != nil {
+		tun.Close()
 		return nil, err
 	}
 
 	tun.netlinkSock, err = createNetlinkSocket()
 	if err != nil {
+		tun.Close()
 		return nil, err
 	}
 	tun.netlinkCancel, err = rwcancel.NewRWCancel(tun.netlinkSock)
 	if err != nil {
+		// No goroutine owns netlinkSock yet, and tun.Close won't reach it.
 		unix.Close(tun.netlinkSock)
+		tun.Close()
 		return nil, err
 	}
 
@@ -619,7 +779,7 @@ func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 
 	err = tun.setMTU(mtu)
 	if err != nil {
-		unix.Close(tun.netlinkSock)
+		tun.Close()
 		return nil, err
 	}
 
@@ -629,26 +789,76 @@ func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 // CreateUnmonitoredTUNFromFD creates a Device from the provided file
 // descriptor.
 func CreateUnmonitoredTUNFromFD(fd int) (Device, string, error) {
-	err := unix.SetNonblock(fd, true)
-	if err != nil {
-		return nil, "", err
+	return CreateUnmonitoredTUNFromFDs([]int{fd})
+}
+
+// CreateUnmonitoredTUNFromFDs creates a Device from one descriptor per kernel
+// queue. It satisfies [MultiQueueDevice] when there is more than one.
+func CreateUnmonitoredTUNFromFDs(fds []int) (Device, string, error) {
+	if len(fds) == 0 {
+		return nil, "", errors.New("CreateUnmonitoredTUNFromFDs requires at least one fd")
+	}
+	// Error if passed duplicate FDs.
+	var dupes []int
+	set := map[int]struct{}{}
+	for _, v := range fds {
+		if _, dup := set[v]; dup {
+			dupes = append(dupes, v)
+			continue
+		}
+		set[v] = struct{}{}
+	}
+	if len(dupes) != 0 {
+		for fd := range set {
+			unix.Close(fd)
+		}
+		return nil, "", fmt.Errorf("passed duplicate file descriptors: %v", dupes)
 	}
-	file := os.NewFile(uintptr(fd), "/dev/tun")
 	tun := &NativeTun{
-		tunFile:     file,
-		events:      make(chan Event, 5),
-		errors:      make(chan error, 5),
-		tcpGROTable: newTCPGROTable(),
-		udpGROTable: newUDPGROTable(),
-		toWrite:     make([]int, 0, conn.IdealBatchSize),
+		events: make(chan Event, 5),
+		errors: make(chan error, 5),
 	}
+	for i, fd := range fds {
+		// See openTUNFile: this must precede os.NewFile.
+		if err := unix.SetNonblock(fd, true); err != nil {
+			tun.Close()
+			for _, rest := range fds[i:] {
+				unix.Close(rest)
+			}
+			return nil, "", err
+		}
+		file := os.NewFile(uintptr(fd), "/dev/tun")
+		if err := appendQueue(tun, file); err != nil {
+			file.Close() // not saved, so tun.Close won't reach it
+			tun.Close()
+			for _, rest := range fds[i+1:] {
+				unix.Close(rest)
+			}
+			return nil, "", err
+		}
+	}
+
 	name, err := tun.Name()
 	if err != nil {
+		tun.Close()
 		return nil, "", err
 	}
-	err = tun.initFromFlags(name)
-	if err != nil {
+	for i, q := range tun.queues[1:] {
+		got, err := tunNameOf(q.file)
+		if err != nil {
+			tun.Close()
+			return nil, "", fmt.Errorf("tun queue %d: %w", i+1, err)
+		}
+		if got != name {
+			tun.Close()
+			return nil, "", fmt.Errorf("tun queue %d is attached to %q, want %q", i+1, got, name)
+		}
+	}
+
+	// Offload is negotiated once for the whole device, on queue 0.
+	if err := tun.initFromFlags(name); err != nil {
+		tun.Close()
 		return nil, "", err
 	}
-	return tun, name, err
+	return tun, name, nil
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_openbsd.go

@@ -101,7 +101,7 @@ func (tun *NativeTun) routineRouteListener(tunIfindex int) {
 	}
 }
 
-func CreateTUN(name string, mtu int) (Device, error) {
+func CreateTUN(name string, mtu int, _ ...Option) (Device, error) {
 	ifIndex := -1
 	if name != "tun" {
 		_, err := fmt.Sscanf(name, "tun%d", &ifIndex)
@@ -140,6 +140,10 @@ func CreateTUN(name string, mtu int) (Device, error) {
 	return tun, err
 }
 
+func CreateTUNFromFiles(files []*os.File, mtu int) (Device, error) {
+	return createTUNFromFilesNoMQ(files, mtu)
+}
+
 func CreateTUNFromFile(file *os.File, mtu int) (Device, error) {
 	tun := &NativeTun{
 		tunFile: file,
@@ -204,17 +208,18 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		buf := bufs[0][offset-4:]
+		buf := slab[ReadPacketSpacing-4 : len(slab)-ReadPacketSpacing]
 		n, err := tun.tunFile.Read(buf[:])
 		if n < 4 {
 			return 0, err
 		}
-		sizes[0] = n - 4
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n - 4
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_plan9.go

@@ -23,7 +23,7 @@ type NativeTun struct {
 	closeOnce sync.Once
 }
 
-func CreateTUN(_ string, mtu int) (Device, error) {
+func CreateTUN(_ string, mtu int, _ ...Option) (Device, error) {
 	ctl, err := os.OpenFile("/net/ipifc/clone", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -81,18 +81,18 @@ func (tun *NativeTun) Events() <-chan Event {
 	return tun.events
 }
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	select {
 	case err := <-tun.errors:
 		return 0, err
 	default:
-		n, err := tun.dataFile.Read(bufs[0][offset:])
-		if n == 1 && bufs[0][offset] == 0 {
+		n, err := tun.dataFile.Read(slab[ReadPacketSpacing : len(slab)-ReadPacketSpacing])
+		if n == 1 && slab[ReadPacketSpacing] == 0 {
 			// EOF
-			err = io.EOF
-			n = 0
+			return 0, io.EOF
 		}
-		sizes[0] = n
+		packets[0].Offset = ReadPacketSpacing
+		packets[0].Size = n
 		return 1, err
 	}
 }

vendor/github.com/tailscale/wireguard-go/tun/tun_windows.go

@@ -59,7 +59,7 @@ func nanotime() int64
 
 // CreateTUN creates a Wintun interface with the given name. Should a Wintun
 // interface with the same name exist, it is reused.
-func CreateTUN(ifname string, mtu int) (Device, error) {
+func CreateTUN(ifname string, mtu int, _ ...Option) (Device, error) {
 	return CreateTUNWithRequestedGUID(ifname, WintunStaticRequestedGUID, mtu)
 }
 
@@ -144,7 +144,7 @@ func (tun *NativeTun) BatchSize() int {
 
 // Note: Read() and Write() assume the caller comes only from a single thread; there's no locking.
 
-func (tun *NativeTun) Read(bufs [][]byte, sizes []int, offset int) (int, error) {
+func (tun *NativeTun) Read(slab []byte, packets []ReadPacket) (int, error) {
 	tun.running.Add(1)
 	defer tun.running.Done()
 retry:
@@ -160,11 +160,16 @@ retry:
 		packet, err := tun.session.ReceivePacket()
 		switch err {
 		case nil:
-			packetSize := len(packet)
-			copy(bufs[0][offset:], packet)
-			sizes[0] = packetSize
+			dst := slab[ReadPacketSpacing : len(slab)-ReadPacketSpacing]
+			if len(packet) > len(dst) {
+				tun.session.ReleaseReceivePacket(packet)
+				return 0, ErrTooManySegments
+			}
+			n := copy(dst, packet)
+			packets[0].Offset = ReadPacketSpacing
+			packets[0].Size = n
 			tun.session.ReleaseReceivePacket(packet)
-			tun.rate.update(uint64(packetSize))
+			tun.rate.update(uint64(n))
 			return 1, nil
 		case windows.ERROR_NO_MORE_ITEMS:
 			if !shouldSpin || uint64(nanotime()-start) >= spinloopDuration {