Update
This commit is contained in:
1 parent
3530afb854
commit
6903fcff27
2146 files changed
+112256
-190703
No files matched your search
+98
-81
@@ -9,6 +9,7 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/netip"
|
||||
"runtime"
|
||||
@@ -202,9 +203,17 @@ again:
|
||||
return fns, uint16(port), nil
|
||||
}
|
||||
|
||||
func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) {
|
||||
for i := range *msgs {
|
||||
(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
|
||||
// putMessages resets each message in (*msgs)[:usedMsgs] for reuse, then returns
|
||||
// msgs to its [sync.Pool].
|
||||
func (s *StdNetBind) putMessages(msgs *[]ipv6.Message, usedMsgs int) {
|
||||
for i := range (*msgs)[:usedMsgs] {
|
||||
// Clear references to previously used packet buffers, otherwise they
|
||||
// may never be GC'd.
|
||||
clear((*msgs)[i].Buffers)
|
||||
(*msgs)[i] = ipv6.Message{
|
||||
Buffers: (*msgs)[i].Buffers[:1], // Non-coalesced write paths require a single element.
|
||||
OOB: (*msgs)[i].OOB,
|
||||
}
|
||||
}
|
||||
s.msgsPool.Put(msgs)
|
||||
}
|
||||
@@ -226,69 +235,62 @@ type batchWriter interface {
|
||||
WriteBatch([]ipv6.Message, int) (int, error)
|
||||
}
|
||||
|
||||
const maxDatagramSize = 1<<16 - 1
|
||||
|
||||
func (s *StdNetBind) receiveIP(
|
||||
br batchReader,
|
||||
conn *net.UDPConn,
|
||||
rxOffload bool,
|
||||
bufs [][]byte,
|
||||
sizes []int,
|
||||
eps []Endpoint,
|
||||
slab []byte,
|
||||
packets []ReceivedPacket,
|
||||
) (n int, err error) {
|
||||
msgs := s.getMessages()
|
||||
for i := range bufs {
|
||||
(*msgs)[i].Buffers[0] = bufs[i]
|
||||
(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
|
||||
usedMsgs := 1
|
||||
if runtime.GOOS == "linux" {
|
||||
// set a floor of 1 in case len(slab) < maxDatagramSize
|
||||
usedMsgs = max(1, len(slab)/maxDatagramSize)
|
||||
// we can't read more datagrams than what we can describe in packets
|
||||
usedMsgs = min(len(packets), usedMsgs)
|
||||
}
|
||||
defer s.putMessages(msgs)
|
||||
defer s.putMessages(msgs, usedMsgs)
|
||||
var numMsgs int
|
||||
if runtime.GOOS == "linux" {
|
||||
if rxOffload {
|
||||
readAt := len(*msgs) - 2
|
||||
numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
numMsgs, err = splitCoalescedMessages(*msgs, readAt, getGSOSize)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
numMsgs, err = br.ReadBatch(*msgs, 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
rem := slab
|
||||
for i := range usedMsgs {
|
||||
end := maxDatagramSize
|
||||
if len(rem) < maxDatagramSize {
|
||||
end = len(rem)
|
||||
}
|
||||
(*msgs)[i].Buffers[0] = rem[:end]
|
||||
(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
|
||||
rem = rem[end:]
|
||||
}
|
||||
numMsgs, err = br.ReadBatch((*msgs)[:usedMsgs], 0)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
msg := &(*msgs)[0]
|
||||
msg.Buffers[0] = slab
|
||||
msg.OOB = msg.OOB[:cap(msg.OOB)]
|
||||
msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
numMsgs = 1
|
||||
}
|
||||
for i := 0; i < numMsgs; i++ {
|
||||
msg := &(*msgs)[i]
|
||||
sizes[i] = msg.N
|
||||
if sizes[i] == 0 {
|
||||
continue
|
||||
}
|
||||
addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
|
||||
ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
|
||||
getSrcFromControl(msg.OOB[:msg.NN], ep)
|
||||
eps[i] = ep
|
||||
}
|
||||
return numMsgs, nil
|
||||
return fillReceivedPackets((*msgs)[:numMsgs], maxDatagramSize, packets, rxOffload, getGSOSize)
|
||||
}
|
||||
|
||||
func (s *StdNetBind) makeReceiveIPv4(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc {
|
||||
return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) {
|
||||
return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps)
|
||||
return func(slab []byte, packets []ReceivedPacket) (n int, err error) {
|
||||
return s.receiveIP(pc, conn, rxOffload, slab, packets)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc {
|
||||
return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) {
|
||||
return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps)
|
||||
return func(slab []byte, packets []ReceivedPacket) (n int, err error) {
|
||||
return s.receiveIP(pc, conn, rxOffload, slab, packets)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -365,7 +367,8 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
|
||||
}
|
||||
|
||||
msgs := s.getMessages()
|
||||
defer s.putMessages(msgs)
|
||||
usedMsgs := len(bufs)
|
||||
defer s.putMessages(msgs, usedMsgs)
|
||||
ua := s.udpAddrPool.Get().(*net.UDPAddr)
|
||||
defer s.udpAddrPool.Put(ua)
|
||||
if is6 {
|
||||
@@ -445,17 +448,34 @@ const (
|
||||
maxIPv6PayloadLen = 1<<16 - 1 - 8
|
||||
|
||||
// This is a hard limit imposed by the kernel.
|
||||
// As long as we use one fragment per datagram, this also serves as a
|
||||
// limit for the number of fragments we can coalesce during scatter-gather writes.
|
||||
//
|
||||
// 64 is below the 1024 of IOV_MAX (Linux) or UIO_MAXIOV (BSD),
|
||||
// and the 256 of WSABUF_MAX_COUNT (Windows).
|
||||
//
|
||||
// (2026-04) If we begin shipping datagrams in more than one fragment,
|
||||
// an independent fragment count limit needs to be implemented.
|
||||
udpSegmentMaxDatagrams = 64
|
||||
)
|
||||
|
||||
type setGSOFunc func(control *[]byte, gsoSize uint16)
|
||||
|
||||
// coalesceMessages iterates 'buffs', setting and coalescing them in 'msgs'
|
||||
// where possible while maintaining datagram order.
|
||||
//
|
||||
// It aggregates message components as a list of buffers without copying,
|
||||
// and expects to be used only on Linux with scatter-gather writes via sendmmsg(2).
|
||||
//
|
||||
// All msgs[i].Buffers len must be one. Will panic if there is not enough msgs
|
||||
// to coalesce all buffs.
|
||||
func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offset int, msgs []ipv6.Message, setGSO setGSOFunc) int {
|
||||
var (
|
||||
base = -1 // index of msg we are currently coalescing into
|
||||
gsoSize int // segmentation size of msgs[base]
|
||||
dgramCnt int // number of dgrams coalesced into msgs[base]
|
||||
endBatch bool // tracking flag to start a new batch on next iteration of bufs
|
||||
base = -1 // index of msg we are currently coalescing into
|
||||
gsoSize int // segmentation size of msgs[base]
|
||||
dgramCnt int // number of dgrams coalesced into msgs[base]
|
||||
endBatch bool // tracking flag to start a new batch on next iteration of bufs
|
||||
coalescedLen int // bytes coalesced into msgs[base]
|
||||
)
|
||||
maxPayloadLen := maxIPv4PayloadLen
|
||||
if ep.DstIP().Is6() {
|
||||
@@ -465,18 +485,18 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
|
||||
buf = buf[offset:]
|
||||
if i > 0 {
|
||||
msgLen := len(buf)
|
||||
baseLenBefore := len(msgs[base].Buffers[0])
|
||||
freeBaseCap := cap(msgs[base].Buffers[0]) - baseLenBefore
|
||||
if msgLen+baseLenBefore <= maxPayloadLen &&
|
||||
if msgLen+coalescedLen <= maxPayloadLen &&
|
||||
msgLen <= gsoSize &&
|
||||
msgLen <= freeBaseCap &&
|
||||
dgramCnt < udpSegmentMaxDatagrams &&
|
||||
!endBatch {
|
||||
msgs[base].Buffers[0] = append(msgs[base].Buffers[0], buf...)
|
||||
// msgs[base].Buffers[0] is set to buf[i] when a new base is set.
|
||||
// This appends a struct iovec element in the underlying struct msghdr (scatter-gather).
|
||||
msgs[base].Buffers = append(msgs[base].Buffers, buf)
|
||||
if i == len(bufs)-1 {
|
||||
setGSO(&msgs[base].OOB, uint16(gsoSize))
|
||||
}
|
||||
dgramCnt++
|
||||
coalescedLen += msgLen
|
||||
if msgLen < gsoSize {
|
||||
// A smaller than gsoSize packet on the tail is legal, but
|
||||
// it must end the batch.
|
||||
@@ -497,51 +517,48 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
|
||||
msgs[base].Buffers[0] = buf
|
||||
msgs[base].Addr = addr
|
||||
dgramCnt = 1
|
||||
coalescedLen = len(buf)
|
||||
}
|
||||
return base + 1
|
||||
}
|
||||
|
||||
type getGSOFunc func(control []byte) (int, error)
|
||||
|
||||
func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFunc) (n int, err error) {
|
||||
for i := firstMsgAt; i < len(msgs); i++ {
|
||||
msg := &msgs[i]
|
||||
if msg.N == 0 {
|
||||
return n, err
|
||||
}
|
||||
func fillReceivedPackets(msgs []ipv6.Message, slabOffset int, packets []ReceivedPacket, rxOffload bool, getGSO getGSOFunc) (n int, err error) {
|
||||
for i, msg := range msgs {
|
||||
var (
|
||||
gsoSize int
|
||||
start int
|
||||
end = msg.N
|
||||
numToSplit = 1
|
||||
)
|
||||
gsoSize, err = getGSO(msg.OOB[:msg.NN])
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
if gsoSize > 0 {
|
||||
numToSplit = (msg.N + gsoSize - 1) / gsoSize
|
||||
end = gsoSize
|
||||
}
|
||||
for j := 0; j < numToSplit; j++ {
|
||||
if n > i {
|
||||
return n, errors.New("splitting coalesced packet resulted in overflow")
|
||||
if rxOffload {
|
||||
gsoSize, err = getGSO(msg.OOB[:msg.NN])
|
||||
if err != nil {
|
||||
return n, err
|
||||
}
|
||||
copied := copy(msgs[n].Buffers[0], msg.Buffers[0][start:end])
|
||||
msgs[n].N = copied
|
||||
msgs[n].Addr = msg.Addr
|
||||
start = end
|
||||
end += gsoSize
|
||||
if end > msg.N {
|
||||
end = msg.N
|
||||
if gsoSize > 0 {
|
||||
numToSplit = (msg.N + gsoSize - 1) / gsoSize
|
||||
}
|
||||
}
|
||||
addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
|
||||
ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
|
||||
getSrcFromControl(msg.OOB[:msg.NN], ep)
|
||||
regionOffset := i * slabOffset // region may contain multiple coalesced packets
|
||||
for j := 0; j < numToSplit; j++ {
|
||||
if n >= len(packets) {
|
||||
return n, fmt.Errorf("%w: filling received packet metadata resulted in overflow", io.ErrShortBuffer)
|
||||
}
|
||||
end := msg.N
|
||||
if gsoSize > 0 && start+gsoSize < end {
|
||||
end = start + gsoSize
|
||||
}
|
||||
packets[n] = ReceivedPacket{
|
||||
Offset: regionOffset + start,
|
||||
Size: end - start,
|
||||
Endpoint: ep,
|
||||
}
|
||||
n++
|
||||
}
|
||||
if i != n-1 {
|
||||
// It is legal for bytes to move within msg.Buffers[0] as a result
|
||||
// of splitting, so we only zero the source msg len when it is not
|
||||
// the destination of the last split operation above.
|
||||
msg.N = 0
|
||||
start = end
|
||||
}
|
||||
}
|
||||
return n, nil
|
||||
|
||||
+12
-8
@@ -416,21 +416,25 @@ retry:
|
||||
return n, &ep, nil
|
||||
}
|
||||
|
||||
func (bind *WinRingBind) receiveIPv4(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
|
||||
func (bind *WinRingBind) receiveIPv4(slab []byte, packets []ReceivedPacket) (int, error) {
|
||||
bind.mu.RLock()
|
||||
defer bind.mu.RUnlock()
|
||||
n, ep, err := bind.v4.Receive(bufs[0], &bind.isOpen)
|
||||
sizes[0] = n
|
||||
eps[0] = ep
|
||||
n, ep, err := bind.v4.Receive(slab, &bind.isOpen)
|
||||
packets[0] = ReceivedPacket{
|
||||
Size: n,
|
||||
Endpoint: ep,
|
||||
}
|
||||
return 1, err
|
||||
}
|
||||
|
||||
func (bind *WinRingBind) receiveIPv6(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
|
||||
func (bind *WinRingBind) receiveIPv6(slab []byte, packets []ReceivedPacket) (int, error) {
|
||||
bind.mu.RLock()
|
||||
defer bind.mu.RUnlock()
|
||||
n, ep, err := bind.v6.Receive(bufs[0], &bind.isOpen)
|
||||
sizes[0] = n
|
||||
eps[0] = ep
|
||||
n, ep, err := bind.v6.Receive(slab, &bind.isOpen)
|
||||
packets[0] = ReceivedPacket{
|
||||
Size: n,
|
||||
Endpoint: ep,
|
||||
}
|
||||
return 1, err
|
||||
}
|
||||
|
||||
|
||||
+18
-6
@@ -19,13 +19,25 @@ const (
|
||||
IdealBatchSize = 128 // maximum number of packets handled per read and write
|
||||
)
|
||||
|
||||
// ReceivedPacket describes a packet read by a [ReceiveFunc].
|
||||
type ReceivedPacket struct {
|
||||
// Offset is the starting byte offset.
|
||||
Offset int
|
||||
// Size is the size of the packet.
|
||||
Size int
|
||||
// Endpoint is the associated [Endpoint].
|
||||
Endpoint Endpoint
|
||||
}
|
||||
|
||||
func (r *ReceivedPacket) Bytes(slab []byte) []byte {
|
||||
return slab[r.Offset : r.Offset+r.Size : r.Offset+r.Size]
|
||||
}
|
||||
|
||||
// A ReceiveFunc receives at least one packet from the network and writes them
|
||||
// into packets. On a successful read it returns the number of elements of
|
||||
// sizes, packets, and endpoints that should be evaluated. Some elements of
|
||||
// sizes may be zero, and callers should ignore them. Callers must pass a sizes
|
||||
// and eps slice with a length greater than or equal to the length of packets.
|
||||
// These lengths must not exceed the length of the associated Bind.BatchSize().
|
||||
type ReceiveFunc func(packets [][]byte, sizes []int, eps []Endpoint) (n int, err error)
|
||||
// into slab. On a successful read it returns the number of elements of
|
||||
// packets that should be evaluated. Callers must pass a length of packets equal
|
||||
// to the associated [Bind.BatchSize].
|
||||
type ReceiveFunc func(slab []byte, packets []ReceivedPacket) (n int, err error)
|
||||
|
||||
// A Bind listens on a port for both IPv6 and IPv4 UDP traffic.
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user