This commit is contained in:
bluepython508 committed 2026-10-22 12:27:19 +01:00
1 parent 3530afb854
commit 6903fcff27
2146 files changed
+112256 -190703

No files matched your search

+98 -81
View File
@@ -9,6 +9,7 @@ import (
"context"
"errors"
"fmt"
"io"
"net"
"net/netip"
"runtime"
@@ -202,9 +203,17 @@ again:
return fns, uint16(port), nil
}
func (s *StdNetBind) putMessages(msgs *[]ipv6.Message) {
for i := range *msgs {
(*msgs)[i] = ipv6.Message{Buffers: (*msgs)[i].Buffers, OOB: (*msgs)[i].OOB}
// putMessages resets each message in (*msgs)[:usedMsgs] for reuse, then returns
// msgs to its [sync.Pool].
func (s *StdNetBind) putMessages(msgs *[]ipv6.Message, usedMsgs int) {
for i := range (*msgs)[:usedMsgs] {
// Clear references to previously used packet buffers, otherwise they
// may never be GC'd.
clear((*msgs)[i].Buffers)
(*msgs)[i] = ipv6.Message{
Buffers: (*msgs)[i].Buffers[:1], // Non-coalesced write paths require a single element.
OOB: (*msgs)[i].OOB,
}
}
s.msgsPool.Put(msgs)
}
@@ -226,69 +235,62 @@ type batchWriter interface {
WriteBatch([]ipv6.Message, int) (int, error)
}
const maxDatagramSize = 1<<16 - 1
func (s *StdNetBind) receiveIP(
br batchReader,
conn *net.UDPConn,
rxOffload bool,
bufs [][]byte,
sizes []int,
eps []Endpoint,
slab []byte,
packets []ReceivedPacket,
) (n int, err error) {
msgs := s.getMessages()
for i := range bufs {
(*msgs)[i].Buffers[0] = bufs[i]
(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
usedMsgs := 1
if runtime.GOOS == "linux" {
// set a floor of 1 in case len(slab) < maxDatagramSize
usedMsgs = max(1, len(slab)/maxDatagramSize)
// we can't read more datagrams than what we can describe in packets
usedMsgs = min(len(packets), usedMsgs)
}
defer s.putMessages(msgs)
defer s.putMessages(msgs, usedMsgs)
var numMsgs int
if runtime.GOOS == "linux" {
if rxOffload {
readAt := len(*msgs) - 2
numMsgs, err = br.ReadBatch((*msgs)[readAt:], 0)
if err != nil {
return 0, err
}
numMsgs, err = splitCoalescedMessages(*msgs, readAt, getGSOSize)
if err != nil {
return 0, err
}
} else {
numMsgs, err = br.ReadBatch(*msgs, 0)
if err != nil {
return 0, err
rem := slab
for i := range usedMsgs {
end := maxDatagramSize
if len(rem) < maxDatagramSize {
end = len(rem)
}
(*msgs)[i].Buffers[0] = rem[:end]
(*msgs)[i].OOB = (*msgs)[i].OOB[:cap((*msgs)[i].OOB)]
rem = rem[end:]
}
numMsgs, err = br.ReadBatch((*msgs)[:usedMsgs], 0)
if err != nil {
return 0, err
}
} else {
msg := &(*msgs)[0]
msg.Buffers[0] = slab
msg.OOB = msg.OOB[:cap(msg.OOB)]
msg.N, msg.NN, _, msg.Addr, err = conn.ReadMsgUDP(msg.Buffers[0], msg.OOB)
if err != nil {
return 0, err
}
numMsgs = 1
}
for i := 0; i < numMsgs; i++ {
msg := &(*msgs)[i]
sizes[i] = msg.N
if sizes[i] == 0 {
continue
}
addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
getSrcFromControl(msg.OOB[:msg.NN], ep)
eps[i] = ep
}
return numMsgs, nil
return fillReceivedPackets((*msgs)[:numMsgs], maxDatagramSize, packets, rxOffload, getGSOSize)
}
func (s *StdNetBind) makeReceiveIPv4(pc *ipv4.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc {
return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) {
return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps)
return func(slab []byte, packets []ReceivedPacket) (n int, err error) {
return s.receiveIP(pc, conn, rxOffload, slab, packets)
}
}
func (s *StdNetBind) makeReceiveIPv6(pc *ipv6.PacketConn, conn *net.UDPConn, rxOffload bool) ReceiveFunc {
return func(bufs [][]byte, sizes []int, eps []Endpoint) (n int, err error) {
return s.receiveIP(pc, conn, rxOffload, bufs, sizes, eps)
return func(slab []byte, packets []ReceivedPacket) (n int, err error) {
return s.receiveIP(pc, conn, rxOffload, slab, packets)
}
}
@@ -365,7 +367,8 @@ func (s *StdNetBind) Send(bufs [][]byte, endpoint Endpoint, offset int) error {
}
msgs := s.getMessages()
defer s.putMessages(msgs)
usedMsgs := len(bufs)
defer s.putMessages(msgs, usedMsgs)
ua := s.udpAddrPool.Get().(*net.UDPAddr)
defer s.udpAddrPool.Put(ua)
if is6 {
@@ -445,17 +448,34 @@ const (
maxIPv6PayloadLen = 1<<16 - 1 - 8
// This is a hard limit imposed by the kernel.
// As long as we use one fragment per datagram, this also serves as a
// limit for the number of fragments we can coalesce during scatter-gather writes.
//
// 64 is below the 1024 of IOV_MAX (Linux) or UIO_MAXIOV (BSD),
// and the 256 of WSABUF_MAX_COUNT (Windows).
//
// (2026-04) If we begin shipping datagrams in more than one fragment,
// an independent fragment count limit needs to be implemented.
udpSegmentMaxDatagrams = 64
)
type setGSOFunc func(control *[]byte, gsoSize uint16)
// coalesceMessages iterates 'buffs', setting and coalescing them in 'msgs'
// where possible while maintaining datagram order.
//
// It aggregates message components as a list of buffers without copying,
// and expects to be used only on Linux with scatter-gather writes via sendmmsg(2).
//
// All msgs[i].Buffers len must be one. Will panic if there is not enough msgs
// to coalesce all buffs.
func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offset int, msgs []ipv6.Message, setGSO setGSOFunc) int {
var (
base = -1 // index of msg we are currently coalescing into
gsoSize int // segmentation size of msgs[base]
dgramCnt int // number of dgrams coalesced into msgs[base]
endBatch bool // tracking flag to start a new batch on next iteration of bufs
base = -1 // index of msg we are currently coalescing into
gsoSize int // segmentation size of msgs[base]
dgramCnt int // number of dgrams coalesced into msgs[base]
endBatch bool // tracking flag to start a new batch on next iteration of bufs
coalescedLen int // bytes coalesced into msgs[base]
)
maxPayloadLen := maxIPv4PayloadLen
if ep.DstIP().Is6() {
@@ -465,18 +485,18 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
buf = buf[offset:]
if i > 0 {
msgLen := len(buf)
baseLenBefore := len(msgs[base].Buffers[0])
freeBaseCap := cap(msgs[base].Buffers[0]) - baseLenBefore
if msgLen+baseLenBefore <= maxPayloadLen &&
if msgLen+coalescedLen <= maxPayloadLen &&
msgLen <= gsoSize &&
msgLen <= freeBaseCap &&
dgramCnt < udpSegmentMaxDatagrams &&
!endBatch {
msgs[base].Buffers[0] = append(msgs[base].Buffers[0], buf...)
// msgs[base].Buffers[0] is set to buf[i] when a new base is set.
// This appends a struct iovec element in the underlying struct msghdr (scatter-gather).
msgs[base].Buffers = append(msgs[base].Buffers, buf)
if i == len(bufs)-1 {
setGSO(&msgs[base].OOB, uint16(gsoSize))
}
dgramCnt++
coalescedLen += msgLen
if msgLen < gsoSize {
// A smaller than gsoSize packet on the tail is legal, but
// it must end the batch.
@@ -497,51 +517,48 @@ func coalesceMessages(addr *net.UDPAddr, ep *StdNetEndpoint, bufs [][]byte, offs
msgs[base].Buffers[0] = buf
msgs[base].Addr = addr
dgramCnt = 1
coalescedLen = len(buf)
}
return base + 1
}
type getGSOFunc func(control []byte) (int, error)
func splitCoalescedMessages(msgs []ipv6.Message, firstMsgAt int, getGSO getGSOFunc) (n int, err error) {
for i := firstMsgAt; i < len(msgs); i++ {
msg := &msgs[i]
if msg.N == 0 {
return n, err
}
func fillReceivedPackets(msgs []ipv6.Message, slabOffset int, packets []ReceivedPacket, rxOffload bool, getGSO getGSOFunc) (n int, err error) {
for i, msg := range msgs {
var (
gsoSize int
start int
end = msg.N
numToSplit = 1
)
gsoSize, err = getGSO(msg.OOB[:msg.NN])
if err != nil {
return n, err
}
if gsoSize > 0 {
numToSplit = (msg.N + gsoSize - 1) / gsoSize
end = gsoSize
}
for j := 0; j < numToSplit; j++ {
if n > i {
return n, errors.New("splitting coalesced packet resulted in overflow")
if rxOffload {
gsoSize, err = getGSO(msg.OOB[:msg.NN])
if err != nil {
return n, err
}
copied := copy(msgs[n].Buffers[0], msg.Buffers[0][start:end])
msgs[n].N = copied
msgs[n].Addr = msg.Addr
start = end
end += gsoSize
if end > msg.N {
end = msg.N
if gsoSize > 0 {
numToSplit = (msg.N + gsoSize - 1) / gsoSize
}
}
addrPort := msg.Addr.(*net.UDPAddr).AddrPort()
ep := &StdNetEndpoint{AddrPort: addrPort} // TODO: remove allocation
getSrcFromControl(msg.OOB[:msg.NN], ep)
regionOffset := i * slabOffset // region may contain multiple coalesced packets
for j := 0; j < numToSplit; j++ {
if n >= len(packets) {
return n, fmt.Errorf("%w: filling received packet metadata resulted in overflow", io.ErrShortBuffer)
}
end := msg.N
if gsoSize > 0 && start+gsoSize < end {
end = start + gsoSize
}
packets[n] = ReceivedPacket{
Offset: regionOffset + start,
Size: end - start,
Endpoint: ep,
}
n++
}
if i != n-1 {
// It is legal for bytes to move within msg.Buffers[0] as a result
// of splitting, so we only zero the source msg len when it is not
// the destination of the last split operation above.
msg.N = 0
start = end
}
}
return n, nil
+12 -8
View File
@@ -416,21 +416,25 @@ retry:
return n, &ep, nil
}
func (bind *WinRingBind) receiveIPv4(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
func (bind *WinRingBind) receiveIPv4(slab []byte, packets []ReceivedPacket) (int, error) {
bind.mu.RLock()
defer bind.mu.RUnlock()
n, ep, err := bind.v4.Receive(bufs[0], &bind.isOpen)
sizes[0] = n
eps[0] = ep
n, ep, err := bind.v4.Receive(slab, &bind.isOpen)
packets[0] = ReceivedPacket{
Size: n,
Endpoint: ep,
}
return 1, err
}
func (bind *WinRingBind) receiveIPv6(bufs [][]byte, sizes []int, eps []Endpoint) (int, error) {
func (bind *WinRingBind) receiveIPv6(slab []byte, packets []ReceivedPacket) (int, error) {
bind.mu.RLock()
defer bind.mu.RUnlock()
n, ep, err := bind.v6.Receive(bufs[0], &bind.isOpen)
sizes[0] = n
eps[0] = ep
n, ep, err := bind.v6.Receive(slab, &bind.isOpen)
packets[0] = ReceivedPacket{
Size: n,
Endpoint: ep,
}
return 1, err
}
+18 -6
View File
@@ -19,13 +19,25 @@ const (
IdealBatchSize = 128 // maximum number of packets handled per read and write
)
// ReceivedPacket describes a packet read by a [ReceiveFunc].
type ReceivedPacket struct {
// Offset is the starting byte offset.
Offset int
// Size is the size of the packet.
Size int
// Endpoint is the associated [Endpoint].
Endpoint Endpoint
}
func (r *ReceivedPacket) Bytes(slab []byte) []byte {
return slab[r.Offset : r.Offset+r.Size : r.Offset+r.Size]
}
// A ReceiveFunc receives at least one packet from the network and writes them
// into packets. On a successful read it returns the number of elements of
// sizes, packets, and endpoints that should be evaluated. Some elements of
// sizes may be zero, and callers should ignore them. Callers must pass a sizes
// and eps slice with a length greater than or equal to the length of packets.
// These lengths must not exceed the length of the associated Bind.BatchSize().
type ReceiveFunc func(packets [][]byte, sizes []int, eps []Endpoint) (n int, err error)
// into slab. On a successful read it returns the number of elements of
// packets that should be evaluated. Callers must pass a length of packets equal
// to the associated [Bind.BatchSize].
type ReceiveFunc func(slab []byte, packets []ReceivedPacket) (n int, err error)
// A Bind listens on a port for both IPv6 and IPv4 UDP traffic.
//