Update dependencies

This commit is contained in:
bluepython508
2025-04-09 01:00:12 +01:00
parent f0641ffd6e
commit 5a9cfc022c
882 changed files with 68930 additions and 24201 deletions

View File

@@ -23,9 +23,9 @@ import (
"math"
"math/big"
"math/rand/v2"
"net"
"net/http"
"net/netip"
"os"
"os/exec"
"runtime"
"strconv"
@@ -36,6 +36,7 @@ import (
"go4.org/mem"
"golang.org/x/sync/errgroup"
"tailscale.com/client/local"
"tailscale.com/client/tailscale"
"tailscale.com/disco"
"tailscale.com/envknob"
@@ -46,6 +47,7 @@ import (
"tailscale.com/tstime/rate"
"tailscale.com/types/key"
"tailscale.com/types/logger"
"tailscale.com/util/ctxkey"
"tailscale.com/util/mak"
"tailscale.com/util/set"
"tailscale.com/util/slicesx"
@@ -56,6 +58,16 @@ import (
// verbosely log whenever DERP drops a packet.
var verboseDropKeys = map[key.NodePublic]bool{}
// IdealNodeHeader is the HTTP request header sent on DERP HTTP client requests
// to indicate that they're connecting to their ideal (Region.Nodes[0]) node.
// The HTTP header value is the name of the node they wish they were connected
// to. This is an optional header.
const IdealNodeHeader = "Ideal-Node"
// IdealNodeContextKey is the context key used to pass the IdealNodeHeader value
// from the HTTP handler to the DERP server's Accept method.
var IdealNodeContextKey = ctxkey.New[string]("ideal-node", "")
func init() {
keys := envknob.String("TS_DEBUG_VERBOSE_DROPS")
if keys == "" {
@@ -72,10 +84,19 @@ func init() {
}
const (
perClientSendQueueDepth = 32 // packets buffered for sending
writeTimeout = 2 * time.Second
defaultPerClientSendQueueDepth = 32 // default packets buffered for sending
DefaultTCPWiteTimeout = 2 * time.Second
privilegedWriteTimeout = 30 * time.Second // for clients with the mesh key
)
func getPerClientSendQueueDepth() int {
if v, ok := envknob.LookupInt("TS_DEBUG_DERP_PER_CLIENT_SEND_QUEUE_DEPTH"); ok {
return v
}
return defaultPerClientSendQueueDepth
}
// dupPolicy is a temporary (2021-08-30) mechanism to change the policy
// of how duplicate connection for the same key are handled.
type dupPolicy int8
@@ -91,6 +112,14 @@ const (
disableFighters
)
// packetKind is the kind of packet being sent through DERP
type packetKind string
const (
packetKindDisco packetKind = "disco"
packetKindOther packetKind = "other"
)
type align64 [0]atomic.Int64 // for side effect of its 64-bit alignment
// Server is a DERP server.
@@ -108,44 +137,40 @@ type Server struct {
metaCert []byte // the encoded x509 cert to send after LetsEncrypt cert+intermediate
dupPolicy dupPolicy
debug bool
localClient local.Client
// Counters:
packetsSent, bytesSent expvar.Int
packetsRecv, bytesRecv expvar.Int
packetsRecvByKind metrics.LabelMap
packetsRecvDisco *expvar.Int
packetsRecvOther *expvar.Int
_ align64
packetsDropped expvar.Int
packetsDroppedReason metrics.LabelMap
packetsDroppedReasonCounters []*expvar.Int // indexed by dropReason
packetsDroppedType metrics.LabelMap
packetsDroppedTypeDisco *expvar.Int
packetsDroppedTypeOther *expvar.Int
_ align64
packetsForwardedOut expvar.Int
packetsForwardedIn expvar.Int
peerGoneDisconnectedFrames expvar.Int // number of peer disconnected frames sent
peerGoneNotHereFrames expvar.Int // number of peer not here frames sent
gotPing expvar.Int // number of ping frames from client
sentPong expvar.Int // number of pong frames enqueued to client
accepts expvar.Int
curClients expvar.Int
curHomeClients expvar.Int // ones with preferred
dupClientKeys expvar.Int // current number of public keys we have 2+ connections for
dupClientConns expvar.Int // current number of connections sharing a public key
dupClientConnTotal expvar.Int // total number of accepted connections when a dup key existed
unknownFrames expvar.Int
homeMovesIn expvar.Int // established clients announce home server moves in
homeMovesOut expvar.Int // established clients announce home server moves out
multiForwarderCreated expvar.Int
multiForwarderDeleted expvar.Int
removePktForwardOther expvar.Int
avgQueueDuration *uint64 // In milliseconds; accessed atomically
tcpRtt metrics.LabelMap // histogram
meshUpdateBatchSize *metrics.Histogram
meshUpdateLoopCount *metrics.Histogram
bufferedWriteFrames *metrics.Histogram // how many sendLoop frames (or groups of related frames) get written per flush
packetsSent, bytesSent expvar.Int
packetsRecv, bytesRecv expvar.Int
packetsRecvByKind metrics.LabelMap
packetsRecvDisco *expvar.Int
packetsRecvOther *expvar.Int
_ align64
packetsForwardedOut expvar.Int
packetsForwardedIn expvar.Int
peerGoneDisconnectedFrames expvar.Int // number of peer disconnected frames sent
peerGoneNotHereFrames expvar.Int // number of peer not here frames sent
gotPing expvar.Int // number of ping frames from client
sentPong expvar.Int // number of pong frames enqueued to client
accepts expvar.Int
curClients expvar.Int
curClientsNotIdeal expvar.Int
curHomeClients expvar.Int // ones with preferred
dupClientKeys expvar.Int // current number of public keys we have 2+ connections for
dupClientConns expvar.Int // current number of connections sharing a public key
dupClientConnTotal expvar.Int // total number of accepted connections when a dup key existed
unknownFrames expvar.Int
homeMovesIn expvar.Int // established clients announce home server moves in
homeMovesOut expvar.Int // established clients announce home server moves out
multiForwarderCreated expvar.Int
multiForwarderDeleted expvar.Int
removePktForwardOther expvar.Int
sclientWriteTimeouts expvar.Int
avgQueueDuration *uint64 // In milliseconds; accessed atomically
tcpRtt metrics.LabelMap // histogram
meshUpdateBatchSize *metrics.Histogram
meshUpdateLoopCount *metrics.Histogram
bufferedWriteFrames *metrics.Histogram // how many sendLoop frames (or groups of related frames) get written per flush
// verifyClientsLocalTailscaled only accepts client connections to the DERP
// server if the clientKey is a known peer in the network, as specified by a
@@ -175,6 +200,11 @@ type Server struct {
// maps from netip.AddrPort to a client's public key
keyOfAddr map[netip.AddrPort]key.NodePublic
// Sets the client send queue depth for the server.
perClientSendQueueDepth int
tcpWriteTimeout time.Duration
clock tstime.Clock
}
@@ -314,16 +344,16 @@ type PacketForwarder interface {
String() string
}
// Conn is the subset of the underlying net.Conn the DERP Server needs.
// It is a defined type so that non-net connections can be used.
type Conn interface {
io.WriteCloser
LocalAddr() net.Addr
// The *Deadline methods follow the semantics of net.Conn.
SetDeadline(time.Time) error
SetReadDeadline(time.Time) error
SetWriteDeadline(time.Time) error
}
var packetsDropped = metrics.NewMultiLabelMap[dropReasonKindLabels](
"derp_packets_dropped",
"counter",
"DERP packets dropped by reason and by kind")
var bytesDropped = metrics.NewMultiLabelMap[dropReasonKindLabels](
"derp_bytes_dropped",
"counter",
"DERP bytes dropped by reason and by kind",
)
// NewServer returns a new DERP server. It doesn't listen on its own.
// Connections are given to it via Server.Accept.
@@ -332,59 +362,100 @@ func NewServer(privateKey key.NodePrivate, logf logger.Logf) *Server {
runtime.ReadMemStats(&ms)
s := &Server{
debug: envknob.Bool("DERP_DEBUG_LOGS"),
privateKey: privateKey,
publicKey: privateKey.Public(),
logf: logf,
limitedLogf: logger.RateLimitedFn(logf, 30*time.Second, 5, 100),
packetsRecvByKind: metrics.LabelMap{Label: "kind"},
packetsDroppedReason: metrics.LabelMap{Label: "reason"},
packetsDroppedType: metrics.LabelMap{Label: "type"},
clients: map[key.NodePublic]*clientSet{},
clientsMesh: map[key.NodePublic]PacketForwarder{},
netConns: map[Conn]chan struct{}{},
memSys0: ms.Sys,
watchers: set.Set[*sclient]{},
peerGoneWatchers: map[key.NodePublic]set.HandleSet[func(key.NodePublic)]{},
avgQueueDuration: new(uint64),
tcpRtt: metrics.LabelMap{Label: "le"},
meshUpdateBatchSize: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000}),
meshUpdateLoopCount: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100}),
bufferedWriteFrames: metrics.NewHistogram([]float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 100}),
keyOfAddr: map[netip.AddrPort]key.NodePublic{},
clock: tstime.StdClock{},
debug: envknob.Bool("DERP_DEBUG_LOGS"),
privateKey: privateKey,
publicKey: privateKey.Public(),
logf: logf,
limitedLogf: logger.RateLimitedFn(logf, 30*time.Second, 5, 100),
packetsRecvByKind: metrics.LabelMap{Label: "kind"},
clients: map[key.NodePublic]*clientSet{},
clientsMesh: map[key.NodePublic]PacketForwarder{},
netConns: map[Conn]chan struct{}{},
memSys0: ms.Sys,
watchers: set.Set[*sclient]{},
peerGoneWatchers: map[key.NodePublic]set.HandleSet[func(key.NodePublic)]{},
avgQueueDuration: new(uint64),
tcpRtt: metrics.LabelMap{Label: "le"},
meshUpdateBatchSize: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1000}),
meshUpdateLoopCount: metrics.NewHistogram([]float64{0, 1, 2, 5, 10, 20, 50, 100}),
bufferedWriteFrames: metrics.NewHistogram([]float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25, 50, 100}),
keyOfAddr: map[netip.AddrPort]key.NodePublic{},
clock: tstime.StdClock{},
tcpWriteTimeout: DefaultTCPWiteTimeout,
}
s.initMetacert()
s.packetsRecvDisco = s.packetsRecvByKind.Get("disco")
s.packetsRecvOther = s.packetsRecvByKind.Get("other")
s.packetsRecvDisco = s.packetsRecvByKind.Get(string(packetKindDisco))
s.packetsRecvOther = s.packetsRecvByKind.Get(string(packetKindOther))
s.packetsDroppedReasonCounters = s.genPacketsDroppedReasonCounters()
genDroppedCounters()
s.packetsDroppedTypeDisco = s.packetsDroppedType.Get("disco")
s.packetsDroppedTypeOther = s.packetsDroppedType.Get("other")
s.perClientSendQueueDepth = getPerClientSendQueueDepth()
return s
}
func (s *Server) genPacketsDroppedReasonCounters() []*expvar.Int {
getMetric := s.packetsDroppedReason.Get
ret := []*expvar.Int{
dropReasonUnknownDest: getMetric("unknown_dest"),
dropReasonUnknownDestOnFwd: getMetric("unknown_dest_on_fwd"),
dropReasonGoneDisconnected: getMetric("gone_disconnected"),
dropReasonQueueHead: getMetric("queue_head"),
dropReasonQueueTail: getMetric("queue_tail"),
dropReasonWriteError: getMetric("write_error"),
dropReasonDupClient: getMetric("dup_client"),
func genDroppedCounters() {
initMetrics := func(reason dropReason) {
packetsDropped.Add(dropReasonKindLabels{
Kind: string(packetKindDisco),
Reason: string(reason),
}, 0)
packetsDropped.Add(dropReasonKindLabels{
Kind: string(packetKindOther),
Reason: string(reason),
}, 0)
bytesDropped.Add(dropReasonKindLabels{
Kind: string(packetKindDisco),
Reason: string(reason),
}, 0)
bytesDropped.Add(dropReasonKindLabels{
Kind: string(packetKindOther),
Reason: string(reason),
}, 0)
}
if len(ret) != int(numDropReasons) {
panic("dropReason metrics out of sync")
}
for i := range numDropReasons {
if ret[i] == nil {
panic("dropReason metrics out of sync")
getMetrics := func(reason dropReason) []expvar.Var {
return []expvar.Var{
packetsDropped.Get(dropReasonKindLabels{
Kind: string(packetKindDisco),
Reason: string(reason),
}),
packetsDropped.Get(dropReasonKindLabels{
Kind: string(packetKindOther),
Reason: string(reason),
}),
bytesDropped.Get(dropReasonKindLabels{
Kind: string(packetKindDisco),
Reason: string(reason),
}),
bytesDropped.Get(dropReasonKindLabels{
Kind: string(packetKindOther),
Reason: string(reason),
}),
}
}
dropReasons := []dropReason{
dropReasonUnknownDest,
dropReasonUnknownDestOnFwd,
dropReasonGoneDisconnected,
dropReasonQueueHead,
dropReasonQueueTail,
dropReasonWriteError,
dropReasonDupClient,
}
for _, dr := range dropReasons {
initMetrics(dr)
m := getMetrics(dr)
if len(m) != 4 {
panic("dropReason metrics out of sync")
}
for _, v := range m {
if v == nil {
panic("dropReason metrics out of sync")
}
}
}
return ret
}
// SetMesh sets the pre-shared key that regional DERP servers used to mesh
@@ -415,6 +486,23 @@ func (s *Server) SetVerifyClientURLFailOpen(v bool) {
s.verifyClientsURLFailOpen = v
}
// SetTailscaledSocketPath sets the unix socket path to use to talk to
// tailscaled if client verification is enabled.
//
// If unset or set to the empty string, the default path for the operating
// system is used.
func (s *Server) SetTailscaledSocketPath(path string) {
s.localClient.Socket = path
s.localClient.UseSocketOnly = path != ""
}
// SetTCPWriteTimeout sets the timeout for writing to connected clients.
// This timeout does not apply to mesh connections.
// Defaults to 2 seconds.
func (s *Server) SetTCPWriteTimeout(d time.Duration) {
s.tcpWriteTimeout = d
}
// HasMeshKey reports whether the server is configured with a mesh key.
func (s *Server) HasMeshKey() bool { return s.meshKey != "" }
@@ -600,6 +688,9 @@ func (s *Server) registerClient(c *sclient) {
}
s.keyOfAddr[c.remoteIPPort] = c.key
s.curClients.Add(1)
if c.isNotIdealConn {
s.curClientsNotIdeal.Add(1)
}
s.broadcastPeerStateChangeLocked(c.key, c.remoteIPPort, c.presentFlags(), true)
}
@@ -690,6 +781,9 @@ func (s *Server) unregisterClient(c *sclient) {
if c.preferred {
s.curHomeClients.Add(-1)
}
if c.isNotIdealConn {
s.curClientsNotIdeal.Add(-1)
}
}
// addPeerGoneFromRegionWatcher adds a function to be called when peer is gone
@@ -806,8 +900,8 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
return fmt.Errorf("receive client key: %v", err)
}
clientAP, _ := netip.ParseAddrPort(remoteAddr)
if err := s.verifyClient(ctx, clientKey, clientInfo, clientAP.Addr()); err != nil {
remoteIPPort, _ := netip.ParseAddrPort(remoteAddr)
if err := s.verifyClient(ctx, clientKey, clientInfo, remoteIPPort.Addr()); err != nil {
return fmt.Errorf("client %v rejected: %v", clientKey, err)
}
@@ -817,8 +911,6 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
ctx, cancel := context.WithCancel(ctx)
defer cancel()
remoteIPPort, _ := netip.ParseAddrPort(remoteAddr)
c := &sclient{
connNum: connNum,
s: s,
@@ -830,11 +922,12 @@ func (s *Server) accept(ctx context.Context, nc Conn, brw *bufio.ReadWriter, rem
done: ctx.Done(),
remoteIPPort: remoteIPPort,
connectedAt: s.clock.Now(),
sendQueue: make(chan pkt, perClientSendQueueDepth),
discoSendQueue: make(chan pkt, perClientSendQueueDepth),
sendQueue: make(chan pkt, s.perClientSendQueueDepth),
discoSendQueue: make(chan pkt, s.perClientSendQueueDepth),
sendPongCh: make(chan [8]byte, 1),
peerGone: make(chan peerGoneMsg),
canMesh: s.isMeshPeer(clientInfo),
isNotIdealConn: IdealNodeContextKey.Value(ctx) != "",
peerGoneLim: rate.NewLimiter(rate.Every(time.Second), 3),
}
@@ -881,6 +974,9 @@ func (c *sclient) run(ctx context.Context) error {
if errors.Is(err, context.Canceled) {
c.debugLogf("sender canceled by reader exiting")
} else {
if errors.Is(err, os.ErrDeadlineExceeded) {
c.s.sclientWriteTimeouts.Add(1)
}
c.logf("sender failed: %v", err)
}
}
@@ -1116,31 +1212,37 @@ func (c *sclient) debugLogf(format string, v ...any) {
}
}
// dropReason is why we dropped a DERP frame.
type dropReason int
type dropReasonKindLabels struct {
Reason string // metric label corresponding to a given dropReason
Kind string // either `disco` or `other`
}
//go:generate go run tailscale.com/cmd/addlicense -file dropreason_string.go go run golang.org/x/tools/cmd/stringer -type=dropReason -trimprefix=dropReason
// dropReason is why we dropped a DERP frame.
type dropReason string
const (
dropReasonUnknownDest dropReason = iota // unknown destination pubkey
dropReasonUnknownDestOnFwd // unknown destination pubkey on a derp-forwarded packet
dropReasonGoneDisconnected // destination tailscaled disconnected before we could send
dropReasonQueueHead // destination queue is full, dropped packet at queue head
dropReasonQueueTail // destination queue is full, dropped packet at queue tail
dropReasonWriteError // OS write() failed
dropReasonDupClient // the public key is connected 2+ times (active/active, fighting)
numDropReasons // unused; keep last
dropReasonUnknownDest dropReason = "unknown_dest" // unknown destination pubkey
dropReasonUnknownDestOnFwd dropReason = "unknown_dest_on_fwd" // unknown destination pubkey on a derp-forwarded packet
dropReasonGoneDisconnected dropReason = "gone_disconnected" // destination tailscaled disconnected before we could send
dropReasonQueueHead dropReason = "queue_head" // destination queue is full, dropped packet at queue head
dropReasonQueueTail dropReason = "queue_tail" // destination queue is full, dropped packet at queue tail
dropReasonWriteError dropReason = "write_error" // OS write() failed
dropReasonDupClient dropReason = "dup_client" // the public key is connected 2+ times (active/active, fighting)
)
func (s *Server) recordDrop(packetBytes []byte, srcKey, dstKey key.NodePublic, reason dropReason) {
s.packetsDropped.Add(1)
s.packetsDroppedReasonCounters[reason].Add(1)
labels := dropReasonKindLabels{
Reason: string(reason),
}
looksDisco := disco.LooksLikeDiscoWrapper(packetBytes)
if looksDisco {
s.packetsDroppedTypeDisco.Add(1)
labels.Kind = string(packetKindDisco)
} else {
s.packetsDroppedTypeOther.Add(1)
labels.Kind = string(packetKindOther)
}
packetsDropped.Add(labels, 1)
bytesDropped.Add(labels, int64(len(packetBytes)))
if verboseDropKeys[dstKey] {
// Preformat the log string prior to calling limitedLogf. The
// limiter acts based on the format string, and we want to
@@ -1229,8 +1331,6 @@ func (c *sclient) requestMeshUpdate() {
}
}
var localClient tailscale.LocalClient
// isMeshPeer reports whether the client is a trusted mesh peer
// node in the DERP region.
func (s *Server) isMeshPeer(info *clientInfo) bool {
@@ -1249,7 +1349,7 @@ func (s *Server) verifyClient(ctx context.Context, clientKey key.NodePublic, inf
// tailscaled-based verification:
if s.verifyClientsLocalTailscaled {
_, err := localClient.WhoIsNodeKey(ctx, clientKey)
_, err := s.localClient.WhoIsNodeKey(ctx, clientKey)
if err == tailscale.ErrPeerNotFound {
return fmt.Errorf("peer %v not authorized (not found in local tailscaled)", clientKey)
}
@@ -1505,6 +1605,7 @@ type sclient struct {
peerGone chan peerGoneMsg // write request that a peer is not at this server (not used by mesh peers)
meshUpdate chan struct{} // write request to write peerStateChange
canMesh bool // clientInfo had correct mesh token for inter-region routing
isNotIdealConn bool // client indicated it is not its ideal node in the region
isDup atomic.Bool // whether more than 1 sclient for key is connected
isDisabled atomic.Bool // whether sends to this peer are disabled due to active/active dups
debug bool // turn on for verbose logging
@@ -1540,6 +1641,9 @@ func (c *sclient) presentFlags() PeerPresentFlags {
if c.canMesh {
f |= PeerPresentIsMeshPeer
}
if c.isNotIdealConn {
f |= PeerPresentNotIdeal
}
if f == 0 {
return PeerPresentIsRegular
}
@@ -1721,7 +1825,30 @@ func (c *sclient) sendLoop(ctx context.Context) error {
}
func (c *sclient) setWriteDeadline() {
c.nc.SetWriteDeadline(time.Now().Add(writeTimeout))
d := c.s.tcpWriteTimeout
if c.canMesh {
// Trusted peers get more tolerance.
//
// The "canMesh" is a bit of a misnomer; mesh peers typically run over a
// different interface for a per-region private VPC and are not
// throttled. But monitoring software elsewhere over the internet also
// use the private mesh key to subscribe to connect/disconnect events
// and might hit throttling and need more time to get the initial dump
// of connected peers.
d = privilegedWriteTimeout
}
if d == 0 {
// A zero value should disable the write deadline per
// --tcp-write-timeout docs. The flag should only be applicable for
// non-mesh connections, again per its docs. If mesh happened to use a
// zero value constant above it would be a bug, so we don't bother
// with a condition on c.canMesh.
return
}
// Ignore the error from setting the write deadline. In practice,
// setting the deadline will only fail if the connection is closed
// or closing, so the subsequent Write() will fail anyway.
_ = c.nc.SetWriteDeadline(time.Now().Add(d))
}
// sendKeepAlive sends a keep-alive frame, without flushing.
@@ -2033,6 +2160,7 @@ func (s *Server) ExpVar() expvar.Var {
m.Set("gauge_current_file_descriptors", expvar.Func(func() any { return metrics.CurrentFDs() }))
m.Set("gauge_current_connections", &s.curClients)
m.Set("gauge_current_home_connections", &s.curHomeClients)
m.Set("gauge_current_notideal_connections", &s.curClientsNotIdeal)
m.Set("gauge_clients_total", expvar.Func(func() any { return len(s.clientsMesh) }))
m.Set("gauge_clients_local", expvar.Func(func() any { return len(s.clients) }))
m.Set("gauge_clients_remote", expvar.Func(func() any { return len(s.clientsMesh) - len(s.clients) }))
@@ -2042,9 +2170,6 @@ func (s *Server) ExpVar() expvar.Var {
m.Set("accepts", &s.accepts)
m.Set("bytes_received", &s.bytesRecv)
m.Set("bytes_sent", &s.bytesSent)
m.Set("packets_dropped", &s.packetsDropped)
m.Set("counter_packets_dropped_reason", &s.packetsDroppedReason)
m.Set("counter_packets_dropped_type", &s.packetsDroppedType)
m.Set("counter_packets_received_kind", &s.packetsRecvByKind)
m.Set("packets_sent", &s.packetsSent)
m.Set("packets_received", &s.packetsRecv)
@@ -2060,6 +2185,7 @@ func (s *Server) ExpVar() expvar.Var {
m.Set("multiforwarder_created", &s.multiForwarderCreated)
m.Set("multiforwarder_deleted", &s.multiForwarderDeleted)
m.Set("packet_forwarder_delete_other_value", &s.removePktForwardOther)
m.Set("sclient_write_timeouts", &s.sclientWriteTimeouts)
m.Set("average_queue_duration_ms", expvar.Func(func() any {
return math.Float64frombits(atomic.LoadUint64(s.avgQueueDuration))
}))
@@ -2123,7 +2249,7 @@ func (s *Server) ConsistencyCheck() error {
func (s *Server) checkVerifyClientsLocalTailscaled() error {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
status, err := localClient.StatusWithoutPeers(ctx)
status, err := s.localClient.StatusWithoutPeers(ctx)
if err != nil {
return fmt.Errorf("localClient.Status: %w", err)
}