Update

vendor/tailscale.com/ipn/ipnlocal/watchdog.go

@@ -0,0 +1,158 @@
+// Copyright (c) Tailscale Inc & contributors
+// SPDX-License-Identifier: BSD-3-Clause
+
+package ipnlocal
+
+import (
+	"log"
+	"runtime"
+	"time"
+
+	"tailscale.com/tstime"
+)
+
+// deadlockProbeDelay is how long a watched call must be in flight before we
+// start probing locks to check for a deadlock. Calls that complete sooner do
+// not trigger any probing.
+const deadlockProbeDelay = 5 * time.Second
+
+// deadlockTimeout is how long the lock-probing goroutine is allowed to run
+// before we declare a deadlock and panic with goroutine stacks. That is, it's
+// the maximum total time we allow any of the probed locks to be held.
+const deadlockTimeout = 30 * time.Second
+
+// CheckDeadlocks schedules a delayed deadlock probe and returns a function to
+// call when the operation being watched is done. Callers typically use it as
+// "defer b.CheckDeadlocks()()" to bracket a region of code that should not
+// take more than [deadlockProbeDelay].
+//
+// This is a backstop for detecting and debugging deadlocks in the process, replacing
+// the earlier watchdogEngine removed in 2b338dd6a8dbd.
+func (b *LocalBackend) CheckDeadlocks() (done func()) {
+	// Bump the in-flight count. If a watched region is already open, the
+	// probe timer is already armed, so the bump is all we need to do: the
+	// matching doneDeadlockCheck will decrement when this caller returns and
+	// only the last one out will stop the timer.
+	if b.deadlockChecksInFlight.Add(1) != 1 {
+		return b.doneDeadlockCheck
+	}
+
+	// Fast path to avoid the deadlockTimerMu+Timer.Reset cost when
+	// CheckDeadlocks is called many times per second by non-overlapping
+	// callers: re-arm the probe timer at most once per wall-clock second.
+	// We use a unix-seconds timestamp (+1 so 0 can mean "never") and a CAS
+	// so that only one caller per second proceeds to touch the timer; the
+	// rest return early.
+	nowUnix := tstime.DefaultClock{Clock: b.Clock()}.Now().Unix() + 1
+	lastUnix := b.lastDeadlockCheckUnix.Load()
+	if lastUnix == nowUnix || !b.lastDeadlockCheckUnix.CompareAndSwap(lastUnix, nowUnix) {
+		return b.doneDeadlockCheck
+	}
+
+	// Slow path: (re)arm the probe timer. Lazily create it on first use.
+	b.deadlockTimerMu.Lock()
+	defer b.deadlockTimerMu.Unlock()
+
+	t := b.deadlockProbeTimer
+	if t == nil {
+		t = time.AfterFunc(deadlockProbeDelay, b.runDeadlockProbe)
+		b.deadlockProbeTimer = t
+	} else {
+		t.Reset(deadlockProbeDelay)
+	}
+	return b.doneDeadlockCheck
+}
+
+func (b *LocalBackend) doneDeadlockCheck() {
+	switch n := b.deadlockChecksInFlight.Add(-1); {
+	case n > 0:
+		return
+	case n < 0:
+		panic("ipnlocal: doneDeadlockCheck called without matching CheckDeadlocks")
+	}
+
+	b.deadlockTimerMu.Lock()
+	defer b.deadlockTimerMu.Unlock()
+	if b.deadlockProbeTimer == nil {
+		return
+	}
+	b.deadlockProbeTimer.Stop()
+}
+
+func (b *LocalBackend) runDeadlockProbe() {
+	b.deadlockTimerMu.Lock()
+	defer b.deadlockTimerMu.Unlock()
+
+	if b.deadlockChecksInFlight.Load() == 0 {
+		return
+	}
+
+	t := b.deadlockTimer
+	if t == nil {
+		t = time.AfterFunc(deadlockTimeout, b.reportDeadlock)
+		b.deadlockTimer = t
+	} else {
+		t.Reset(deadlockTimeout)
+	}
+	defer t.Stop()
+
+	b.probeLocks()
+}
+
+func (b *LocalBackend) probeLocks() {
+	b.probeLocalBackendLock()
+
+	sys := b.sys
+	if sys == nil {
+		return
+	}
+	if bus, ok := sys.Bus.GetOK(); ok && bus != nil {
+		bus.ProbeLocks()
+	}
+	if dialer, ok := sys.Dialer.GetOK(); ok && dialer != nil {
+		dialer.ProbeLocks()
+	}
+	if dm, ok := sys.DNSManager.GetOK(); ok && dm != nil {
+		dm.ProbeLocks()
+	}
+	if e, ok := sys.Engine.GetOK(); ok && e != nil {
+		e.ProbeLocks()
+	}
+	if nm, ok := sys.NetMon.GetOK(); ok && nm != nil {
+		nm.ProbeLocks()
+	}
+	if mc, ok := sys.MagicSock.GetOK(); ok && mc != nil {
+		mc.ProbeLocks()
+	}
+	if tun, ok := sys.Tun.GetOK(); ok && tun != nil {
+		tun.ProbeLocks()
+	}
+	if ht, ok := sys.HealthTracker.GetOK(); ok && ht != nil {
+		ht.ProbeLocks()
+	}
+}
+
+func (b *LocalBackend) probeLocalBackendLock() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+}
+
+func (b *LocalBackend) reportDeadlock() {
+	logf := b.logf
+	if logf == nil {
+		logf = log.Printf
+	}
+	logf("ipnlocal watchdog goroutine stacks:\n%s", goroutineStacks())
+	panic("ipnlocal: watchdog timeout")
+}
+
+func goroutineStacks() []byte {
+	buf := make([]byte, 256<<10)
+	for {
+		n := runtime.Stack(buf, true)
+		if n < len(buf) {
+			return buf[:n]
+		}
+		buf = make([]byte, 2*len(buf))
+	}
+}