Update

vendor/tailscale.com/logtail/logtail.go

@@ -1,4 +1,4 @@
-// Copyright (c) Tailscale Inc & AUTHORS
+// Copyright (c) Tailscale Inc & contributors
 // SPDX-License-Identifier: BSD-3-Clause
 
 //go:build !ts_omit_logtail
@@ -15,6 +15,7 @@ import (
 	"expvar"
 	"fmt"
 	"io"
+	"iter"
 	"log"
 	mrand "math/rand/v2"
 	"net/http"
@@ -27,6 +28,7 @@ import (
 	"time"
 
 	"github.com/creachadair/msync/trigger"
+	jsonv2 "github.com/go-json-experiment/json"
 	"github.com/go-json-experiment/json/jsontext"
 	"tailscale.com/envknob"
 	"tailscale.com/metrics"
@@ -49,15 +51,15 @@ const maxSize = 256 << 10
 // Note that JSON log messages can be as large as maxSize.
 const maxTextSize = 16 << 10
 
-// lowMemRatio reduces maxSize and maxTextSize by this ratio in lowMem mode.
-const lowMemRatio = 4
-
 // bufferSize is the typical buffer size to retain.
 // It is large enough to handle most log messages,
 // but not too large to be a notable waste of memory if retained forever.
 const bufferSize = 4 << 10
 
-func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
+// newLogger constructs a *Logger from cfg, applying defaults, but does not start
+// the background uploading goroutine. It is shared by [NewLogger] and the
+// stateless [UploadLogs].
+func newLogger(cfg Config) *Logger {
 	if cfg.BaseURL == "" {
 		cfg.BaseURL = "https://" + DefaultHost
 	}
@@ -71,11 +73,7 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 		cfg.Stderr = os.Stderr
 	}
 	if cfg.Buffer == nil {
-		pendingSize := 256
-		if cfg.LowMemory {
-			pendingSize = 64
-		}
-		cfg.Buffer = NewMemoryBuffer(pendingSize)
+		cfg.Buffer = NewMemoryBuffer(256)
 	}
 	var procID uint32
 	if cfg.IncludeProcID {
@@ -108,7 +106,6 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 		stderrLevel:    int64(cfg.StderrLevel),
 		httpc:          cfg.HTTPC,
 		url:            cfg.BaseURL + "/c/" + cfg.Collection + "/" + cfg.PrivateID.String() + urlSuffix,
-		lowMem:         cfg.LowMemory,
 		buffer:         cfg.Buffer,
 		maxUploadSize:  cfg.MaxUploadSize,
 		skipClientTime: cfg.SkipClientTime,
@@ -132,15 +129,161 @@ func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
 	}
 	logger.SetSockstatsLabel(sockstats.LabelLogtailLogger)
 	logger.compressLogs = cfg.CompressLogs
+	logger.disabled.Store(cfg.Disabled)
+
+	return logger
+}
+
+// NewLogger returns a new Logger that splits logs as configured between local
+// logging facilities and uploading to a log server in the background.
+func NewLogger(cfg Config, logf tslogger.Logf) *Logger {
+	logger := newLogger(cfg)
 
 	ctx, cancel := context.WithCancel(context.Background())
 	logger.uploadCancel = cancel
 
 	go logger.uploading(ctx)
-	logger.Write([]byte("logtail started"))
+	if envknob.Bool("TS_DEBUG_LOGTAIL") {
+		logger.Write([]byte("logtail started"))
+	}
 	return logger
 }
 
+// Logtail is the reserved "logtail" metadata member of a [LogEntry].
+//
+// Zero-valued fields are omitted when an entry is uploaded. [UploadLogs] fills
+// in any fields the caller leaves zero from its [Config] (e.g. ClientTime and
+// ProcID), so most callers can leave this empty.
+type Logtail struct {
+	// ClientTime is the time the entry was generated. If zero, [UploadLogs]
+	// fills it with the current time unless Config.SkipClientTime is set.
+	ClientTime time.Time `json:"client_time,omitzero"`
+	// ProcID is an ephemeral process identifier; see Config.IncludeProcID.
+	ProcID uint32 `json:"proc_id,omitzero"`
+	// ProcSeq is an ephemeral per-process sequence number; see
+	// Config.IncludeProcSequence.
+	ProcSeq uint64 `json:"proc_seq,omitzero"`
+}
+
+// LogEntry is a single log entry to be uploaded via [UploadLogs].
+//
+// It marshals to a JSON object whose reserved "logtail" member carries the
+// metadata in Logtail and whose remaining members are taken from Value, which
+// is inlined at the top level alongside "logtail".
+//
+// Value must marshal to a JSON object: T must be a Go struct (or pointer to
+// one), a Go map with a string key, or a [jsontext.Value] holding an object
+// (for example jsontext.Value(`{"text":"hello"}`)). This is enforced when the
+// entry is uploaded, not at compile time. Use T = [jsontext.Value] to mix
+// differently-shaped payloads in a single upload.
+type LogEntry[T any] struct {
+	Logtail Logtail `json:"logtail,omitzero"`
+	// The `inline` tag option was renamed to `embed` in Go 1.27's
+	// encoding/json/v2, but the pinned go-json-experiment module
+	// (used on older Go versions) only knows `inline`. Each
+	// implementation ignores the option it doesn't know, so specify
+	// both until we require Go 1.27 and drop `inline`.
+	Value T `json:",inline,embed"`
+}
+
+// UploadLogs uploads entries to the log server described by conf and returns
+// once they have all been sent (or an upload fails). It returns the number of
+// entries that were successfully uploaded.
+//
+// It is a stateless alternative to [NewLogger] for callers that just want to
+// push a batch of log entries without the background uploader, ring buffer,
+// stderr echoing, or network-up gating that a [Logger] provides. Each entry is
+// marshaled to JSON, its [Logtail] metadata is filled in from conf where the
+// caller left it zero (honoring conf.SkipClientTime, conf.IncludeProcID, and
+// conf.IncludeProcSequence), and entries are batched up to the server's maximum
+// upload size and POSTed synchronously (compressed when conf.CompressLogs is
+// set).
+//
+// Unlike [Logger], UploadLogs does not retry: if a batch fails to upload it
+// returns the count of entries sent in prior batches and the error immediately,
+// and any remaining entries are not sent. The conf.Stderr and conf.Bus fields
+// are ignored.
+func UploadLogs[T any](ctx context.Context, conf Config, entries iter.Seq[LogEntry[T]]) (int, error) {
+	conf.Stderr = io.Discard // pure uploader: never echo to stderr
+	conf.Bus = nil           // no netmon/eventbus subscription for a one-shot
+	lg := newLogger(conf)
+
+	maxLen := cmp.Or(lg.maxUploadSize, maxSize)
+
+	// body accumulates a JSON array of encoded entries: "[e1,e2,...]".
+	// The framing mirrors Logger.drainPending.
+	body := make([]byte, 0, bufferSize) // reused across batches
+	body = append(body, '[')
+
+	// sent counts entries confirmed uploaded; pending counts entries
+	// accumulated in body but not yet uploaded. A successful sendBatch moves
+	// pending into sent.
+	var sent, pending int
+	sendBatch := func() error {
+		if len(body) <= len("[") {
+			return nil
+		}
+		out := bytes.TrimRight(body, ",")
+		out = append(out, ']')
+		origlen := -1 // sentinel value: uncompressed
+		// Don't attempt to compress tiny bodies; not worth the CPU cycles.
+		if lg.compressLogs && len(out) > 256 {
+			zbody := zstdframe.AppendEncode(nil, out,
+				zstdframe.FastestCompression, zstdframe.LowMemory(true))
+			// Only send it compressed if the bandwidth savings are sufficient.
+			if len(out)-len(zbody) > 64 {
+				origlen = len(out)
+				out = zbody
+			}
+		}
+		// upload is synchronous, so it is safe to reuse body's backing array
+		// for the next batch once upload returns.
+		if _, err := lg.upload(ctx, out, origlen); err != nil {
+			return err
+		}
+		body = body[:len("[")]
+		sent += pending
+		pending = 0
+		return nil
+	}
+
+	var procSeq uint64
+	for e := range entries {
+		if err := ctx.Err(); err != nil {
+			return sent, err
+		}
+
+		// Fill in metadata from conf, preserving any caller-set values.
+		if e.Logtail.ClientTime.IsZero() && !lg.skipClientTime {
+			e.Logtail.ClientTime = lg.clock.Now().UTC()
+		}
+		if e.Logtail.ProcID == 0 {
+			e.Logtail.ProcID = lg.procID
+		}
+		procSeq++
+		if lg.includeProcSequence {
+			e.Logtail.ProcSeq = procSeq
+		}
+
+		enc, err := jsonv2.Marshal(e)
+		if err != nil {
+			return sent, fmt.Errorf("logtail: encoding entry %d: %w", procSeq, err)
+		}
+
+		// Flush the current batch before adding an entry that would overflow it.
+		if len(body) > len("[") && len(body)+len(enc) > maxLen {
+			if err := sendBatch(); err != nil {
+				return sent, err
+			}
+		}
+		body = append(body, enc...)
+		body = append(body, ',')
+		pending++
+	}
+	err := sendBatch()
+	return sent, err
+}
+
 // Logger writes logs, splitting them as configured between local
 // logging facilities and uploading to a log server.
 type Logger struct {
@@ -148,7 +291,6 @@ type Logger struct {
 	stderrLevel    int64 // accessed atomically
 	httpc          *http.Client
 	url            string
-	lowMem         bool
 	skipClientTime bool
 	netMonitor     *netmon.Monitor
 	buffer         Buffer
@@ -172,6 +314,11 @@ type Logger struct {
 	procID              uint32
 	includeProcSequence bool
 
+	// disabled, when true, causes this logger to drop incoming log entries
+	// without buffering or uploading. It is independent of the process-wide
+	// Disable kill switch, which takes precedence. Toggled by SetEnabled.
+	disabled atomic.Bool
+
 	writeLock    sync.Mutex // guards procSequence, flushTimer, buffer.Write calls
 	procSequence uint64
 	flushTimer   tstime.TimerController // used when flushDelay is >0
@@ -298,13 +445,6 @@ func (lg *Logger) drainPending() (b []byte) {
 	}()
 
 	maxLen := cmp.Or(lg.maxUploadSize, maxSize)
-	if lg.lowMem {
-		// When operating in a low memory environment, it is better to upload
-		// in multiple operations than it is to allocate a large body and OOM.
-		// Even if maxLen is less than maxSize, we can still upload an entry
-		// that is up to maxSize if we happen to encounter one.
-		maxLen /= lowMemRatio
-	}
 	for len(b) < maxLen {
 		line, err := lg.buffer.TryReadLine()
 		switch {
@@ -389,6 +529,9 @@ func (lg *Logger) uploading(ctx context.Context) {
 		var numFailures int
 		var firstFailure time.Time
 		for len(body) > 0 && ctx.Err() == nil {
+			if logtailDisabled.Load() || lg.disabled.Load() {
+				break
+			}
 			retryAfter, err := lg.upload(ctx, body, origlen)
 			if err != nil {
 				numFailures++
@@ -411,7 +554,7 @@ func (lg *Logger) uploading(ctx context.Context) {
 				if retryAfter <= 0 {
 					retryAfter = mrand.N(30*time.Second) + 30*time.Second
 				}
-				tstime.Sleep(ctx, retryAfter)
+				tstime.Sleep(ctx, min(retryAfter, 5*time.Minute)) // ignore absurdly large retryAfter values
 			} else {
 				// Only print a success message after recovery.
 				if numFailures > 0 {
@@ -594,6 +737,15 @@ func Disable() {
 	logtailDisabled.Store(true)
 }
 
+// SetEnabled enables or disables log uploading by lg. When disabled, log
+// entries passed to lg are dropped rather than buffered or uploaded; already
+// buffered entries may still drain. The process-wide [Disable] kill switch
+// takes precedence: if Disable has been called, SetEnabled(true) does not
+// re-enable uploads.
+func (lg *Logger) SetEnabled(enabled bool) {
+	lg.disabled.Store(!enabled)
+}
+
 var debugWakesAndUploads = envknob.RegisterBool("TS_DEBUG_LOGTAIL_WAKES")
 
 // tryDrainWake tries to send to lg.drainWake, to cause an uploading wakeup.
@@ -613,7 +765,7 @@ func (lg *Logger) tryDrainWake() {
 
 func (lg *Logger) sendLocked(jsonBlob []byte) (int, error) {
 	tapSend(jsonBlob)
-	if logtailDisabled.Load() {
+	if logtailDisabled.Load() || lg.disabled.Load() {
 		return len(jsonBlob), nil
 	}
 
@@ -711,12 +863,8 @@ func (lg *Logger) appendText(dst, src []byte, skipClientTime bool, procID uint32
 
 	// Append the text string, which may be truncated.
 	// Invalid UTF-8 will be mangled with the Unicode replacement character.
-	max := maxTextSize
-	if lg.lowMem {
-		max /= lowMemRatio
-	}
 	dst = append(dst, `"text":`...)
-	dst = appendTruncatedString(dst, src, max)
+	dst = appendTruncatedString(dst, src, maxTextSize)
 	return append(dst, "}\n"...)
 }
 
@@ -810,7 +958,7 @@ func (lg *Logger) appendTextOrJSONLocked(dst, src []byte, level int) []byte {
 	}
 
 	// Check whether the reserved logtail member occurs in the log data.
-	// If so, it is moved to the the logtail/error member.
+	// If so, it is moved to the logtail/error member.
 	const jsonSeperators = ",:"      // per RFC 8259, section 2
 	const jsonWhitespace = " \n\r\t" // per RFC 8259, section 2
 	var errDetail string
@@ -902,8 +1050,8 @@ func parseAndRemoveLogLevel(buf []byte) (level int, cleanBuf []byte) {
 	if bytes.Contains(buf, v2) {
 		return 2, bytes.ReplaceAll(buf, v2, nil)
 	}
-	if i := bytes.Index(buf, vJSON); i != -1 {
-		rest := buf[i+len(vJSON):]
+	if _, after, ok := bytes.Cut(buf, vJSON); ok {
+		rest := after
 		if len(rest) >= 2 {
 			v := rest[0]
 			if v >= '0' && v <= '9' {