This commit is contained in:
bluepython508 committed 2026-10-22 12:27:19 +01:00
1 parent 3530afb854
commit 6903fcff27
2146 files changed
+112256 -190703

No files matched your search

+329
View File
@@ -0,0 +1,329 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package compserve
import (
"bufio"
"io"
"net"
"net/http"
"strings"
"tailscale.com/util/httpm"
)
// CompressWriter wraps w so that its response body is transparently
// compressed, incrementally as the handler writes it, with the first
// encoding from encodings (default: [DefaultEncodings]) that offers live
// compression and that r accepts.
//
// The caller must call Close exactly once after the wrapped handler
// returns, to finish any compressed stream:
//
// cw := compserve.CompressWriter(w, r)
// defer cw.Close()
// handler.ServeHTTP(cw, r)
func CompressWriter(w http.ResponseWriter, r *http.Request, encodings ...Encoding) *compressWriter {
if len(encodings) == 0 {
encodings = DefaultEncodings
}
var encs []Encoding
for _, e := range encodings {
if e.Compress != nil {
encs = append(encs, e)
}
}
// Compression applies to plain responses of standard body-carrying
// requests: not HEAD (no body), not CONNECT (tunnel), not protocol
// upgrades, not range requests (range semantics apply to the
// uncompressed representation).
safe := (r.Method == httpm.GET || r.Method == httpm.POST) &&
r.Header.Get("Upgrade") == "" &&
r.Header.Get("Range") == ""
return &compressWriter{
ResponseWriter: w,
encs: encs,
ae: r.Header.Get("Accept-Encoding"),
safe: safe,
}
}
// compressWriter is a ResponseWriter wrapper that compresses the response
// body of dynamic handlers. See [CompressWriter].
//
// The body is compressed as it is written, with no buffering. The compress
// decision is made once from the response headers, sniffing the first
// write if the handler sets no Content-Type.
//
// Compressed responses are chunked: the compressed size is not known up
// front, so a handler-set Content-Length is removed. zstd framing may add
// a few bytes to incompressible data.
//
// Responses pass through uncompressed when they already have a
// Content-Encoding, on non-GET/POST requests (HEAD, CONNECT, upgrades,
// range requests), non-200 statuses, or incompressible content types (see
// compressibleContentType).
//
// Every response gets Vary: Accept-Encoding. Strong ETags are weakened:
// validators are representation-specific.
type compressWriter struct {
http.ResponseWriter
encs []Encoding
ae string // Accept-Encoding header of the request
safe bool // request is safe to compress the response of
code int // stashed status code; 0 until the first WriteHeader or Write
decided bool // whether the compress-or-passthrough decision was made
compress bool // decision outcome: stream through the compressor
hijacked bool // connection hijacked; the wrapper is out of the path
cw io.WriteCloser
}
// Unwrap exposes the wrapped ResponseWriter to http.ResponseController for
// deadline support; Flush and Hijack stay handled by the wrapper.
func (c *compressWriter) Unwrap() http.ResponseWriter {
return c.ResponseWriter
}
// Hijack switches the connection out from under the wrapper: after a
// successful hijack the connection is raw, no compression applies, and
// wrapper writes fail. Handlers reach it via http.NewResponseController or
// a direct type assertion, so hijacked paths carry no io overhead from
// this wrapper. ResponseController.Hijack unwraps through further
// middleware below this wrapper.
func (c *compressWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
if !c.decided {
// No response was produced; the handler will speak for itself on
// the hijacked connection. Do not attach a compressor or send
// response headers.
c.decided = true
} else if c.compress {
// Compressed bytes were already streamed; finish the frame and
// return the encoder to its pool before the connection leaves our
// control.
err := c.cw.Close()
c.compress, c.cw = false, nil
if err != nil {
return nil, nil, err
}
}
c.hijacked = true
return http.NewResponseController(c.ResponseWriter).Hijack()
}
// WriteHeader stashes the status code, deciding early if the response is
// already disqualified from compression. See [CompressWriter].
func (c *compressWriter) WriteHeader(code int) {
if c.hijacked {
return // the handler owns the connection
}
if c.code != 0 {
return // ignore superfluous calls, like net/http does
}
c.code = code
if !c.mayCompress() {
c.decide(nil)
}
}
// mayCompress reports whether the stashed response could still be
// compressed based on the request and the handler's headers alone.
func (c *compressWriter) mayCompress() bool {
if len(c.encs) == 0 || c.code != http.StatusOK || !c.safe {
return false
}
// Never double-encode (e.g. precompressed variants served by
// [ServeFile] through this wrapper).
return c.Header().Get("Content-Encoding") == ""
}
// Write passes the body through the streaming compressor or the wrapped
// writer, deciding on the first write. See [CompressWriter].
func (c *compressWriter) Write(p []byte) (int, error) {
if c.hijacked {
return 0, http.ErrHijacked
}
if !c.decided {
c.decide(p)
}
if c.compress {
return c.cw.Write(p)
}
return c.ResponseWriter.Write(p)
}
// Flush flushes buffered compressed data and the underlying writer. See
// [CompressWriter].
func (c *compressWriter) Flush() {
if !c.decided {
c.decide(nil)
}
if c.compress {
type flusher interface{ Flush() }
if f, ok := c.cw.(flusher); ok {
f.Flush()
}
}
if f, ok := c.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
// Close finishes any compressed stream and must be called exactly once
// after the wrapped handler returns. See [CompressWriter].
func (c *compressWriter) Close() error {
if c.hijacked {
return nil // the connection left our control in Hijack
}
if !c.decided {
c.decide(nil)
}
if c.compress {
return c.cw.Close()
}
return nil
}
// decide makes the compress-or-passthrough decision from the request, the
// response headers, and (when no Content-Type is set) the first body
// bytes. It forwards the response headers.
func (c *compressWriter) decide(first []byte) {
c.decided = true
if c.code == 0 {
c.code = http.StatusOK
}
ct := c.Header().Get("Content-Type")
sniffed := false
if ct == "" && first != nil {
// Mirror net/http's sniffing of the first bytes.
ct = http.DetectContentType(first)
sniffed = true
}
if !c.mayCompress() || ct == "" || !compressibleContentType(ct) {
c.commitPassthrough()
return
}
enc := negotiate(c.ae, c.encs)
if enc == nil {
c.commitPassthrough()
return
}
cw, err := enc.Compress(c.ResponseWriter)
if err != nil {
c.commitPassthrough()
return
}
h := c.Header()
if sniffed {
// net/http would otherwise sniff the compressed bytes and label
// the response application/octet-stream.
h.Set("Content-Type", ct)
}
// The handler's Content-Length describes the uncompressed
// representation; the compressed length is unknown, so the response is
// chunked.
h.Del("Content-Length")
h.Set("Content-Encoding", enc.Token)
// Strong validators must be representation-specific.
if etag := h.Get("ETag"); etag != "" && !strings.HasPrefix(etag, "W/") {
h.Set("ETag", "W/"+etag)
}
addVary(h, "Accept-Encoding")
c.cw = cw
c.compress = true
c.ResponseWriter.WriteHeader(c.code)
}
// commitPassthrough forwards the stashed status to the wrapped writer;
// subsequent writes stream through directly.
func (c *compressWriter) commitPassthrough() {
if c.code == 0 {
c.code = http.StatusOK
}
addVary(c.Header(), "Accept-Encoding")
c.ResponseWriter.WriteHeader(c.code)
}
// compressibleContentType reports whether the given content type is worth
// live compression. Most types are assumed compressible: only content that
// is already compressed by construction is excluded (media codecs, archive
// formats), except raw media types that are not, listed below.
// +json/+xml suffix types are always compressed.
func compressibleContentType(ct string) bool {
ct = strings.ToLower(ct)
if i := strings.IndexByte(ct, ';'); i >= 0 {
ct = strings.TrimSpace(ct[:i])
}
if strings.HasSuffix(ct, "+json") || strings.HasSuffix(ct, "+xml") {
return true
}
switch {
case strings.HasPrefix(ct, "audio/"):
return rawAudioContentType(ct)
case strings.HasPrefix(ct, "video/"):
return rawVideoContentType(ct)
}
switch ct {
case "application/zip",
"application/gzip",
"application/x-gzip",
"application/x-bzip2",
"application/x-xz",
"application/zstd",
"application/x-zstd",
"application/x-7z-compressed",
"application/x-rar-compressed",
"application/vnd.rar",
"application/x-compress",
"application/x-snappy-framed",
"application/x-lz4",
// Archive formats that are zip or other compressed containers.
"application/java-archive", // jar: zip
"application/vnd.android.package-archive", // apk: zip
"application/x-apple-diskimage": // dmg: compressed
return false
}
return true
}
// rawAudioContentType reports whether ct is an uncompressed audio format
// worth compressing despite audio/ being excluded by default: audio content
// is almost always compressed by its codec, but raw PCM is not.
func rawAudioContentType(ct string) bool {
switch ct {
case "audio/wav", "audio/wave", "audio/x-wav", "audio/vnd.wave", // WAV: PCM
"audio/pcm",
"audio/l16", "audio/l24": // raw linear PCM, RFC 2586
return true
}
return false
}
// rawVideoContentType reports whether ct is an uncompressed video format
// worth compressing despite video/ being excluded by default: video content
// is almost always compressed by its codec, but raw frame data is not.
func rawVideoContentType(ct string) bool {
switch ct {
case "video/raw", "video/x-raw", "video/x-raw-yuv",
"video/x-y4m": // YUV4MPEG2
return true
}
return false
}
// addVary adds token to the response's Vary header, preserving any values
// already set by the handler.
func addVary(h http.Header, token string) {
for _, vary := range h.Values("Vary") {
for _, t := range strings.Split(vary, ",") {
if strings.EqualFold(strings.TrimSpace(t), token) {
return
}
}
}
h.Add("Vary", token)
}
+312
View File
@@ -0,0 +1,312 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
// Package compserve serves static files with content negotiation for
// precompressed variants, and provides live compression for dynamic
// responses.
//
// An [Encoding] describes a content coding's wire token and where its
// precompressed variants live.
package compserve
import (
"errors"
"fmt"
"io"
"io/fs"
"net/http"
"strconv"
"strings"
"time"
)
// Encoding describes a single content coding (e.g. zstd) that this package
// can negotiate and serve.
type Encoding struct {
// Token is the Content-Encoding token for this coding, e.g. "zstd".
// Comparison against Accept-Encoding is case-insensitive.
Token string
// Ext is the file name extension (including the leading dot) of
// precompressed variants as produced by util/precompress, e.g. ".zst".
Ext string
// Decompress, if non-nil, wraps f and yields its decompressed content,
// taking ownership of f: the returned ReadCloser's Close closes f, and
// if Decompress returns an error it must have closed f already.
//
// When Decompress is set, a precompressed variant can also be served to
// clients that do not accept Token by decompressing it on the fly, so a
// file system shipping only precompressed variants can still serve
// every client. If nil, the variant is only served passthrough to
// clients that accept Token.
Decompress func(f fs.File) (io.ReadCloser, error)
// Compress, if non-nil, wraps w in a streaming compressor used by
// [CompressWriter] for live (non-precompressed) response compression:
// everything written to the returned WriteCloser is compressed and
// forwarded to w, and its Close finishes the stream and releases any
// pooled resources. Implementations should use a fast compression
// mode: live compression runs per request and must not add meaningful
// latency. If nil, the encoding is not offered for live compression.
Compress func(w io.Writer) (io.WriteCloser, error)
}
// DefaultEncodings is the set of encodings used when Options does not
// specify any: zstd, then gzip for compatibility with file systems built
// before zstd-only asset generation. Ties prefer the earlier encoding.
var DefaultEncodings = []Encoding{Zstd, Gzip}
// Options configures ServeFile.
type Options struct {
// Encodings lists the precompressed variants available in the file
// system, in preference order. The zero value selects
// [DefaultEncodings].
Encodings []Encoding
}
// ServeFile serves the file at path from fsys to w, negotiating a
// precompressed variant when the client accepts one and the variant exists.
// Clients that do not accept any offered encoding are served the raw file;
// if the raw file is absent (e.g. a file system shipping only
// precompressed variants), a variant with a Decompress hook is served
// decompressed instead.
//
// ServeFile sets Vary: Accept-Encoding on every response and serves the
// content via http.ServeContent with the file's modification time (for
// embedded file systems, wrap fsys in [tailscale.com/tsweb/vcstime.FS] to
// supply one). Callers may set additional headers (e.g. Cache-Control)
// before calling. If ServeFile returns an error it has not written a
// response body, so callers may adjust headers and retry with a different
// path. If no representation of the file exists, the returned error wraps
// fs.ErrNotExist.
func ServeFile(w http.ResponseWriter, r *http.Request, fsys fs.FS, path string, opts Options) error {
encodings := opts.Encodings
if encodings == nil {
encodings = DefaultEncodings
}
if !fs.ValidPath(path) {
// An invalid name can never exist. Report not-found rather than
// the fs.ErrInvalid that validating file systems such as fs.Sub
// return, so callers can fall back to another path.
return &fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}
}
// The served representation (an encoded variant or identity) is
// selected from Accept-Encoding, so caches must treat the response as
// varying even when the negotiated representation is identity.
addVary(w.Header(), "Accept-Encoding")
// Prefer serving an accepted variant passthrough: no server CPU cost.
if enc := negotiate(r.Header.Get("Accept-Encoding"), encodings); enc != nil {
if f, err := fsys.Open(path + enc.Ext); err == nil {
rs, ok := f.(io.ReadSeeker)
if !ok {
// Fall through to the raw file; it may still be servable.
f.Close()
} else {
w.Header().Set("Content-Encoding", enc.Token)
return serveContent(w, r, path, fileModTime(f), rs)
}
} else if !errors.Is(err, fs.ErrNotExist) {
return err
}
}
// Serve the raw file if it exists: it is identical to what transcoding
// a variant would produce, without the CPU cost.
f, err := fsys.Open(path)
if err != nil && !errors.Is(err, fs.ErrNotExist) {
return err
}
if err == nil {
rs, ok := f.(io.ReadSeeker)
if !ok {
f.Close()
return fmt.Errorf("%s is not seekable", path)
}
return serveContent(w, r, path, fileModTime(f), rs)
}
// The raw file does not exist: transcode a precompressed variant to
// identity for clients that do not accept it.
for _, e := range encodings {
if e.Decompress == nil {
continue
}
f, err := fsys.Open(path + e.Ext)
if err != nil {
if errors.Is(err, fs.ErrNotExist) {
continue
}
return err
}
modTime := fileModTime(f)
tf := &transcodedFile{fsys: fsys, path: path + e.Ext, dec: e.Decompress}
if err := tf.start(f); err != nil {
return err
}
return serveContent(w, r, path, modTime, tf)
}
return &fs.PathError{Op: "open", Path: path, Err: fs.ErrNotExist}
}
// fileModTime reports f's modification time, or the zero time (which
// disables conditional requests in http.ServeContent) if f does not stat.
func fileModTime(f fs.File) time.Time {
fi, err := f.Stat()
if err != nil {
return time.Time{}
}
return fi.ModTime()
}
// serveContent serves rs via http.ServeContent, closing it if it is an
// io.Closer.
func serveContent(w http.ResponseWriter, r *http.Request, name string, modTime time.Time, rs io.ReadSeeker) error {
http.ServeContent(w, r, name, modTime, rs)
if c, ok := rs.(io.Closer); ok {
// The response body has been written; a close error is not
// actionable.
c.Close()
}
return nil
}
// negotiate returns the encoding from offered that the given
// Accept-Encoding header value accepts with the highest quality value, or
// nil if it accepts none of them. Ties prefer the earlier entry in offered.
func negotiate(ae string, offered []Encoding) *Encoding {
if ae == "" {
return nil
}
exact, wildcard, hasWildcard := parseAcceptEncoding(ae)
var best *Encoding
bestQ := 0.0
for i := range offered {
enc := &offered[i]
q, ok := exact[strings.ToLower(enc.Token)]
if !ok && hasWildcard {
q, ok = wildcard, true
}
// A quality of zero explicitly rejects the coding, and equal
// qualities prefer the earlier entry in offered.
if ok && q > bestQ {
best, bestQ = enc, q
}
}
return best
}
// parseAcceptEncoding parses an Accept-Encoding header value into the
// quality values of the explicitly named (lowercased) coding tokens and of
// the "*" wildcard, if present. Quality values default to 1 per RFC 9110,
// and a malformed quality value rejects the coding.
func parseAcceptEncoding(h string) (exact map[string]float64, wildcard float64, hasWildcard bool) {
exact = make(map[string]float64)
for _, part := range strings.Split(h, ",") {
token, params, _ := strings.Cut(part, ";")
token = strings.ToLower(strings.TrimSpace(token))
if token == "" {
continue
}
q := 1.0
for _, p := range strings.FieldsFunc(params, func(r rune) bool { return r == ';' }) {
name, value, _ := strings.Cut(strings.TrimSpace(p), "=")
if !strings.EqualFold(name, "q") {
continue
}
if v, err := strconv.ParseFloat(strings.TrimSpace(value), 64); err == nil {
q = v
} else {
q = 0
}
break
}
if token == "*" {
wildcard, hasWildcard = q, true
continue
}
exact[token] = q
}
return exact, wildcard, hasWildcard
}
// transcodedFile serves the decompressed content of a precompressed file to
// clients that do not accept its encoding. It implements io.ReadSeeker so
// that it can be used with http.ServeContent; seeking re-opens the
// underlying file and decompresses from the start, so it is only suitable
// for small files where the double decompression cost is acceptable.
type transcodedFile struct {
fsys fs.FS
path string // path of the compressed file in fsys
dec func(f fs.File) (io.ReadCloser, error) // decompressor for that file
rc io.ReadCloser // current decompressed stream
pos int64 // current position in the decompressed stream
}
// start replaces the decompressed stream with one reading f (whose
// ownership is taken) from its start.
func (t *transcodedFile) start(f fs.File) error {
rc, err := t.dec(f)
if err != nil {
return err
}
t.closeStream()
t.rc, t.pos = rc, 0
return nil
}
// restart replaces the decompressed stream with a fresh one reading the
// underlying file from its start.
func (t *transcodedFile) restart() error {
f, err := t.fsys.Open(t.path)
if err != nil {
return err
}
return t.start(f)
}
func (t *transcodedFile) closeStream() {
if t.rc != nil {
t.rc.Close()
t.rc = nil
}
}
func (t *transcodedFile) Read(p []byte) (int, error) {
if t.rc == nil {
return 0, fs.ErrClosed
}
n, err := t.rc.Read(p)
t.pos += int64(n)
return n, err
}
func (t *transcodedFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case io.SeekStart:
if err := t.restart(); err != nil {
return 0, err
}
_, err := io.CopyN(io.Discard, t, offset)
return t.pos, err
case io.SeekCurrent:
if offset < 0 {
return 0, fmt.Errorf("negative offset for SeekCurrent: %w", fs.ErrInvalid)
}
_, err := io.CopyN(io.Discard, t, offset)
return t.pos, err
case io.SeekEnd:
if offset != 0 {
return 0, fmt.Errorf("non-zero offset for SeekEnd: %w", fs.ErrInvalid)
}
_, err := io.Copy(io.Discard, t)
return t.pos, err
}
return 0, fs.ErrInvalid
}
func (t *transcodedFile) Close() error {
t.closeStream()
return nil
}
+43
View File
@@ -0,0 +1,43 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package compserve
import (
"compress/gzip"
"io"
"io/fs"
)
// Gzip serves gzip-precompressed variants ("*.gz"). It exists for
// transitional compatibility with file systems built before zstd-only
// asset generation, such as web-client-prebuilt modules published before
// cmd/build-webclient stopped writing gzip; file systems built since
// contain no .gz variants and never select it.
var Gzip = Encoding{
Token: "gzip",
Ext: ".gz",
Decompress: decompressGzip,
}
// decompressGzip wraps f in a streaming gzip decompressor, taking
// ownership of f.
func decompressGzip(f fs.File) (io.ReadCloser, error) {
zr, err := gzip.NewReader(f)
if err != nil {
f.Close()
return nil, err
}
return &gzipFile{f: f, Reader: zr}, nil
}
// gzipFile is a gzip.Reader reading from f; Close closes both.
type gzipFile struct {
f fs.File
*gzip.Reader
}
func (g *gzipFile) Close() error {
g.Reader.Close()
return g.f.Close()
}
+81
View File
@@ -0,0 +1,81 @@
// Copyright (c) Tailscale Inc & contributors
// SPDX-License-Identifier: BSD-3-Clause
package compserve
import (
"io"
"io/fs"
"github.com/klauspost/compress/zstd"
"tailscale.com/util/zstdframe"
)
// Zstd serves zstd-precompressed variants ("*.zst", as generated by
// util/precompress). [Encoding.Decompress] lets ServeFile transcode the
// variant for clients that do not accept zstd.
var Zstd = Encoding{
Token: "zstd",
Ext: ".zst",
Decompress: decompressZstd,
Compress: compressZstdLive,
}
// compressZstdLive wraps w in a streaming zstd compressor using the fastest
// compression level and a small window suitable for low-latency live
// compression. Close on the returned writer finishes the zstd frame and
// returns the encoder to the shared pool.
func compressZstdLive(w io.Writer) (io.WriteCloser, error) {
enc, put := zstdframe.GetStreamingEncoder(
zstdframe.FastestCompression,
zstdframe.MaxWindowSize(64<<10),
)
// Encoder.Reset (streaming) cannot fail; only Decoder.Reset can.
enc.Reset(w)
return &pooledZstdWriter{Encoder: enc, put: put}, nil
}
// pooledZstdWriter is a zstd.Encoder writing to the underlying writer;
// Close finishes the frame and returns the encoder to the shared pool.
type pooledZstdWriter struct {
put func()
closed bool
*zstd.Encoder
}
func (p *pooledZstdWriter) Close() error {
if p.closed {
return nil
}
p.closed = true
err := p.Encoder.Close()
p.put()
return err
}
// decompressZstd wraps f in a streaming zstd decompressor backed by the
// shared decoder pool in util/zstdframe (concurrency=1, so streaming decode
// neither spawns goroutines nor retains unbounded memory). It takes
// ownership of f.
func decompressZstd(f fs.File) (io.ReadCloser, error) {
dec, put := zstdframe.GetDecoder()
if err := dec.Reset(f); err != nil {
put()
f.Close()
return nil, err
}
return &pooledZstdReader{f: f, put: put, Decoder: dec}, nil
}
// pooledZstdReader is a zstd.Decoder reading from f; Close returns the
// decoder to the shared pool and closes f.
type pooledZstdReader struct {
f fs.File
put func()
*zstd.Decoder
}
func (p *pooledZstdReader) Close() error {
p.put()
return p.f.Close()
}