bluepython508/tsnet-proxy
1
0 Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update

Browse Source
This commit is contained in:
bluepython508 committed 2026-10-22 12:27:19 +01:00
1 parent 3530afb854
commit 6903fcff27
2146 files changed
+112256 -190703
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

vendor/github.com/klauspost/compress/zstd/README.md
Generated Vendored
+35 -2
View File
@@ -75,14 +75,47 @@ The above is fine for big encodes. However, whenever possible try to *reuse* the
To reuse the encoder, you can use the `Reset(io.Writer)` function to change to another output.
This will allow the encoder to reuse all resources and avoid wasteful allocations.
Currently stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part
of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change
By default, stream encoding has 'light' concurrency, meaning up to 2 goroutines can be working on part
of a stream. This is independent of the `WithEncoderConcurrency(n)`, but that is likely to change
in the future. So if you want to limit concurrency for future updates, specify the concurrency
you would like.
If you would like stream encoding to be done without spawning async goroutines, use `WithEncoderConcurrency(1)`
which will compress input as each block is completed, blocking on writes until each has completed.
#### Parallel Stream Compression
For maximum throughput on large streams, use `WithConcurrentBlocks(true)` together with
`WithEncoderConcurrency(n)` where n is the number of CPU cores you want to use.
This splits the input into large sections (jobs) that are compressed simultaneously by multiple goroutines,
similar to how the C zstd library does multithreaded compression.
```Go
enc, err := zstd.NewWriter(out,
zstd.WithEncoderLevel(zstd.SpeedDefault),
zstd.WithEncoderConcurrency(runtime.GOMAXPROCS(0)),
zstd.WithConcurrentBlocks(true),
)
```
Each non-first job receives an overlap prefix from the previous job for match context,
so compression ratio is only marginally affected. Output is flushed in order,
producing a valid single-frame zstd stream.
Benchmark on 1.8GB GOB stream (AMD Ryzen 9 9950X):
| Level | 1 thread | 4 threads | 16 threads | 1T ratio | 16T ratio |
|---------|:----------:|:------------------:|:-------------------:|:--------:|:---------:|
| fastest | 783 MB/s | 2950 MB/s (3.8×) | 6939 MB/s (8.9×) | 12.24% | 12.26% |
| default | 728 MB/s | 2533 MB/s (3.5×) | 5340 MB/s (7.3×) | 10.67% | 10.68% |
| better | 434 MB/s | 1105 MB/s (2.5×) | 2206 MB/s (5.1×) | 9.14% | 9.21% |
| best | 129 MB/s | 367 MB/s (2.8×) | 884 MB/s (6.8×) | 8.48% | 8.63% |
Notes:
* Not compatible with dictionary encoding.
* `Flush()` dispatches the current partial job, so latency-sensitive callers can force output.
* `EncodeAll` is unaffected — it uses its own concurrency via the encoder pool.
You can specify your desired compression level using `WithEncoderLevel()` option. Currently only pre-defined
compression settings can be specified.
vendor/github.com/klauspost/compress/zstd/blockdec.go
Generated Vendored
+3 -6
View File
@@ -400,8 +400,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err
}
}
var err error
// Use our out buffer.
huff.MaxDecodedSize = litRegenSize
// Decoder.Decompress* uses cap(dst) for the size limit. Do not write
// MaxDecodedSize on hist.huffTree: with a trained dictionary that
// pointer aliases the shared dict.litEnc and concurrent DecodeAll races.
if fourStreams {
literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals)
} else {
@@ -673,10 +674,6 @@ func (b *blockDec) executeSequences(hist *history) error {
hbytes := hist.b
if len(hbytes) > hist.windowSize {
hbytes = hbytes[len(hbytes)-hist.windowSize:]
// We do not need history anymore.
if hist.dict != nil {
hist.dict.content = nil
}
}
hist.decoders.windowSize = hist.windowSize
hist.decoders.out = b.dst[:0]
vendor/github.com/klauspost/compress/zstd/blockenc.go
Generated Vendored
+1
View File
@@ -78,6 +78,7 @@ func (b *blockEnc) initNewEncode() {
b.recentOffsets = [3]uint32{1, 4, 8}
b.litEnc.Reuse = huff0.ReusePolicyNone
b.coders.setPrev(nil, nil, nil)
b.dictLitEnc = nil
}
// reset will reset the block for a new encode, but in the same stream,
vendor/github.com/klauspost/compress/zstd/decoder.go
Generated Vendored
+19 -9
View File
@@ -39,9 +39,6 @@ type Decoder struct {
frame *frameDec
// Custom dictionaries.
dicts map[uint32]*dict
// streamWg is the waitgroup for all streams
streamWg sync.WaitGroup
}
@@ -101,12 +98,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
d.current.err = ErrDecoderNilInput
}
// Transfer option dicts.
d.dicts = make(map[uint32]*dict, len(d.o.dicts))
for _, dc := range d.o.dicts {
d.dicts[dc.id] = dc
// Initialize dict map if needed.
if d.o.dicts == nil {
d.o.dicts = make(map[uint32]*dict)
}
d.o.dicts = nil
// Create decoders
d.decoders = make(chan *blockDec, d.o.concurrent)
@@ -238,6 +233,21 @@ func (d *Decoder) Reset(r io.Reader) error {
return nil
}
// ResetWithOptions will reset the decoder and apply the given options
// for the next stream or DecodeAll operation.
// Options are applied on top of the existing options.
// Some options cannot be changed on reset and will return an error.
func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error {
d.o.resetOpt = true
defer func() { d.o.resetOpt = false }()
for _, o := range opts {
if err := o(&d.o); err != nil {
return err
}
}
return d.Reset(r)
}
// drainOutput will drain the output until errEndOfStream is sent.
func (d *Decoder) drainOutput() {
if d.current.cancel != nil {
@@ -930,7 +940,7 @@ decodeStream:
}
func (d *Decoder) setDict(frame *frameDec) (err error) {
dict, ok := d.dicts[frame.DictionaryID]
dict, ok := d.o.dicts[frame.DictionaryID]
if ok {
if debugDecoder {
println("setting dict", frame.DictionaryID)
vendor/github.com/klauspost/compress/zstd/decoder_options.go
Generated Vendored
+52 -8
View File
@@ -20,10 +20,11 @@ type decoderOptions struct {
concurrent int
maxDecodedSize uint64
maxWindowSize uint64
dicts []*dict
dicts map[uint32]*dict
ignoreChecksum bool
limitToCap bool
decodeBufsBelow int
resetOpt bool
}
func (o *decoderOptions) setDefault() {
@@ -42,8 +43,15 @@ func (o *decoderOptions) setDefault() {
// WithDecoderLowmem will set whether to use a lower amount of memory,
// but possibly have to allocate more while running.
// Cannot be changed with ResetWithOptions.
func WithDecoderLowmem(b bool) DOption {
return func(o *decoderOptions) error { o.lowMem = b; return nil }
return func(o *decoderOptions) error {
if o.resetOpt && b != o.lowMem {
return errors.New("WithDecoderLowmem cannot be changed on Reset")
}
o.lowMem = b
return nil
}
}
// WithDecoderConcurrency sets the number of created decoders.
@@ -53,18 +61,23 @@ func WithDecoderLowmem(b bool) DOption {
// inflight blocks.
// When decoding streams and setting maximum to 1,
// no async decoding will be done.
// The value supplied must be at least 0.
// When a value of 0 is provided GOMAXPROCS will be used.
// By default this will be set to 4 or GOMAXPROCS, whatever is lower.
// Cannot be changed with ResetWithOptions.
func WithDecoderConcurrency(n int) DOption {
return func(o *decoderOptions) error {
if n < 0 {
return errors.New("concurrency must be at least 1")
return errors.New("concurrency must be at least 0")
}
newVal := n
if n == 0 {
o.concurrent = runtime.GOMAXPROCS(0)
} else {
o.concurrent = n
newVal = runtime.GOMAXPROCS(0)
}
if o.resetOpt && newVal != o.concurrent {
return errors.New("WithDecoderConcurrency cannot be changed on Reset")
}
o.concurrent = newVal
return nil
}
}
@@ -73,6 +86,7 @@ func WithDecoderConcurrency(n int) DOption {
// non-streaming operations or maximum window size for streaming operations.
// This can be used to control memory usage of potentially hostile content.
// Maximum is 1 << 63 bytes. Default is 64GiB.
// Can be changed with ResetWithOptions.
func WithDecoderMaxMemory(n uint64) DOption {
return func(o *decoderOptions) error {
if n == 0 {
@@ -92,16 +106,20 @@ func WithDecoderMaxMemory(n uint64) DOption {
// "zstd --train" from the Zstandard reference implementation.
//
// If several dictionaries with the same ID are provided, the last one will be used.
// Can be changed with ResetWithOptions.
//
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
func WithDecoderDicts(dicts ...[]byte) DOption {
return func(o *decoderOptions) error {
if o.dicts == nil {
o.dicts = make(map[uint32]*dict)
}
for _, b := range dicts {
d, err := loadDict(b)
if err != nil {
return err
}
o.dicts = append(o.dicts, d)
o.dicts[d.id] = d
}
return nil
}
@@ -109,12 +127,16 @@ func WithDecoderDicts(dicts ...[]byte) DOption {
// WithDecoderDictRaw registers a dictionary that may be used by the decoder.
// The slice content can be arbitrary data.
// Can be changed with ResetWithOptions.
func WithDecoderDictRaw(id uint32, content []byte) DOption {
return func(o *decoderOptions) error {
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
}
o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}})
if o.dicts == nil {
o.dicts = make(map[uint32]*dict)
}
o.dicts[id] = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}
return nil
}
}
@@ -124,6 +146,7 @@ func WithDecoderDictRaw(id uint32, content []byte) DOption {
// The Decoder will likely allocate more memory based on the WithDecoderLowmem setting.
// If WithDecoderMaxMemory is set to a lower value, that will be used.
// Default is 512MB, Maximum is ~3.75 TB as per zstandard spec.
// Can be changed with ResetWithOptions.
func WithDecoderMaxWindow(size uint64) DOption {
return func(o *decoderOptions) error {
if size < MinWindowSize {
@@ -141,6 +164,7 @@ func WithDecoderMaxWindow(size uint64) DOption {
// or any size set in WithDecoderMaxMemory.
// This can be used to limit decoding to a specific maximum output size.
// Disabled by default.
// Can be changed with ResetWithOptions.
func WithDecodeAllCapLimit(b bool) DOption {
return func(o *decoderOptions) error {
o.limitToCap = b
@@ -153,17 +177,37 @@ func WithDecodeAllCapLimit(b bool) DOption {
// This typically uses less allocations but will have the full decompressed object in memory.
// Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less.
// Default is 128KiB.
// Cannot be changed with ResetWithOptions.
func WithDecodeBuffersBelow(size int) DOption {
return func(o *decoderOptions) error {
if o.resetOpt && size != o.decodeBufsBelow {
return errors.New("WithDecodeBuffersBelow cannot be changed on Reset")
}
o.decodeBufsBelow = size
return nil
}
}
// IgnoreChecksum allows to forcibly ignore checksum checking.
// Can be changed with ResetWithOptions.
func IgnoreChecksum(b bool) DOption {
return func(o *decoderOptions) error {
o.ignoreChecksum = b
return nil
}
}
// WithDecoderDictDelete removes dictionaries by ID.
// If no ids are passed, all dictionaries are deleted.
// Should be used with ResetWithOptions.
func WithDecoderDictDelete(ids ...uint32) DOption {
return func(o *decoderOptions) error {
if len(ids) == 0 {
clear(o.dicts)
}
for _, id := range ids {
delete(o.dicts, id)
}
return nil
}
}
vendor/github.com/klauspost/compress/zstd/dict.go
Generated Vendored
+66 -22
View File
@@ -230,7 +230,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
}
block := blockEnc{lowMem: false}
block.init()
enc := encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}})
var enc encoder
if o.Level != 0 {
eOpts := encoderOptions{
level: o.Level,
@@ -242,6 +242,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
enc = eOpts.encoder()
} else {
o.Level = SpeedBestCompression
enc = encoder(&bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(maxMatchLen), bufferReset: math.MaxInt32 - int32(maxMatchLen*2), lowMem: false}})
}
var (
remain [256]int
@@ -295,40 +296,81 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
if offset > 3 {
newOffsets[offset-3]++
} else {
newOffsets[uint32(o.Offsets[offset-1])]++
// Repeat codes reference the training Offsets. Skip unset
// (zero) entries so they are not ranked as real offsets.
prev := o.Offsets[offset-1]
if prev > 0 {
newOffsets[uint32(prev)]++
}
}
}
}
// Find most used offsets.
var sortedOffsets []uint32
for k := range newOffsets {
if k == 0 {
continue
}
sortedOffsets = append(sortedOffsets, k)
}
sort.Slice(sortedOffsets, func(i, j int) bool {
a, b := sortedOffsets[i], sortedOffsets[j]
if a == b {
ca, cb := newOffsets[a], newOffsets[b]
if ca == cb {
// Prefer the longer offset
return sortedOffsets[i] > sortedOffsets[j]
return a > b
}
return newOffsets[sortedOffsets[i]] > newOffsets[sortedOffsets[j]]
return ca > cb
})
if len(sortedOffsets) > 3 {
if debug {
print("Offsets:")
for i, v := range sortedOffsets {
if i > 20 {
break
}
printf("[%d: %d],", v, newOffsets[v])
if debug {
print("Offsets:")
for i, v := range sortedOffsets {
if i > 20 {
break
}
println("")
printf("[%d: %d],", v, newOffsets[v])
}
sortedOffsets = sortedOffsets[:3]
println("")
}
for i, v := range sortedOffsets {
o.Offsets[i] = int(v)
// Dictionary recent-offsets must be three positive values within the
// history. Ranked matches may be fewer (or empty when only unset
// repeat codes were seen), so fill remaining slots with defaults.
used := make(map[int]bool, 3)
var finalOffsets [3]int
nOff := 0
for _, v := range sortedOffsets {
iv := int(v)
if iv <= 0 || iv > len(hist) || used[iv] {
continue
}
finalOffsets[nOff] = iv
used[iv] = true
nOff++
if nOff == 3 {
break
}
}
for _, def := range []int{1, 4, 8} {
if nOff == 3 {
break
}
if def <= len(hist) && !used[def] {
finalOffsets[nOff] = def
used[def] = true
nOff++
}
}
for def := 1; nOff < 3 && def <= len(hist); def++ {
if !used[def] {
finalOffsets[nOff] = def
used[def] = true
nOff++
}
}
if nOff < 3 {
return nil, fmt.Errorf("could not determine 3 valid dictionary offsets (history size %d)", len(hist))
}
o.Offsets = finalOffsets
if debug {
println("New repeat offsets", o.Offsets)
}
@@ -336,6 +378,9 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
if nUsed == 0 || seqs == 0 {
return nil, fmt.Errorf("%d blocks, %d sequences found", nUsed, seqs)
}
if litTotal == 0 {
return nil, errors.New("0 literals found")
}
if debug {
println("Sequences:", seqs, "Blocks:", nUsed, "Literals:", litTotal)
}
@@ -516,11 +561,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[1])))
out.Write(binary.LittleEndian.AppendUint32(nil, uint32(o.Offsets[2])))
out.Write(hist)
if _, err := loadDict(out.Bytes()); err != nil {
return nil, fmt.Errorf("built dictionary failed validation: %w", err)
}
if debug {
_, err := loadDict(out.Bytes())
if err != nil {
panic(err)
}
i, err := InspectDictionary(out.Bytes())
if err != nil {
panic(err)
vendor/github.com/klauspost/compress/zstd/enc_base.go
Generated Vendored
+29 -1
View File
@@ -21,7 +21,7 @@ type fastBase struct {
crc *xxhash.Digest
tmp [8]byte
blk *blockEnc
lastDictID uint32
lastDict *dict
lowMem bool
}
@@ -128,6 +128,34 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
return int32(matchLen(src[s:], src[t:]))
}
// resetBasePrefix resets the encoder state and loads prefix as initial history.
// This is used for parallel job encoding where non-first jobs need overlap context.
// Rep offsets are set to defaults [1,4,8] (invalidated, matching C behavior).
func (e *fastBase) resetBasePrefix(prefix []byte) {
if e.blk == nil {
e.blk = &blockEnc{lowMem: e.lowMem}
e.blk.init()
} else {
e.blk.reset(nil)
}
e.blk.initNewEncode()
if e.crc == nil {
e.crc = xxhash.New()
} else {
e.crc.Reset()
}
e.blk.dictLitEnc = nil
e.ensureHist(len(prefix) + maxCompressedBlockSize)
// Bump cur so old table entries fall outside the window.
// When cur >= bufferReset, leave it; the first Encode call
// will shift/clear tables, preserving valid prefix entries.
if e.cur < e.bufferReset {
e.cur += e.maxMatchOff + int32(len(e.hist))
}
e.hist = e.hist[:0]
e.hist = append(e.hist, prefix...)
}
// Reset the encoding table.
func (e *fastBase) resetBase(d *dict, singleBlock bool) {
if e.blk == nil {
vendor/github.com/klauspost/compress/zstd/enc_best.go
Generated Vendored
+24 -5
View File
@@ -479,10 +479,13 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) {
if d == nil {
return
}
dictChanged := d != e.lastDict
// Init or copy dict table
if len(e.dictTable) != len(e.table) || d.id != e.lastDictID {
if len(e.dictTable) != len(e.table) || dictChanged {
if len(e.dictTable) != len(e.table) {
e.dictTable = make([]prevEntry, len(e.table))
} else {
clear(e.dictTable)
}
end := int32(len(d.content)) - 8 + e.maxMatchOff
for i := e.maxMatchOff; i < end; i += 4 {
@@ -510,13 +513,14 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) {
offset: i + 3,
}
}
e.lastDictID = d.id
}
// Init or copy dict table
if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID {
// Init or copy dict long table
if len(e.dictLongTable) != len(e.longTable) || dictChanged {
if len(e.dictLongTable) != len(e.longTable) {
e.dictLongTable = make([]prevEntry, len(e.longTable))
} else {
clear(e.dictLongTable)
}
if len(d.content) >= 8 {
cv := load6432(d.content, 0)
@@ -538,8 +542,8 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) {
off++
}
}
e.lastDictID = d.id
}
e.lastDict = d
// Reset table to initial state
copy(e.longTable[:], e.dictLongTable)
@@ -547,3 +551,18 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) {
// Reset table to initial state
copy(e.table[:], e.dictTable)
}
func (e *bestFastEncoder) ResetPrefix(prefix []byte) {
e.resetBasePrefix(prefix)
if len(prefix) < 8 {
return
}
end := e.cur + int32(len(prefix)) - 8
for i := e.cur; i < end; i++ {
cv := load6432(prefix, i-e.cur)
h := hashLen(cv, bestLongTableBits, bestLongLen)
e.longTable[h] = prevEntry{offset: i, prev: e.longTable[h].offset}
h0 := hashLen(cv, bestShortTableBits, bestShortLen)
e.table[h0] = prevEntry{offset: i, prev: e.table[h0].offset}
}
}
vendor/github.com/klauspost/compress/zstd/enc_better.go
Generated Vendored
+27 -5
View File
@@ -1096,16 +1096,33 @@ func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) {
}
}
func (e *betterFastEncoder) ResetPrefix(prefix []byte) {
e.resetBasePrefix(prefix)
if len(prefix) < 8 {
return
}
end := e.cur + int32(len(prefix)) - 8
for i := e.cur; i < end; i += 2 {
cv := load6432(prefix, i-e.cur)
h := hashLen(cv, betterLongTableBits, betterLongLen)
e.longTable[h] = prevEntry{offset: i, prev: e.longTable[h].offset}
e.table[hashLen(cv>>8, betterShortTableBits, betterShortLen)] = tableEntry{val: uint32(cv >> 8), offset: i + 1}
}
}
// ResetDict will reset and set a dictionary if not nil
func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) {
e.resetBase(d, singleBlock)
if d == nil {
return
}
dictChanged := d != e.lastDict
// Init or copy dict table
if len(e.dictTable) != len(e.table) || d.id != e.lastDictID {
if len(e.dictTable) != len(e.table) || dictChanged {
if len(e.dictTable) != len(e.table) {
e.dictTable = make([]tableEntry, len(e.table))
} else {
clear(e.dictTable)
}
end := int32(len(d.content)) - 8 + e.maxMatchOff
for i := e.maxMatchOff; i < end; i += 4 {
@@ -1133,14 +1150,15 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) {
offset: i + 3,
}
}
e.lastDictID = d.id
e.allDirty = true
}
// Init or copy dict table
if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID {
// Init or copy dict long table
if len(e.dictLongTable) != len(e.longTable) || dictChanged {
if len(e.dictLongTable) != len(e.longTable) {
e.dictLongTable = make([]prevEntry, len(e.longTable))
} else {
clear(e.dictLongTable)
}
if len(d.content) >= 8 {
cv := load6432(d.content, 0)
@@ -1162,9 +1180,9 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) {
off++
}
}
e.lastDictID = d.id
e.allDirty = true
}
e.lastDict = d
// Reset table to initial state
{
@@ -1225,6 +1243,10 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) {
e.allDirty = false
}
func (e *betterFastEncoderDict) ResetPrefix([]byte) {
panic("ResetPrefix not supported for dict encoders")
}
func (e *betterFastEncoderDict) markLongShardDirty(entryNum uint32) {
e.longTableShardDirty[entryNum/betterLongTableShardSize] = true
}
vendor/github.com/klauspost/compress/zstd/enc_dfast.go
Generated Vendored
+20 -2
View File
@@ -1037,18 +1037,33 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) {
}
}
func (e *doubleFastEncoder) ResetPrefix(prefix []byte) {
e.fastEncoder.ResetPrefix(prefix)
if len(prefix) < 8 {
return
}
end := e.cur + int32(len(prefix)) - 8
for i := e.cur + 1; i < end; i += 2 {
cv := load6432(prefix, i-e.cur)
e.longTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{val: uint32(cv), offset: i}
}
}
// ResetDict will reset and set a dictionary if not nil
func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
allDirty := e.allDirty
dictChanged := d != e.lastDict
e.fastEncoderDict.Reset(d, singleBlock)
if d == nil {
return
}
// Init or copy dict table
if len(e.dictLongTable) != len(e.longTable) || d.id != e.lastDictID {
if len(e.dictLongTable) != len(e.longTable) || dictChanged {
if len(e.dictLongTable) != len(e.longTable) {
e.dictLongTable = make([]tableEntry, len(e.longTable))
} else {
clear(e.dictLongTable)
}
if len(d.content) >= 8 {
cv := load6432(d.content, 0)
@@ -1065,7 +1080,6 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
}
}
}
e.lastDictID = d.id
allDirty = true
}
// Reset table to initial state
@@ -1100,6 +1114,10 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
}
}
func (e *doubleFastEncoderDict) ResetPrefix([]byte) {
panic("ResetPrefix not supported for dict encoders")
}
func (e *doubleFastEncoderDict) markLongShardDirty(entryNum uint32) {
e.longTableShardDirty[entryNum/dLongTableShardSize] = true
}
vendor/github.com/klauspost/compress/zstd/enc_fast.go
Generated Vendored
+21 -2
View File
@@ -797,6 +797,19 @@ func (e *fastEncoder) Reset(d *dict, singleBlock bool) {
}
}
func (e *fastEncoder) ResetPrefix(prefix []byte) {
e.resetBasePrefix(prefix)
if len(prefix) < 8 {
return
}
end := e.cur + int32(len(prefix)) - 8
// Index every 4th
for i := e.cur + 1; i < end; i += 4 {
cv := load6432(prefix, i-e.cur)
e.table[hashLen(cv, tableBits, tableFastHashLen)] = tableEntry{val: uint32(cv), offset: i}
}
}
// ResetDict will reset and set a dictionary if not nil
func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
e.resetBase(d, singleBlock)
@@ -805,9 +818,11 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
}
// Init or copy dict table
if len(e.dictTable) != len(e.table) || d.id != e.lastDictID {
if len(e.dictTable) != len(e.table) || d != e.lastDict {
if len(e.dictTable) != len(e.table) {
e.dictTable = make([]tableEntry, len(e.table))
} else {
clear(e.dictTable)
}
if true {
end := e.maxMatchOff + int32(len(d.content)) - 8
@@ -827,7 +842,7 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
}
}
}
e.lastDictID = d.id
e.lastDict = d
e.allDirty = true
}
@@ -864,6 +879,10 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
e.allDirty = false
}
func (e *fastEncoderDict) ResetPrefix([]byte) {
panic("ResetPrefix not supported for dict encoders")
}
func (e *fastEncoderDict) markAllShardsDirty() {
e.allDirty = true
}
vendor/github.com/klauspost/compress/zstd/enc_jobs.go
Generated Vendored
+352
View File
@@ -0,0 +1,352 @@
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
// Based on work by Yann Collet, released under BSD License.
package zstd
import (
"fmt"
rdebug "runtime/debug"
"sync"
)
type encJob struct {
prefix []byte // overlap from previous job (nil for first)
input []byte // job's own input data (swapped from filling)
last bool // last block of last job gets last=true
output []byte // compressed blocks (filled by worker)
err error // encoding error
done chan struct{} // closed when complete
}
type jobState struct {
jobSize int
overlapSize int
filling []byte // accumulates input up to jobSize
nextPrefix []byte // overlap prefix prepared for the next dispatched job
jobSeq int // next job sequence number
jobCh chan *encJob // dispatch to workers
resultCh chan *encJob // ordered results to flusher
workerWg sync.WaitGroup
flusherWg sync.WaitGroup
mu sync.Mutex
flushedSeq int // last flushed sequence number
cond *sync.Cond
flusherErr error
started bool
inputPool sync.Pool // *[]byte buffers of jobSize cap
outputPool sync.Pool // *[]byte buffers for compressed output
overlapPool sync.Pool // *[]byte buffers for overlap prefixes
}
func (e *Encoder) startJobWorkers() {
js := &e.state.jobs
n := e.o.concurrent
js.jobCh = make(chan *encJob, n)
js.resultCh = make(chan *encJob, n)
js.flushedSeq = 0
js.cond = sync.NewCond(&js.mu)
// Workers borrow encoders from the shared e.encoders pool per-job.
// Ensure the pool is initialized before any worker tries to borrow.
e.init.Do(e.initialize)
for range n {
js.workerWg.Add(1)
go e.jobWorker()
}
js.flusherWg.Add(1)
go e.jobFlusher()
js.started = true
}
func (e *Encoder) jobWorker() {
js := &e.state.jobs
defer js.workerWg.Done()
for job := range js.jobCh {
enc := <-e.encoders
e.compressJob(enc, job)
e.encoders <- enc
close(job.done)
}
}
func (e *Encoder) compressJob(enc encoder, job *encJob) {
defer func() {
if r := recover(); r != nil {
job.err = fmt.Errorf("panic in parallel job: %v", r)
rdebug.PrintStack()
}
}()
if len(job.prefix) > 0 {
enc.ResetPrefix(job.prefix)
} else {
enc.Reset(nil, false)
}
data := job.input
if len(data) == 0 && job.last {
blk := enc.Block()
blk.reset(nil)
blk.last = true
blk.encodeRaw(nil)
job.output = append(job.output, blk.output...)
return
}
blk := enc.Block()
for len(data) > 0 {
todo := data
if len(todo) > e.o.blockSize {
todo = todo[:e.o.blockSize]
}
data = data[len(todo):]
blk.pushOffsets()
enc.Encode(blk, todo)
blk.last = len(data) == 0 && job.last
err := blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy)
if err != nil {
job.err = err
return
}
job.output = append(job.output, blk.output...)
blk.reset(nil)
}
}
func (js *jobState) getInputBuf(size int) []byte {
if v := js.inputPool.Get(); v != nil {
bp := v.(*[]byte)
b := *bp
if cap(b) >= size {
return b[:0]
}
}
return make([]byte, 0, size)
}
func (js *jobState) putInputBuf(b []byte) {
if cap(b) > 0 {
b = b[:0]
js.inputPool.Put(&b)
}
}
func (js *jobState) getOutputBuf(size int) []byte {
if v := js.outputPool.Get(); v != nil {
bp := v.(*[]byte)
b := *bp
if cap(b) >= size {
return b[:0]
}
}
return make([]byte, 0, size)
}
func (js *jobState) putOutputBuf(b []byte) {
if cap(b) > 0 {
b = b[:0]
js.outputPool.Put(&b)
}
}
func (js *jobState) getOverlapBuf(size int) []byte {
if v := js.overlapPool.Get(); v != nil {
bp := v.(*[]byte)
b := *bp
if cap(b) >= size {
return b[:size]
}
}
return make([]byte, size)
}
func (js *jobState) putOverlapBuf(b []byte) {
if cap(b) > 0 {
b = b[:0]
js.overlapPool.Put(&b)
}
}
func (e *Encoder) jobFlusher() {
js := &e.state.jobs
defer js.flusherWg.Done()
for job := range js.resultCh {
<-job.done
// Worker has fully exited compressJob, so the prefix is no longer
// in use. Return it to the pool regardless of outcome.
if job.prefix != nil {
js.putOverlapBuf(job.prefix)
job.prefix = nil
}
if job.err != nil {
js.mu.Lock()
js.flusherErr = job.err
js.cond.Broadcast()
js.mu.Unlock()
for range js.resultCh {
}
return
}
if len(job.output) > 0 {
_, err := e.state.w.Write(job.output)
if err != nil {
js.mu.Lock()
js.flusherErr = err
js.cond.Broadcast()
js.mu.Unlock()
for range js.resultCh {
}
return
}
e.state.nWritten += int64(len(job.output))
}
// Return buffers to pools.
js.putInputBuf(job.input)
js.putOutputBuf(job.output)
job.input = nil
job.output = nil
js.mu.Lock()
js.flushedSeq++
js.cond.Broadcast()
js.mu.Unlock()
}
}
func (e *Encoder) shutdownJobWorkers() {
js := &e.state.jobs
if !js.started {
return
}
close(js.jobCh)
js.workerWg.Wait()
close(js.resultCh)
js.flusherWg.Wait()
js.started = false
}
// waitAllJobs blocks until all dispatched jobs have been flushed.
func (e *Encoder) waitAllJobs() {
js := &e.state.jobs
if !js.started {
return
}
js.mu.Lock()
for js.flushedSeq < js.jobSeq && js.flusherErr == nil {
js.cond.Wait()
}
js.mu.Unlock()
}
func (e *Encoder) dispatchJob(final bool) error {
s := &e.state
js := &s.jobs
js.mu.Lock()
fErr := js.flusherErr
js.mu.Unlock()
if fErr != nil {
return fErr
}
if !s.headerWritten {
// Single-block optimization: fall through to encodeAll path.
if final && len(js.filling) > 0 && len(js.filling) <= e.o.blockSize {
s.current = e.encodeAll(s.encoder, js.filling, s.current[:0])
var n2 int
n2, s.err = s.w.Write(s.current)
if s.err != nil {
return s.err
}
s.nWritten += int64(n2)
s.nInput += int64(len(js.filling))
s.current = s.current[:0]
js.filling = js.filling[:0]
s.headerWritten = true
s.fullFrameWritten = true
s.eofWritten = true
return nil
}
if final && len(js.filling) == 0 && !e.o.fullZero {
s.headerWritten = true
s.fullFrameWritten = true
s.eofWritten = true
return nil
}
var tmp [maxHeaderSize]byte
fh := frameHeader{
ContentSize: uint64(s.frameContentSize),
WindowSize: uint32(s.encoder.WindowSize(s.frameContentSize)),
SingleSegment: false,
Checksum: e.o.crc,
DictID: 0,
}
dst := fh.appendTo(tmp[:0])
var n2 int
n2, s.err = s.w.Write(dst)
if s.err != nil {
return s.err
}
s.nWritten += int64(n2)
s.headerWritten = true
}
if len(js.filling) == 0 && !final {
return nil
}
if !js.started {
e.startJobWorkers()
}
// Estimate output size for pooled buffer.
outputEst := max(len(js.filling)/2, 512)
job := &encJob{
last: final,
done: make(chan struct{}),
output: js.getOutputBuf(outputEst),
}
// Each job owns its prefix slice; the flusher returns it to the pool
// after <-job.done, so workers and dispatch never share a buffer.
if js.nextPrefix != nil {
job.prefix = js.nextPrefix
js.nextPrefix = nil
}
// Build the next job's prefix from the tail of this job's input.
if !final && len(js.filling) > 0 {
overlapLen := min(js.overlapSize, len(js.filling))
np := js.getOverlapBuf(overlapLen)
copy(np, js.filling[len(js.filling)-overlapLen:])
js.nextPrefix = np
}
// Swap filling buffer into job — zero-copy for the input data.
job.input = js.filling
js.filling = js.getInputBuf(js.jobSize)
s.nInput += int64(len(job.input))
js.jobSeq++
if final {
s.eofWritten = true
}
js.resultCh <- job
js.jobCh <- job
return nil
}
vendor/github.com/klauspost/compress/zstd/encoder.go
Generated Vendored
+235 -4
View File
@@ -38,6 +38,7 @@ type encoder interface {
WindowSize(size int64) int32
UseBlock(*blockEnc)
Reset(d *dict, singleBlock bool)
ResetPrefix(prefix []byte)
}
type encoderState struct {
@@ -60,6 +61,9 @@ type encoderState struct {
wg sync.WaitGroup
// This waitgroup indicates we have a block encoding/writing.
wWg sync.WaitGroup
// Parallel job state (used when concurrentBlocks is enabled).
jobs jobState
}
// NewWriter will create a new Zstandard encoder.
@@ -74,6 +78,9 @@ func NewWriter(w io.Writer, opts ...EOption) (*Encoder, error) {
return nil, err
}
}
if e.o.concurrentBlocks && (e.o.dict != nil || e.o.concurrent <= 1) {
e.o.concurrentBlocks = false
}
if w != nil {
e.Reset(w)
}
@@ -95,12 +102,31 @@ func (e *Encoder) initialize() {
// as a new, independent stream.
func (e *Encoder) Reset(w io.Writer) {
s := &e.state
if e.o.concurrentBlocks {
e.shutdownJobWorkers()
js := &s.jobs
js.jobSize = e.o.jobSize()
js.overlapSize = e.o.overlapSize()
// js.filling is allocated lazily on first Write/ReadFrom so callers
// that only use EncodeAll don't pay the (up to ~32 MB) jobSize cost.
js.filling = js.filling[:0]
if js.nextPrefix != nil {
js.putOverlapBuf(js.nextPrefix)
js.nextPrefix = nil
}
js.jobSeq = 0
js.flushedSeq = 0
js.flusherErr = nil
js.started = false
}
s.wg.Wait()
s.wWg.Wait()
if cap(s.filling) == 0 {
s.filling = make([]byte, 0, e.o.blockSize)
}
if e.o.concurrent > 1 {
if e.o.concurrent > 1 && !e.o.concurrentBlocks {
if cap(s.current) == 0 {
s.current = make([]byte, 0, e.o.blockSize)
}
@@ -131,6 +157,32 @@ func (e *Encoder) Reset(w io.Writer) {
s.frameContentSize = 0
}
// ResetWithOptions will re-initialize the writer and apply the given options
// as a new, independent stream.
// Options are applied on top of the existing options.
// Some options cannot be changed on reset and will return an error.
func (e *Encoder) ResetWithOptions(w io.Writer, opts ...EOption) error {
e.o.resetOpt = true
defer func() { e.o.resetOpt = false }()
hadDict := e.o.dict != nil
for _, o := range opts {
if err := o(&e.o); err != nil {
return err
}
}
hasDict := e.o.dict != nil
if e.o.concurrentBlocks && hasDict {
e.o.concurrentBlocks = false
}
if hadDict != hasDict {
// Dict presence changed — encoder type must be recreated.
e.state.encoder = nil
e.init = sync.Once{}
}
e.Reset(w)
return nil
}
// ResetContentSize will reset and set a content size for the next stream.
// If the bytes written does not match the size given an error will be returned
// when calling Close().
@@ -153,6 +205,49 @@ func (e *Encoder) Write(p []byte) (n int, err error) {
if s.eofWritten {
return 0, ErrEncoderClosed
}
if e.o.concurrentBlocks {
return e.writeJobs(p)
}
return e.writeBlocks(p)
}
func (e *Encoder) writeJobs(p []byte) (n int, err error) {
s := &e.state
js := &s.jobs
jobSize := js.jobSize
if cap(js.filling) == 0 && len(p) > 0 {
js.filling = make([]byte, 0, jobSize)
}
for len(p) > 0 {
if len(p)+len(js.filling) < jobSize {
if e.o.crc {
_, _ = s.encoder.CRC().Write(p)
}
js.filling = append(js.filling, p...)
return n + len(p), nil
}
add := p
if len(p)+len(js.filling) > jobSize {
add = add[:jobSize-len(js.filling)]
}
if e.o.crc {
_, _ = s.encoder.CRC().Write(add)
}
js.filling = append(js.filling, add...)
p = p[len(add):]
n += len(add)
if len(js.filling) < jobSize {
return n, nil
}
if err := e.dispatchJob(false); err != nil {
return n, err
}
}
return n, nil
}
func (e *Encoder) writeBlocks(p []byte) (n int, err error) {
s := &e.state
for len(p) > 0 {
if len(p)+len(s.filling) < e.o.blockSize {
if e.o.crc {
@@ -351,6 +446,10 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) {
println("Using ReadFrom")
}
if e.o.concurrentBlocks {
return e.readFromJobs(r)
}
// Flush any current writes.
if len(e.state.filling) > 0 {
if err := e.nextBlock(false); err != nil {
@@ -364,7 +463,6 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) {
if e.o.crc {
_, _ = e.state.encoder.CRC().Write(src[:n2])
}
// src is now the unfilled part...
src = src[n2:]
n += int64(n2)
switch err {
@@ -397,15 +495,63 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) {
}
}
func (e *Encoder) readFromJobs(r io.Reader) (n int64, err error) {
js := &e.state.jobs
jobSize := js.jobSize
// Flush any current filling.
if len(js.filling) > 0 {
if err := e.dispatchJob(false); err != nil {
return 0, err
}
}
if cap(js.filling) < jobSize {
js.filling = make([]byte, 0, jobSize)
}
js.filling = js.filling[:jobSize]
src := js.filling
for {
n2, err := r.Read(src)
if e.o.crc {
_, _ = e.state.encoder.CRC().Write(src[:n2])
}
src = src[n2:]
n += int64(n2)
switch err {
case io.EOF:
js.filling = js.filling[:len(js.filling)-len(src)]
return n, nil
case nil:
default:
e.state.err = err
return n, err
}
if len(src) > 0 {
continue
}
if err = e.dispatchJob(false); err != nil {
return n, err
}
if cap(js.filling) < jobSize {
js.filling = make([]byte, 0, jobSize)
}
js.filling = js.filling[:jobSize]
src = js.filling
}
}
// Flush will send the currently written data to output
// and block until everything has been written.
// This should only be used on rare occasions where pushing the currently queued data is critical.
func (e *Encoder) Flush() error {
s := &e.state
if e.o.concurrentBlocks {
return e.flushJobs()
}
if len(s.filling) > 0 {
err := e.nextBlock(false)
if err != nil {
// Ignore Flush after Close.
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
@@ -415,7 +561,6 @@ func (e *Encoder) Flush() error {
s.wg.Wait()
s.wWg.Wait()
if s.err != nil {
// Ignore Flush after Close.
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
@@ -424,6 +569,20 @@ func (e *Encoder) Flush() error {
return s.writeErr
}
func (e *Encoder) flushJobs() error {
js := &e.state.jobs
if len(js.filling) > 0 {
if err := e.dispatchJob(false); err != nil {
return err
}
}
e.waitAllJobs()
js.mu.Lock()
fErr := js.flusherErr
js.mu.Unlock()
return fErr
}
// Close will flush the final output and close the stream.
// The function will block until everything has been written.
// The Encoder can still be re-used after calling this.
@@ -432,6 +591,16 @@ func (e *Encoder) Close() error {
if s.encoder == nil {
return nil
}
if e.o.concurrentBlocks {
return e.closeJobs()
}
if s.w == nil {
if len(s.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 {
return nil
}
return errors.New("zstd: encoder has no writer")
}
err := e.nextBlock(true)
if err != nil {
if errors.Is(s.err, ErrEncoderClosed) {
@@ -482,6 +651,68 @@ func (e *Encoder) Close() error {
return s.err
}
func (e *Encoder) closeJobs() error {
s := &e.state
js := &s.jobs
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
if s.w == nil {
if len(js.filling) == 0 && !s.headerWritten && !s.eofWritten && s.nInput == 0 {
return nil
}
return errors.New("zstd: encoder has no writer")
}
if err := e.dispatchJob(true); err != nil {
e.shutdownJobWorkers()
if errors.Is(s.err, ErrEncoderClosed) {
return nil
}
return err
}
if s.frameContentSize > 0 && s.nInput != s.frameContentSize {
e.shutdownJobWorkers()
return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput)
}
if s.fullFrameWritten {
e.shutdownJobWorkers()
s.err = ErrEncoderClosed
return nil
}
e.shutdownJobWorkers()
if js.flusherErr != nil {
return js.flusherErr
}
// Write CRC
if e.o.crc {
var tmp [4]byte
_, s.err = s.w.Write(s.encoder.AppendCRC(tmp[:0]))
s.nWritten += 4
}
// Add padding
if s.err == nil && e.o.pad > 0 {
add := calcSkippableFrame(s.nWritten, int64(e.o.pad))
frame, err := skippableFrame(js.filling[:0], add, rand.Reader)
if err != nil {
return err
}
_, s.err = s.w.Write(frame)
}
if s.err == nil {
s.err = ErrEncoderClosed
return nil
}
return s.err
}
// EncodeAll will encode all input in src and append it to dst.
// This function can be called concurrently, but each call will only run on a single goroutine.
// If empty input is given, nothing is returned, unless WithZeroFrames is specified.
vendor/github.com/klauspost/compress/zstd/encoder_options.go
Generated Vendored
+94 -18
View File
@@ -14,21 +14,23 @@ type EOption func(*encoderOptions) error
// options retains accumulated state of multiple options.
type encoderOptions struct {
concurrent int
level EncoderLevel
single *bool
pad int
blockSize int
windowSize int
crc bool
fullZero bool
noEntropy bool
allLitEntropy bool
customWindow bool
customALEntropy bool
customBlockSize bool
lowMem bool
dict *dict
resetOpt bool
concurrent int
level EncoderLevel
single *bool
pad int
blockSize int
windowSize int
crc bool
fullZero bool
noEntropy bool
allLitEntropy bool
customWindow bool
customALEntropy bool
customBlockSize bool
lowMem bool
dict *dict
concurrentBlocks bool
}
func (o *encoderOptions) setDefault() {
@@ -41,6 +43,7 @@ func (o *encoderOptions) setDefault() {
level: SpeedDefault,
allLitEntropy: false,
lowMem: false,
fullZero: true,
}
}
@@ -71,19 +74,28 @@ func (o encoderOptions) encoder() encoder {
// WithEncoderCRC will add CRC value to output.
// Output will be 4 bytes larger.
// Can be changed with ResetWithOptions.
func WithEncoderCRC(b bool) EOption {
return func(o *encoderOptions) error { o.crc = b; return nil }
}
// WithEncoderConcurrency will set the concurrency,
// meaning the maximum number of encoders to run concurrently.
// The value supplied must be at least 1.
// The value supplied must be at least 0.
// When a value of 0 is provided GOMAXPROCS will be used.
// For streams, setting a value of 1 will disable async compression.
// By default this will be set to GOMAXPROCS.
// Cannot be changed with ResetWithOptions.
func WithEncoderConcurrency(n int) EOption {
return func(o *encoderOptions) error {
if n <= 0 {
return fmt.Errorf("concurrency must be at least 1")
if n < 0 {
return errors.New("concurrency must at least 0")
}
if n == 0 {
n = runtime.GOMAXPROCS(0)
}
if o.resetOpt && n != o.concurrent {
return errors.New("WithEncoderConcurrency cannot be changed on Reset")
}
o.concurrent = n
return nil
@@ -95,6 +107,7 @@ func WithEncoderConcurrency(n int) EOption {
// A larger value will enable better compression but allocate more memory and,
// for above-default values, take considerably longer.
// The default value is determined by the compression level and max 8MB.
// Cannot be changed with ResetWithOptions.
func WithWindowSize(n int) EOption {
return func(o *encoderOptions) error {
switch {
@@ -105,6 +118,9 @@ func WithWindowSize(n int) EOption {
case (n & (n - 1)) != 0:
return errors.New("window size must be a power of 2")
}
if o.resetOpt && n != o.windowSize {
return errors.New("WithWindowSize cannot be changed on Reset")
}
o.windowSize = n
o.customWindow = true
@@ -122,6 +138,7 @@ func WithWindowSize(n int) EOption {
// n must be > 0 and <= 1GB, 1<<30 bytes.
// The padded area will be filled with data from crypto/rand.Reader.
// If `EncodeAll` is used with data already in the destination, the total size will be multiple of this.
// Can be changed with ResetWithOptions.
func WithEncoderPadding(n int) EOption {
return func(o *encoderOptions) error {
if n <= 0 {
@@ -215,12 +232,16 @@ func (e EncoderLevel) String() string {
}
// WithEncoderLevel specifies a predefined compression level.
// Cannot be changed with ResetWithOptions.
func WithEncoderLevel(l EncoderLevel) EOption {
return func(o *encoderOptions) error {
switch {
case l <= speedNotSet || l >= speedLast:
return fmt.Errorf("unknown encoder level")
}
if o.resetOpt && l != o.level {
return errors.New("WithEncoderLevel cannot be changed on Reset")
}
o.level = l
if !o.customWindow {
switch o.level {
@@ -248,6 +269,7 @@ func WithEncoderLevel(l EncoderLevel) EOption {
// WithZeroFrames will encode 0 length input as full frames.
// This can be needed for compatibility with zstandard usage,
// but is not needed for this package.
// Can be changed with ResetWithOptions.
func WithZeroFrames(b bool) EOption {
return func(o *encoderOptions) error {
o.fullZero = b
@@ -259,6 +281,7 @@ func WithZeroFrames(b bool) EOption {
// Disabling this will skip incompressible data faster, but in cases with no matches but
// skewed character distribution compression is lost.
// Default value depends on the compression level selected.
// Can be changed with ResetWithOptions.
func WithAllLitEntropyCompression(b bool) EOption {
return func(o *encoderOptions) error {
o.customALEntropy = true
@@ -270,6 +293,7 @@ func WithAllLitEntropyCompression(b bool) EOption {
// WithNoEntropyCompression will always skip entropy compression of literals.
// This can be useful if content has matches, but unlikely to benefit from entropy
// compression. Usually the slight speed improvement is not worth enabling this.
// Can be changed with ResetWithOptions.
func WithNoEntropyCompression(b bool) EOption {
return func(o *encoderOptions) error {
o.noEntropy = b
@@ -287,6 +311,7 @@ func WithNoEntropyCompression(b bool) EOption {
// This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
// If this is not specified, block encodes will automatically choose this based on the input size and the window size.
// This setting has no effect on streamed encodes.
// Can be changed with ResetWithOptions.
func WithSingleSegment(b bool) EOption {
return func(o *encoderOptions) error {
o.single = &b
@@ -298,19 +323,60 @@ func WithSingleSegment(b bool) EOption {
// slower encoding speed.
// This will not change the window size which is the primary function for reducing
// memory usage. See WithWindowSize.
// Cannot be changed with ResetWithOptions.
func WithLowerEncoderMem(b bool) EOption {
return func(o *encoderOptions) error {
if o.resetOpt && b != o.lowMem {
return errors.New("WithLowerEncoderMem cannot be changed on Reset")
}
o.lowMem = b
return nil
}
}
// WithConcurrentBlocks enables job-based parallel compression for streams.
// When enabled and concurrent > 1, input is split into large sections (jobs)
// that are compressed simultaneously by multiple goroutines.
// Each non-first job receives an overlap prefix from the previous job for match context.
// Output is flushed in order, producing a valid single-frame zstd stream.
//
// Currently disabled when used with dictionary encoding.
// Cannot be changed with ResetWithOptions.
func WithConcurrentBlocks(b bool) EOption {
return func(o *encoderOptions) error {
if o.resetOpt && b != o.concurrentBlocks {
return errors.New("WithConcurrentBlocks cannot be changed on Reset")
}
o.concurrentBlocks = b
return nil
}
}
// jobSize returns the input section size per parallel job.
func (o *encoderOptions) jobSize() int {
s := max(o.windowSize*4, 512<<10)
return s
}
// overlapSize returns the overlap prefix size for parallel jobs.
func (o *encoderOptions) overlapSize() int {
switch o.level {
case SpeedBestCompression:
return o.windowSize / 2
case SpeedBetterCompression:
return o.windowSize / 4
default:
return o.windowSize / 8
}
}
// WithEncoderDict allows to register a dictionary that will be used for the encode.
//
// The slice dict must be in the [dictionary format] produced by
// "zstd --train" from the Zstandard reference implementation.
//
// The encoder *may* choose to use no dictionary instead for certain payloads.
// Can be changed with ResetWithOptions.
//
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
func WithEncoderDict(dict []byte) EOption {
@@ -328,6 +394,7 @@ func WithEncoderDict(dict []byte) EOption {
//
// The slice content may contain arbitrary data. It will be used as an initial
// history.
// Can be changed with ResetWithOptions.
func WithEncoderDictRaw(id uint32, content []byte) EOption {
return func(o *encoderOptions) error {
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
@@ -337,3 +404,12 @@ func WithEncoderDictRaw(id uint32, content []byte) EOption {
return nil
}
}
// WithEncoderDictDelete clears the dictionary, so no dictionary will be used.
// Should be used with ResetWithOptions.
func WithEncoderDictDelete() EOption {
return func(o *encoderOptions) error {
o.dict = nil
return nil
}
}
vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.s
Generated Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
// Code generated by command: go run gen_fse.go -out ../fse_decoder_amd64.s -pkg=zstd. DO NOT EDIT.
// Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT.
//go:build !appengine && !noasm && gc && !noasm
vendor/github.com/klauspost/compress/zstd/fse_decoder_arm64.s
Generated Vendored
+146
View File
@@ -0,0 +1,146 @@
// Code generated by command: go run gen_fse.go -out ../fse_decoder.s -arch amd64,arm64 -pkg=zstd. DO NOT EDIT.
// EXPERIMENTAL arm64 output lowered from an amd64 avo program.
//go:build arm64 && (!appengine && !noasm && gc && !noasm)
// func buildDtable_asm(s *fseDecoder, ctx *buildDtableAsmContext) int
TEXT ·buildDtable_asm(SB), $0-24
MOVD ctx+8(FP), R1
MOVD s+0(FP), R6
// Load values
MOVBU 4098(R6), R2
MOVD $0, R0
MOVD $1, R16
LSL R2, R16, R16
ORR R16, R0, R0
MOVD (R1), R3
MOVD 16(R1), R5
SUB $1, R0, R7
MOVD 8(R1), R1
MOVHU 4096(R6), R6
// End load values
// Init, lay down lowprob symbols
MOVD $0, R8
JMP init_main_loop_condition
init_main_loop:
MOVH (R1)(R8<<1), R9
MOVD $-1, R16
AND $0xffff, R16, R16
AND $0xffff, R9, R15
CMP R16, R15
BNE do_not_update_high_threshold
ADD R7<<3, R5, R15
MOVB R8, 1(R15)
SUB $1, R7, R7
MOVD $0x0000000000000001, R9
do_not_update_high_threshold:
MOVH R9, (R3)(R8<<1)
ADD $1, R8, R8
init_main_loop_condition:
CMP R6, R8
BLT init_main_loop
// Spread symbols
// Calculate table step
MOVD R0, R8
LSR $0x01, R8, R8
MOVD R0, R9
LSR $0x03, R9, R9
ADD R9, R8, R8
ADD $3, R8, R8
// Fill add bits values
SUB $1, R0, R9
MOVD $0, R10
MOVD $0, R11
JMP spread_main_loop_condition
spread_main_loop:
MOVD $0, R12
MOVH (R1)(R11<<1), R13
JMP spread_inner_loop_condition
spread_inner_loop:
ADD R10<<3, R5, R15
MOVB R11, 1(R15)
adjust_position:
ADD R8, R10, R10
AND R9, R10, R10
CMP R7, R10
BGT adjust_position
ADD $1, R12, R12
spread_inner_loop_condition:
CMP R13, R12
BLT spread_inner_loop
ADD $1, R11, R11
spread_main_loop_condition:
CMP R6, R11
BLT spread_main_loop
TST R10, R10
BEQ spread_check_ok
MOVD ctx+8(FP), R0
MOVD R10, 24(R0)
MOVD $+1, R16
MOVD R16, ret+16(FP)
RET
spread_check_ok:
// Build Decoding table
MOVD $0, R6
build_table_main_table:
ADD R6<<3, R5, R15
MOVBU 1(R15), R1
MOVHU (R3)(R1<<1), R7
ADD $1, R7, R8
MOVH R8, (R3)(R1<<1)
MOVD R7, R8
CLZ R8, R16
MOVD $63, R8
SUB R16, R8, R8
MOVD R2, R1
SUB R8, R1, R1
LSL R1, R7, R7
SUB R0, R7, R7
ADD R6<<3, R5, R15
MOVB R1, (R15)
ADD R6<<3, R5, R15
MOVH R7, 2(R15)
CMP R0, R7
BLE build_table_check1_ok
MOVD ctx+8(FP), R1
MOVD R7, 24(R1)
MOVD R0, 32(R1)
MOVD $+2, R16
MOVD R16, ret+16(FP)
RET
build_table_check1_ok:
TST $0xff, R1
BNE build_table_check2_ok
AND $0xffff, R6, R16
AND $0xffff, R7, R15
CMP R16, R15
BNE build_table_check2_ok
MOVD ctx+8(FP), R0
MOVD R7, 24(R0)
MOVD R6, 32(R0)
MOVD $+3, R16
MOVD R16, ret+16(FP)
RET
build_table_check2_ok:
ADD $1, R6, R6
CMP R0, R6
BLT build_table_main_table
MOVD $+0, R16
MOVD R16, ret+16(FP)
RET
vendor/github.com/klauspost/compress/zstd/fse_decoder_amd64.go → vendor/github.com/klauspost/compress/zstd/fse_decoder_asm.go
Generated Vendored
+6 -3
View File
@@ -1,5 +1,4 @@
//go:build amd64 && !appengine && !noasm && gc
// +build amd64,!appengine,!noasm,gc
//go:build (amd64 || arm64) && !appengine && !noasm && gc
package zstd
@@ -7,6 +6,10 @@ import (
"fmt"
)
// buildDtable_asm is generated by _generate/gen_fse.go and lowered to each
// architecture (amd64 by goasm, arm64 by the avo arm64 lowering printer). The
// Go side is identical across architectures, so it lives here.
type buildDtableAsmContext struct {
// inputs
stateTable *uint16
@@ -19,7 +22,7 @@ type buildDtableAsmContext struct {
errParam2 uint64
}
// buildDtable_asm is an x86 assembly implementation of fseDecoder.buildDtable.
// buildDtable_asm is an assembly implementation of fseDecoder.buildDtable.
// Function returns non-zero exit code on error.
//
//go:noescape
vendor/github.com/klauspost/compress/zstd/fse_decoder_generic.go
Generated Vendored
+1 -2
View File
@@ -1,5 +1,4 @@
//go:build !amd64 || appengine || !gc || noasm
// +build !amd64 appengine !gc noasm
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
package zstd
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go
Generated Vendored
+12
View File
@@ -52,6 +52,11 @@ func (d *Digest) Reset() {
d.n = 0
}
// maxAsmSize bounds the input handed to a single writeBlocks call. Assembly is
// never preemptible, so an unbounded call holds every P in stop-the-world for
// its duration. Whole 32-byte blocks, so Write can feed it in pieces.
const maxAsmSize = 128 << 10 // 4096 whole 32-byte blocks
// Size always returns 8 bytes.
func (d *Digest) Size() int { return 8 }
@@ -83,6 +88,13 @@ func (d *Digest) Write(b []byte) (n int, err error) {
d.n = 0
}
for len(b) >= maxAsmSize {
// Assembly is not preemptible, so hashing a large buffer in one call
// blocks every stop-the-world for the whole call. Feed it in chunks.
writeBlocks(d, b[:maxAsmSize])
b = b[maxAsmSize:]
}
if len(b) >= 32 {
// One or more full blocks left.
nw := writeBlocks(d, b)
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s
Generated Vendored
+5 -1
View File
@@ -175,7 +175,11 @@ finalize:
RET
// func writeBlocks(d *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
//
// Deliberately not NOSPLIT: the stack check is the preemption point that lets a
// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall
// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check.
TEXT ·writeBlocks(SB), $128-40
// Load fixed primes needed for round.
MOVQ ·primes+0(SB), prime1
MOVQ ·primes+8(SB), prime2
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s
Generated Vendored
+5 -1
View File
@@ -163,7 +163,11 @@ finalize:
RET
// func writeBlocks(s *Digest, b []byte) int
TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40
//
// Deliberately not NOSPLIT: the stack check is the preemption point that lets a
// stop-the-world proceed between chunks. The frame must be >= abi.StackSmall
// (128), or the assembler marks this leaf NOSPLIT anyway and drops that check.
TEXT ·writeBlocks(SB), $128-40
LDP ·primes+0(SB), (prime1, prime2)
// Load state. Assume v[1-4] are stored contiguously.
vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go
Generated Vendored
-1
View File
@@ -1,5 +1,4 @@
//go:build (!amd64 && !arm64) || appengine || !gc || purego || noasm
// +build !amd64,!arm64 appengine !gc purego noasm
package xxhash
vendor/github.com/klauspost/compress/zstd/matchlen_amd64.go
Generated Vendored
-1
View File
@@ -1,5 +1,4 @@
//go:build amd64 && !appengine && !noasm && gc
// +build amd64,!appengine,!noasm,gc
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
vendor/github.com/klauspost/compress/zstd/matchlen_generic.go
Generated Vendored
-1
View File
@@ -1,5 +1,4 @@
//go:build !amd64 || appengine || !gc || noasm
// +build !amd64 appengine !gc noasm
// Copyright 2019+ Klaus Post. All rights reserved.
// License information can be found in the LICENSE file.
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go
Generated Vendored
+51 -334
View File
@@ -1,33 +1,49 @@
//go:build amd64 && !appengine && !noasm && gc
// +build amd64,!appengine,!noasm,gc
package zstd
import (
"fmt"
"io"
"github.com/klauspost/compress/internal/cpuinfo"
)
type decodeSyncAsmContext struct {
llTable []decSymbol
mlTable []decSymbol
ofTable []decSymbol
llState uint64
mlState uint64
ofState uint64
iteration int
litRemain int
out []byte
outPosition int
literals []byte
litPosition int
history []byte
windowSize int
ll int // set on error (not for all errors, please refer to _generate/gen.go)
ml int // set on error (not for all errors, please refer to _generate/gen.go)
mo int // set on error (not for all errors, please refer to _generate/gen.go)
// The shared decode/decodeSync/executeSimple wrappers and context structs live
// in seqdec_asm.go; this file only declares the amd64 asm routines and the
// dispatch helpers that pick the BMI2 / non-BMI2 (and 56-bit / safe) variant.
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
//
// Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape
func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode_56_amd64 implements the main loop of sequenceDecs in x86 asm.
//
//go:noescape
func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode_bmi2 implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
//
//go:noescape
func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode_56_bmi2 implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
//
//go:noescape
func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// decodeAsm runs the sequenceDecs decode loop, choosing the BMI2 / 56-bit variant.
func decodeAsm(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext, lte56bits bool) int {
if cpuinfo.HasBMI2() {
if lte56bits {
return sequenceDecs_decode_56_bmi2(s, br, ctx)
}
return sequenceDecs_decode_bmi2(s, br, ctx)
}
if lte56bits {
return sequenceDecs_decode_56_amd64(s, br, ctx)
}
return sequenceDecs_decode_amd64(s, br, ctx)
}
// sequenceDecs_decodeSync_amd64 implements the main loop of sequenceDecs.decodeSync in x86 asm.
@@ -52,273 +68,18 @@ func sequenceDecs_decodeSync_safe_amd64(s *sequenceDecs, br *bitReader, ctx *dec
//go:noescape
func sequenceDecs_decodeSync_safe_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// decode sequences from the stream with the provided history but without a dictionary.
func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
if len(s.dict) > 0 {
return false, nil
}
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize {
return false, nil
}
// FIXME: Using unsafe memory copies leads to rare, random crashes
// with fuzz testing. It is therefore disabled for now.
const useSafe = true
/*
useSafe := false
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc {
useSafe = true
}
if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) {
useSafe = true
}
if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc {
useSafe = true
}
*/
br := s.br
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
ctx := decodeSyncAsmContext{
llTable: s.litLengths.fse.dt[:maxTablesize],
mlTable: s.matchLengths.fse.dt[:maxTablesize],
ofTable: s.offsets.fse.dt[:maxTablesize],
llState: uint64(s.litLengths.state.state),
mlState: uint64(s.matchLengths.state.state),
ofState: uint64(s.offsets.state.state),
iteration: s.nSeqs - 1,
litRemain: len(s.literals),
out: s.out,
outPosition: len(s.out),
literals: s.literals,
windowSize: s.windowSize,
history: hist,
}
s.seqSize = 0
startSize := len(s.out)
var errCode int
// decodeSyncAsm runs the decodeSync loop, choosing the BMI2 / safe variant.
func decodeSyncAsm(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext, safe bool) int {
if cpuinfo.HasBMI2() {
if useSafe {
errCode = sequenceDecs_decodeSync_safe_bmi2(s, br, &ctx)
} else {
errCode = sequenceDecs_decodeSync_bmi2(s, br, &ctx)
}
} else {
if useSafe {
errCode = sequenceDecs_decodeSync_safe_amd64(s, br, &ctx)
} else {
errCode = sequenceDecs_decodeSync_amd64(s, br, &ctx)
if safe {
return sequenceDecs_decodeSync_safe_bmi2(s, br, ctx)
}
return sequenceDecs_decodeSync_bmi2(s, br, ctx)
}
switch errCode {
case noError:
break
case errorMatchLenOfsMismatch:
return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml)
case errorMatchLenTooBig:
return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml)
case errorMatchOffTooBig:
return true, fmt.Errorf("match offset (%d) bigger than current history (%d)",
ctx.mo, ctx.outPosition+len(hist)-startSize)
case errorNotEnoughLiterals:
return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available",
ctx.ll, ctx.litRemain+ctx.ll)
case errorOverread:
return true, io.ErrUnexpectedEOF
case errorNotEnoughSpace:
size := ctx.outPosition + ctx.ll + ctx.ml
if debugDecoder {
println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize)
}
return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
default:
return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode)
if safe {
return sequenceDecs_decodeSync_safe_amd64(s, br, ctx)
}
s.seqSize += ctx.litRemain
if s.seqSize > maxBlockSize {
return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
}
err := br.close()
if err != nil {
printf("Closing sequences: %v, %+v\n", err, *br)
return true, err
}
s.literals = s.literals[ctx.litPosition:]
t := ctx.outPosition
s.out = s.out[:t]
// Add final literals
s.out = append(s.out, s.literals...)
if debugDecoder {
t += len(s.literals)
if t != len(s.out) {
panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t))
}
}
return true, nil
}
// --------------------------------------------------------------------------------
type decodeAsmContext struct {
llTable []decSymbol
mlTable []decSymbol
ofTable []decSymbol
llState uint64
mlState uint64
ofState uint64
iteration int
seqs []seqVals
litRemain int
}
const noError = 0
// error reported when mo == 0 && ml > 0
const errorMatchLenOfsMismatch = 1
// error reported when ml > maxMatchLen
const errorMatchLenTooBig = 2
// error reported when mo > available history or mo > s.windowSize
const errorMatchOffTooBig = 3
// error reported when the sum of literal lengths exeeceds the literal buffer size
const errorNotEnoughLiterals = 4
// error reported when capacity of `out` is too small
const errorNotEnoughSpace = 5
// error reported when bits are overread.
const errorOverread = 6
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
//
// Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape
func sequenceDecs_decode_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm.
//
// Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape
func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
//
//go:noescape
func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm with BMI2 extensions.
//
//go:noescape
func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// decode sequences from the stream without the provided history.
func (s *sequenceDecs) decode(seqs []seqVals) error {
br := s.br
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
ctx := decodeAsmContext{
llTable: s.litLengths.fse.dt[:maxTablesize],
mlTable: s.matchLengths.fse.dt[:maxTablesize],
ofTable: s.offsets.fse.dt[:maxTablesize],
llState: uint64(s.litLengths.state.state),
mlState: uint64(s.matchLengths.state.state),
ofState: uint64(s.offsets.state.state),
seqs: seqs,
iteration: len(seqs) - 1,
litRemain: len(s.literals),
}
if debugDecoder {
println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream")
}
s.seqSize = 0
lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56
var errCode int
if cpuinfo.HasBMI2() {
if lte56bits {
errCode = sequenceDecs_decode_56_bmi2(s, br, &ctx)
} else {
errCode = sequenceDecs_decode_bmi2(s, br, &ctx)
}
} else {
if lte56bits {
errCode = sequenceDecs_decode_56_amd64(s, br, &ctx)
} else {
errCode = sequenceDecs_decode_amd64(s, br, &ctx)
}
}
if errCode != 0 {
i := len(seqs) - ctx.iteration - 1
switch errCode {
case errorMatchLenOfsMismatch:
ml := ctx.seqs[i].ml
return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
case errorMatchLenTooBig:
ml := ctx.seqs[i].ml
return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
case errorNotEnoughLiterals:
ll := ctx.seqs[i].ll
return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll)
case errorOverread:
return io.ErrUnexpectedEOF
}
return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode)
}
if ctx.litRemain < 0 {
return fmt.Errorf("literal count is too big: total available %d, total requested %d",
len(s.literals), len(s.literals)-ctx.litRemain)
}
s.seqSize += ctx.litRemain
if s.seqSize > maxBlockSize {
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
}
if debugDecoder {
println("decode: ", br.remain(), "bits remain on stream. code:", errCode)
}
err := br.close()
if err != nil {
printf("Closing sequences: %v, %+v\n", err, *br)
}
return err
}
// --------------------------------------------------------------------------------
type executeAsmContext struct {
seqs []seqVals
seqIndex int
out []byte
history []byte
literals []byte
outPosition int
litPosition int
windowSize int
return sequenceDecs_decodeSync_amd64(s, br, ctx)
}
// sequenceDecs_executeSimple_amd64 implements the main loop of sequenceDecs.executeSimple in x86 asm.
@@ -335,54 +96,10 @@ func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool
//go:noescape
func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool
// executeSimple handles cases when dictionary is not used.
func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error {
// Ensure we have enough output size...
if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) {
addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc
s.out = append(s.out, make([]byte, addBytes)...)
s.out = s.out[:len(s.out)-addBytes]
// executeSimpleAsm runs the executeSimple loop, choosing the safe variant.
func executeSimpleAsm(ctx *executeAsmContext, safe bool) bool {
if safe {
return sequenceDecs_executeSimple_safe_amd64(ctx)
}
if debugDecoder {
printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize)
}
var t = len(s.out)
out := s.out[:t+s.seqSize]
ctx := executeAsmContext{
seqs: seqs,
seqIndex: 0,
out: out,
history: hist,
outPosition: t,
litPosition: 0,
literals: s.literals,
windowSize: s.windowSize,
}
var ok bool
if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc {
ok = sequenceDecs_executeSimple_safe_amd64(&ctx)
} else {
ok = sequenceDecs_executeSimple_amd64(&ctx)
}
if !ok {
return fmt.Errorf("match offset (%d) bigger than current history (%d)",
seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist))
}
s.literals = s.literals[ctx.litPosition:]
t = ctx.outPosition
// Add final literals
copy(out[t:], s.literals)
if debugDecoder {
t += len(s.literals)
if t != len(out) {
panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize))
}
}
s.out = out
return nil
return sequenceDecs_executeSimple_amd64(ctx)
}
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s
Generated Vendored
+369 -369
View File
File diff suppressed because it is too large. Load diff
vendor/github.com/klauspost/compress/zstd/seqdec_arm64.go
Generated Vendored
+70
View File
@@ -0,0 +1,70 @@
//go:build arm64 && !appengine && !noasm && gc
package zstd
// The shared decode/decodeSync/executeSimple wrappers and context structs live
// in seqdec_asm.go; this file only declares the arm64 asm routines (generated
// by the avo arm64 lowering printer) and the dispatch helpers. arm64 has no
// BMI2, so each helper selects only between the 56-bit / safe variants.
// sequenceDecs_decode_arm64 implements the main loop of sequenceDecs in arm64 asm.
//
// Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape
func sequenceDecs_decode_arm64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// sequenceDecs_decode_56_arm64 implements the main loop of sequenceDecs in arm64 asm.
//
//go:noescape
func sequenceDecs_decode_56_arm64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int
// decodeAsm runs the sequenceDecs decode loop, choosing the 56-bit variant.
func decodeAsm(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext, lte56bits bool) int {
if lte56bits {
return sequenceDecs_decode_56_arm64(s, br, ctx)
}
return sequenceDecs_decode_arm64(s, br, ctx)
}
// sequenceDecs_decodeSync_arm64 implements the main loop of sequenceDecs.decodeSync in arm64 asm.
//
// Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape
func sequenceDecs_decodeSync_arm64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// sequenceDecs_decodeSync_safe_arm64 does the same as above, but does not write more than output buffer.
//
//go:noescape
func sequenceDecs_decodeSync_safe_arm64(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext) int
// decodeSyncAsm runs the decodeSync loop, choosing the safe variant.
func decodeSyncAsm(s *sequenceDecs, br *bitReader, ctx *decodeSyncAsmContext, safe bool) int {
if safe {
return sequenceDecs_decodeSync_safe_arm64(s, br, ctx)
}
return sequenceDecs_decodeSync_arm64(s, br, ctx)
}
// sequenceDecs_executeSimple_arm64 implements the main loop of sequenceDecs.executeSimple in arm64 asm.
//
// Returns false if a match offset is too big.
//
// Please refer to seqdec_generic.go for the reference implementation.
//
//go:noescape
func sequenceDecs_executeSimple_arm64(ctx *executeAsmContext) bool
// Same as above, but with safe memcopies
//
//go:noescape
func sequenceDecs_executeSimple_safe_arm64(ctx *executeAsmContext) bool
// executeSimpleAsm runs the executeSimple loop, choosing the safe variant.
func executeSimpleAsm(ctx *executeAsmContext, safe bool) bool {
if safe {
return sequenceDecs_executeSimple_safe_arm64(ctx)
}
return sequenceDecs_executeSimple_arm64(ctx)
}
vendor/github.com/klauspost/compress/zstd/seqdec_arm64.s
Generated Vendored
+2686
View File
File diff suppressed because it is too large. Load diff
vendor/github.com/klauspost/compress/zstd/seqdec_asm.go
Generated Vendored
+316
View File
@@ -0,0 +1,316 @@
//go:build (amd64 || arm64) && !appengine && !noasm && gc
package zstd
import (
"fmt"
"io"
)
// This file holds the parts of the assembly sequence decoder that are identical
// across architectures: the context structs exchanged with the asm, the error
// codes, and the decode/decodeSync/executeSimple wrappers. Each architecture
// supplies the small dispatch helpers (decodeAsm, decodeSyncAsm,
// executeSimpleAsm) that select the concrete asm routine — amd64 also chooses a
// BMI2 variant, arm64 has a single implementation.
type decodeSyncAsmContext struct {
llTable []decSymbol
mlTable []decSymbol
ofTable []decSymbol
llState uint64
mlState uint64
ofState uint64
iteration int
litRemain int
out []byte
outPosition int
literals []byte
litPosition int
history []byte
windowSize int
ll int // set on error (not for all errors, please refer to _generate/gen.go)
ml int // set on error (not for all errors, please refer to _generate/gen.go)
mo int // set on error (not for all errors, please refer to _generate/gen.go)
}
type decodeAsmContext struct {
llTable []decSymbol
mlTable []decSymbol
ofTable []decSymbol
llState uint64
mlState uint64
ofState uint64
iteration int
seqs []seqVals
litRemain int
}
type executeAsmContext struct {
seqs []seqVals
seqIndex int
out []byte
history []byte
literals []byte
outPosition int
litPosition int
windowSize int
}
const noError = 0
// error reported when mo == 0 && ml > 0
const errorMatchLenOfsMismatch = 1
// error reported when ml > maxMatchLen
const errorMatchLenTooBig = 2
// error reported when mo > available history or mo > s.windowSize
const errorMatchOffTooBig = 3
// error reported when the sum of literal lengths exeeceds the literal buffer size
const errorNotEnoughLiterals = 4
// error reported when capacity of `out` is too small
const errorNotEnoughSpace = 5
// error reported when bits are overread.
const errorOverread = 6
// useSafeDecodeSync reports whether decodeSyncSimple must use the bounds-exact
// ("safe") copy variants for the current buffer geometry.
//
// When the output and literal buffers have compressedBlockOverAlloc (16)
// bytes of slack past their logical use, the assembly may use extended
// memory copies that read and write in 16-byte blocks, overrunning the end
// of a literal run or match by up to 15 bytes. Otherwise it must use the
// bounds-exact ("safe") copies. This mirrors the analogous, always-dynamic
// selection in executeSimple below.
//
// The unsafe copies were disabled in #644 (2022) as a mitigation for a
// crash, but that crash's root cause — an unguarded bitReader overread that
// produced out-of-range match offsets/lengths — was fixed three days later
// in #645, which also added the fuzz corpus that has guarded this path since.
// See #1168. An asan-instrumented fuzz job (see .github/workflows/go.yml)
// covers the extended-copy path, which -race and plain fuzzing cannot.
func (s *sequenceDecs) useSafeDecodeSync() bool {
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSizeAlloc {
return true
}
if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) {
return true
}
if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc {
return true
}
return false
}
// decode sequences from the stream with the provided history but without a dictionary.
func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
if len(s.dict) > 0 {
return false, nil
}
if s.maxSyncLen == 0 && cap(s.out)-len(s.out) < maxCompressedBlockSize {
return false, nil
}
useSafe := s.useSafeDecodeSync()
br := s.br
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
ctx := decodeSyncAsmContext{
llTable: s.litLengths.fse.dt[:maxTablesize],
mlTable: s.matchLengths.fse.dt[:maxTablesize],
ofTable: s.offsets.fse.dt[:maxTablesize],
llState: uint64(s.litLengths.state.state),
mlState: uint64(s.matchLengths.state.state),
ofState: uint64(s.offsets.state.state),
iteration: s.nSeqs - 1,
litRemain: len(s.literals),
out: s.out,
outPosition: len(s.out),
literals: s.literals,
windowSize: s.windowSize,
history: hist,
}
s.seqSize = 0
startSize := len(s.out)
errCode := decodeSyncAsm(s, br, &ctx, useSafe)
switch errCode {
case noError:
break
case errorMatchLenOfsMismatch:
return true, fmt.Errorf("zero matchoff and matchlen (%d) > 0", ctx.ml)
case errorMatchLenTooBig:
return true, fmt.Errorf("match len (%d) bigger than max allowed length", ctx.ml)
case errorMatchOffTooBig:
return true, fmt.Errorf("match offset (%d) bigger than current history (%d)",
ctx.mo, ctx.outPosition+len(hist)-startSize)
case errorNotEnoughLiterals:
return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available",
ctx.ll, ctx.litRemain+ctx.ll)
case errorOverread:
return true, io.ErrUnexpectedEOF
case errorNotEnoughSpace:
size := ctx.outPosition + ctx.ll + ctx.ml
if debugDecoder {
println("msl:", s.maxSyncLen, "cap", cap(s.out), "bef:", startSize, "sz:", size-startSize, "mbs:", maxBlockSize, "outsz:", cap(s.out)-startSize)
}
return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
default:
return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode)
}
s.seqSize += ctx.litRemain
if s.seqSize > maxBlockSize {
return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
}
err := br.close()
if err != nil {
printf("Closing sequences: %v, %+v\n", err, *br)
return true, err
}
s.literals = s.literals[ctx.litPosition:]
t := ctx.outPosition
s.out = s.out[:t]
// Add final literals
s.out = append(s.out, s.literals...)
if debugDecoder {
t += len(s.literals)
if t != len(s.out) {
panic(fmt.Errorf("length mismatch, want %d, got %d", len(s.out), t))
}
}
return true, nil
}
// decode sequences from the stream without the provided history.
func (s *sequenceDecs) decode(seqs []seqVals) error {
br := s.br
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
ctx := decodeAsmContext{
llTable: s.litLengths.fse.dt[:maxTablesize],
mlTable: s.matchLengths.fse.dt[:maxTablesize],
ofTable: s.offsets.fse.dt[:maxTablesize],
llState: uint64(s.litLengths.state.state),
mlState: uint64(s.matchLengths.state.state),
ofState: uint64(s.offsets.state.state),
seqs: seqs,
iteration: len(seqs) - 1,
litRemain: len(s.literals),
}
if debugDecoder {
println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream")
}
s.seqSize = 0
lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56
errCode := decodeAsm(s, br, &ctx, lte56bits)
if errCode != 0 {
i := len(seqs) - ctx.iteration - 1
switch errCode {
case errorMatchLenOfsMismatch:
ml := ctx.seqs[i].ml
return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
case errorMatchLenTooBig:
ml := ctx.seqs[i].ml
return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
case errorNotEnoughLiterals:
ll := ctx.seqs[i].ll
return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll)
case errorOverread:
return io.ErrUnexpectedEOF
}
return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode)
}
if ctx.litRemain < 0 {
return fmt.Errorf("literal count is too big: total available %d, total requested %d",
len(s.literals), len(s.literals)-ctx.litRemain)
}
s.seqSize += ctx.litRemain
if s.seqSize > maxBlockSize {
return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize)
}
if debugDecoder {
println("decode: ", br.remain(), "bits remain on stream. code:", errCode)
}
err := br.close()
if err != nil {
printf("Closing sequences: %v, %+v\n", err, *br)
}
return err
}
// executeSimple handles cases when dictionary is not used.
func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error {
// Ensure we have enough output size...
if len(s.out)+s.seqSize+compressedBlockOverAlloc > cap(s.out) {
addBytes := s.seqSize + len(s.out) + compressedBlockOverAlloc
s.out = append(s.out, make([]byte, addBytes)...)
s.out = s.out[:len(s.out)-addBytes]
}
if debugDecoder {
printf("Execute %d seqs with literals: %d into %d bytes\n", len(seqs), len(s.literals), s.seqSize)
}
var t = len(s.out)
out := s.out[:t+s.seqSize]
ctx := executeAsmContext{
seqs: seqs,
seqIndex: 0,
out: out,
history: hist,
outPosition: t,
litPosition: 0,
literals: s.literals,
windowSize: s.windowSize,
}
// useSafe avoids overwriting the output buffer when the literals slice has
// not been allocated with the required over-allocation slack.
useSafe := cap(s.literals) < len(s.literals)+compressedBlockOverAlloc
ok := executeSimpleAsm(&ctx, useSafe)
if !ok {
return fmt.Errorf("match offset (%d) bigger than current history (%d)",
seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist))
}
s.literals = s.literals[ctx.litPosition:]
t = ctx.outPosition
// Add final literals
copy(out[t:], s.literals)
if debugDecoder {
t += len(s.literals)
if t != len(out) {
panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize))
}
}
s.out = out
return nil
}
vendor/github.com/klauspost/compress/zstd/seqdec_generic.go
Generated Vendored
+1 -2
View File
@@ -1,5 +1,4 @@
//go:build !amd64 || appengine || !gc || noasm
// +build !amd64 appengine !gc noasm
//go:build (!amd64 && !arm64) || appengine || !gc || noasm
package zstd
vendor/github.com/klauspost/compress/zstd/snappy.go
Generated Vendored
+4 -3
View File
@@ -334,9 +334,10 @@ func decodeSnappy(blk *blockEnc, src []byte) error {
return errUnsupportedLiteralLength
}
//if length > snappyMaxBlockSize-d || uint32(length) > len(src)-s {
// return ErrSnappyCorrupt
//}
if length > len(src)-s {
println("length > len(src)-s", length, len(src)-s)
return ErrSnappyCorrupt
}
blk.literals = append(blk.literals, src[s:s+length]...)
//println(length, "litLen")