Update
This commit is contained in:
157
vendor/github.com/klauspost/compress/README.md
generated
vendored
157
vendor/github.com/klauspost/compress/README.md
generated
vendored
@@ -14,8 +14,47 @@ This package provides various compression algorithms.
|
||||
[](https://github.com/klauspost/compress/actions/workflows/go.yml)
|
||||
[](https://sourcegraph.com/github.com/klauspost/compress?badge)
|
||||
|
||||
# package usage
|
||||
|
||||
Use `go get github.com/klauspost/compress@latest` to add it to your project.
|
||||
|
||||
This package will support the current Go version and 2 versions back.
|
||||
|
||||
* Use the `nounsafe` tag to disable all use of the "unsafe" package.
|
||||
* Use the `noasm` tag to disable all assembly across packages.
|
||||
|
||||
Use the links above for more information on each.
|
||||
|
||||
# changelog
|
||||
|
||||
* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1)
|
||||
* zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079
|
||||
* zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059
|
||||
* s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080
|
||||
* zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086
|
||||
* gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090
|
||||
* flate: Faster load+store https://github.com/klauspost/compress/pull/1104
|
||||
* flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101
|
||||
* flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103
|
||||
|
||||
* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0)
|
||||
* Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036
|
||||
* fix: check `r.err != nil` but return a nil value error `err` by @alingse in https://github.com/klauspost/compress/pull/1028
|
||||
* flate: Simplify L4-6 loading https://github.com/klauspost/compress/pull/1043
|
||||
* flate: Simplify matchlen (remove asm) https://github.com/klauspost/compress/pull/1045
|
||||
* s2: Improve small block compression speed w/o asm https://github.com/klauspost/compress/pull/1048
|
||||
* flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049
|
||||
* flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050
|
||||
|
||||
<details>
|
||||
<summary>See changes to v1.17.x</summary>
|
||||
|
||||
* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11)
|
||||
* zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017
|
||||
* s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014
|
||||
* gzhttp: No content-type on no body response code by @juliens in https://github.com/klauspost/compress/pull/1011
|
||||
* gzhttp: Do not set the content-type when response has no body by @kevinpollet in https://github.com/klauspost/compress/pull/1013
|
||||
|
||||
* Sep 23rd, 2024 - [1.17.10](https://github.com/klauspost/compress/releases/tag/v1.17.10)
|
||||
* gzhttp: Add TransportAlwaysDecompress option. https://github.com/klauspost/compress/pull/978
|
||||
* gzhttp: Add supported decompress request body by @mirecl in https://github.com/klauspost/compress/pull/1002
|
||||
@@ -65,9 +104,9 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
|
||||
* zstd: Fix rare *CORRUPTION* output in "best" mode. See https://github.com/klauspost/compress/pull/876
|
||||
|
||||
* Oct 14th, 2023 - [v1.17.1](https://github.com/klauspost/compress/releases/tag/v1.17.1)
|
||||
* s2: Fix S2 "best" dictionary wrong encoding by @klauspost in https://github.com/klauspost/compress/pull/871
|
||||
* s2: Fix S2 "best" dictionary wrong encoding https://github.com/klauspost/compress/pull/871
|
||||
* flate: Reduce allocations in decompressor and minor code improvements by @fakefloordiv in https://github.com/klauspost/compress/pull/869
|
||||
* s2: Fix EstimateBlockSize on 6&7 length input by @klauspost in https://github.com/klauspost/compress/pull/867
|
||||
* s2: Fix EstimateBlockSize on 6&7 length input https://github.com/klauspost/compress/pull/867
|
||||
|
||||
* Sept 19th, 2023 - [v1.17.0](https://github.com/klauspost/compress/releases/tag/v1.17.0)
|
||||
* Add experimental dictionary builder https://github.com/klauspost/compress/pull/853
|
||||
@@ -76,7 +115,8 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
|
||||
* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
|
||||
* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
|
||||
* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
|
||||
|
||||
|
||||
</details>
|
||||
<details>
|
||||
<summary>See changes to v1.16.x</summary>
|
||||
|
||||
@@ -124,7 +164,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
|
||||
<summary>See changes to v1.15.x</summary>
|
||||
|
||||
* Jan 21st, 2023 (v1.15.15)
|
||||
* deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739
|
||||
* deflate: Improve level 7-9 https://github.com/klauspost/compress/pull/739
|
||||
* zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728
|
||||
* zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745
|
||||
* gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740
|
||||
@@ -167,7 +207,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
|
||||
|
||||
* zstd: Fix decoder crash on amd64 (no BMI) on invalid input https://github.com/klauspost/compress/pull/645
|
||||
* zstd: Disable decoder extended memory copies (amd64) due to possible crashes https://github.com/klauspost/compress/pull/644
|
||||
* zstd: Allow single segments up to "max decoded size" by @klauspost in https://github.com/klauspost/compress/pull/643
|
||||
* zstd: Allow single segments up to "max decoded size" https://github.com/klauspost/compress/pull/643
|
||||
|
||||
* July 13, 2022 (v1.15.8)
|
||||
|
||||
@@ -209,7 +249,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
|
||||
* zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599
|
||||
* zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593
|
||||
* huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586
|
||||
* flate: Inplace hashing for level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/590
|
||||
* flate: Inplace hashing for level 7-9 https://github.com/klauspost/compress/pull/590
|
||||
|
||||
|
||||
* May 11, 2022 (v1.15.4)
|
||||
@@ -236,12 +276,12 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
|
||||
* zstd: Add stricter block size checks in [#523](https://github.com/klauspost/compress/pull/523)
|
||||
|
||||
* Mar 3, 2022 (v1.15.0)
|
||||
* zstd: Refactor decoder by @klauspost in [#498](https://github.com/klauspost/compress/pull/498)
|
||||
* zstd: Add stream encoding without goroutines by @klauspost in [#505](https://github.com/klauspost/compress/pull/505)
|
||||
* zstd: Refactor decoder [#498](https://github.com/klauspost/compress/pull/498)
|
||||
* zstd: Add stream encoding without goroutines [#505](https://github.com/klauspost/compress/pull/505)
|
||||
* huff0: Prevent single blocks exceeding 16 bits by @klauspost in[#507](https://github.com/klauspost/compress/pull/507)
|
||||
* flate: Inline literal emission by @klauspost in [#509](https://github.com/klauspost/compress/pull/509)
|
||||
* gzhttp: Add zstd to transport by @klauspost in [#400](https://github.com/klauspost/compress/pull/400)
|
||||
* gzhttp: Make content-type optional by @klauspost in [#510](https://github.com/klauspost/compress/pull/510)
|
||||
* flate: Inline literal emission [#509](https://github.com/klauspost/compress/pull/509)
|
||||
* gzhttp: Add zstd to transport [#400](https://github.com/klauspost/compress/pull/400)
|
||||
* gzhttp: Make content-type optional [#510](https://github.com/klauspost/compress/pull/510)
|
||||
|
||||
Both compression and decompression now supports "synchronous" stream operations. This means that whenever "concurrency" is set to 1, they will operate without spawning goroutines.
|
||||
|
||||
@@ -258,7 +298,7 @@ While the release has been extensively tested, it is recommended to testing when
|
||||
* flate: Fix rare huffman only (-2) corruption. [#503](https://github.com/klauspost/compress/pull/503)
|
||||
* zip: Update deprecated CreateHeaderRaw to correctly call CreateRaw by @saracen in [#502](https://github.com/klauspost/compress/pull/502)
|
||||
* zip: don't read data descriptor early by @saracen in [#501](https://github.com/klauspost/compress/pull/501) #501
|
||||
* huff0: Use static decompression buffer up to 30% faster by @klauspost in [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500)
|
||||
* huff0: Use static decompression buffer up to 30% faster [#499](https://github.com/klauspost/compress/pull/499) [#500](https://github.com/klauspost/compress/pull/500)
|
||||
|
||||
* Feb 17, 2022 (v1.14.3)
|
||||
* flate: Improve fastest levels compression speed ~10% more throughput. [#482](https://github.com/klauspost/compress/pull/482) [#489](https://github.com/klauspost/compress/pull/489) [#490](https://github.com/klauspost/compress/pull/490) [#491](https://github.com/klauspost/compress/pull/491) [#494](https://github.com/klauspost/compress/pull/494) [#478](https://github.com/klauspost/compress/pull/478)
|
||||
@@ -565,12 +605,14 @@ While the release has been extensively tested, it is recommended to testing when
|
||||
|
||||
The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:
|
||||
|
||||
| old import | new import | Documentation
|
||||
|--------------------|-----------------------------------------|--------------------|
|
||||
| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc)
|
||||
| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc)
|
||||
| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc)
|
||||
| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc)
|
||||
Typical speed is about 2x of the standard library packages.
|
||||
|
||||
| old import | new import | Documentation |
|
||||
|------------------|---------------------------------------|-------------------------------------------------------------------------|
|
||||
| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc) |
|
||||
| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc) |
|
||||
| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) |
|
||||
| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) |
|
||||
|
||||
* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
|
||||
|
||||
@@ -625,84 +667,6 @@ This will only use up to 4KB in memory when the writer is idle.
|
||||
Compression is almost always worse than the fastest compression level
|
||||
and each write will allocate (a little) memory.
|
||||
|
||||
# Performance Update 2018
|
||||
|
||||
It has been a while since we have been looking at the speed of this package compared to the standard library, so I thought I would re-do my tests and give some overall recommendations based on the current state. All benchmarks have been performed with Go 1.10 on my Desktop Intel(R) Core(TM) i7-2600 CPU @3.40GHz. Since I last ran the tests, I have gotten more RAM, which means tests with big files are no longer limited by my SSD.
|
||||
|
||||
The raw results are in my [updated spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). Due to cgo changes and upstream updates i could not get the cgo version of gzip to compile. Instead I included the [zstd](https://github.com/datadog/zstd) cgo implementation. If I get cgo gzip to work again, I might replace the results in the sheet.
|
||||
|
||||
The columns to take note of are: *MB/s* - the throughput. *Reduction* - the data size reduction in percent of the original. *Rel Speed* relative speed compared to the standard library at the same level. *Smaller* - how many percent smaller is the compressed output compared to stdlib. Negative means the output was bigger. *Loss* means the loss (or gain) in compression as a percentage difference of the input.
|
||||
|
||||
The `gzstd` (standard library gzip) and `gzkp` (this package gzip) only uses one CPU core. [`pgzip`](https://github.com/klauspost/pgzip), [`bgzf`](https://github.com/biogo/hts/tree/master/bgzf) uses all 4 cores. [`zstd`](https://github.com/DataDog/zstd) uses one core, and is a beast (but not Go, yet).
|
||||
|
||||
|
||||
## Overall differences.
|
||||
|
||||
There appears to be a roughly 5-10% speed advantage over the standard library when comparing at similar compression levels.
|
||||
|
||||
The biggest difference you will see is the result of [re-balancing](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) the compression levels. I wanted by library to give a smoother transition between the compression levels than the standard library.
|
||||
|
||||
This package attempts to provide a more smooth transition, where "1" is taking a lot of shortcuts, "5" is the reasonable trade-off and "9" is the "give me the best compression", and the values in between gives something reasonable in between. The standard library has big differences in levels 1-4, but levels 5-9 having no significant gains - often spending a lot more time than can be justified by the achieved compression.
|
||||
|
||||
There are links to all the test data in the [spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) in the top left field on each tab.
|
||||
|
||||
## Web Content
|
||||
|
||||
This test set aims to emulate typical use in a web server. The test-set is 4GB data in 53k files, and is a mixture of (mostly) HTML, JS, CSS.
|
||||
|
||||
Since level 1 and 9 are close to being the same code, they are quite close. But looking at the levels in-between the differences are quite big.
|
||||
|
||||
Looking at level 6, this package is 88% faster, but will output about 6% more data. For a web server, this means you can serve 88% more data, but have to pay for 6% more bandwidth. You can draw your own conclusions on what would be the most expensive for your case.
|
||||
|
||||
## Object files
|
||||
|
||||
This test is for typical data files stored on a server. In this case it is a collection of Go precompiled objects. They are very compressible.
|
||||
|
||||
The picture is similar to the web content, but with small differences since this is very compressible. Levels 2-3 offer good speed, but is sacrificing quite a bit of compression.
|
||||
|
||||
The standard library seems suboptimal on level 3 and 4 - offering both worse compression and speed than level 6 & 7 of this package respectively.
|
||||
|
||||
## Highly Compressible File
|
||||
|
||||
This is a JSON file with very high redundancy. The reduction starts at 95% on level 1, so in real life terms we are dealing with something like a highly redundant stream of data, etc.
|
||||
|
||||
It is definitely visible that we are dealing with specialized content here, so the results are very scattered. This package does not do very well at levels 1-4, but picks up significantly at level 5 and levels 7 and 8 offering great speed for the achieved compression.
|
||||
|
||||
So if you know you content is extremely compressible you might want to go slightly higher than the defaults. The standard library has a huge gap between levels 3 and 4 in terms of speed (2.75x slowdown), so it offers little "middle ground".
|
||||
|
||||
## Medium-High Compressible
|
||||
|
||||
This is a pretty common test corpus: [enwik9](http://mattmahoney.net/dc/textdata.html). It contains the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. This is a very good test of typical text based compression and more data heavy streams.
|
||||
|
||||
We see a similar picture here as in "Web Content". On equal levels some compression is sacrificed for more speed. Level 5 seems to be the best trade-off between speed and size, beating stdlib level 3 in both.
|
||||
|
||||
## Medium Compressible
|
||||
|
||||
I will combine two test sets, one [10GB file set](http://mattmahoney.net/dc/10gb.html) and a VM disk image (~8GB). Both contain different data types and represent a typical backup scenario.
|
||||
|
||||
The most notable thing is how quickly the standard library drops to very low compression speeds around level 5-6 without any big gains in compression. Since this type of data is fairly common, this does not seem like good behavior.
|
||||
|
||||
|
||||
## Un-compressible Content
|
||||
|
||||
This is mainly a test of how good the algorithms are at detecting un-compressible input. The standard library only offers this feature with very conservative settings at level 1. Obviously there is no reason for the algorithms to try to compress input that cannot be compressed. The only downside is that it might skip some compressible data on false detections.
|
||||
|
||||
|
||||
## Huffman only compression
|
||||
|
||||
This compression library adds a special compression level, named `HuffmanOnly`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character.
|
||||
|
||||
This means that often used characters, like 'e' and ' ' (space) in text use the fewest bits to represent, and rare characters like '¤' takes more bits to represent. For more information see [wikipedia](https://en.wikipedia.org/wiki/Huffman_coding) or this nice [video](https://youtu.be/ZdooBTdW5bM).
|
||||
|
||||
Since this type of compression has much less variance, the compression speed is mostly unaffected by the input data, and is usually more than *180MB/s* for a single core.
|
||||
|
||||
The downside is that the compression ratio is usually considerably worse than even the fastest conventional compression. The compression ratio can never be better than 8:1 (12.5%).
|
||||
|
||||
The linear time compression can be used as a "better than nothing" mode, where you cannot risk the encoder to slow down on some content. For comparison, the size of the "Twain" text is *233460 bytes* (+29% vs. level 1) and encode speed is 144MB/s (4.5x level 1). So in this case you trade a 30% size increase for a 4 times speedup.
|
||||
|
||||
For more information see my blog post on [Fast Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
|
||||
|
||||
This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
|
||||
|
||||
# Other packages
|
||||
|
||||
@@ -719,3 +683,4 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv
|
||||
# license
|
||||
|
||||
This code is licensed under the same conditions as the original Go code. See LICENSE file.
|
||||
|
||||
|
||||
2
vendor/github.com/klauspost/compress/fse/bitwriter.go
generated
vendored
2
vendor/github.com/klauspost/compress/fse/bitwriter.go
generated
vendored
@@ -143,7 +143,7 @@ func (b *bitWriter) flush32() {
|
||||
// flushAlign will flush remaining full bytes and align to next byte boundary.
|
||||
func (b *bitWriter) flushAlign() {
|
||||
nbBytes := (b.nBits + 7) >> 3
|
||||
for i := uint8(0); i < nbBytes; i++ {
|
||||
for i := range nbBytes {
|
||||
b.out = append(b.out, byte(b.bitContainer>>(i*8)))
|
||||
}
|
||||
b.nBits = 0
|
||||
|
||||
2
vendor/github.com/klauspost/compress/fse/compress.go
generated
vendored
2
vendor/github.com/klauspost/compress/fse/compress.go
generated
vendored
@@ -396,7 +396,7 @@ func (s *Scratch) buildCTable() error {
|
||||
if v > largeLimit {
|
||||
s.zeroBits = true
|
||||
}
|
||||
for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ {
|
||||
for range v {
|
||||
tableSymbol[position] = symbol
|
||||
position = (position + step) & tableMask
|
||||
for position > highThreshold {
|
||||
|
||||
25
vendor/github.com/klauspost/compress/huff0/bitreader.go
generated
vendored
25
vendor/github.com/klauspost/compress/huff0/bitreader.go
generated
vendored
@@ -6,10 +6,11 @@
|
||||
package huff0
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// bitReader reads a bitstream in reverse.
|
||||
@@ -46,7 +47,7 @@ func (b *bitReaderBytes) init(in []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// peekBitsFast requires that at least one bit is requested every time.
|
||||
// peekByteFast requires that at least one byte is requested every time.
|
||||
// There are no checks if the buffer is filled.
|
||||
func (b *bitReaderBytes) peekByteFast() uint8 {
|
||||
got := uint8(b.value >> 56)
|
||||
@@ -66,8 +67,7 @@ func (b *bitReaderBytes) fillFast() {
|
||||
}
|
||||
|
||||
// 2 bounds checks.
|
||||
v := b.in[b.off-4 : b.off]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
low := le.Load32(b.in, b.off-4)
|
||||
b.value |= uint64(low) << (b.bitsRead - 32)
|
||||
b.bitsRead -= 32
|
||||
b.off -= 4
|
||||
@@ -76,7 +76,7 @@ func (b *bitReaderBytes) fillFast() {
|
||||
// fillFastStart() assumes the bitReaderBytes is empty and there is at least 8 bytes to read.
|
||||
func (b *bitReaderBytes) fillFastStart() {
|
||||
// Do single re-slice to avoid bounds checks.
|
||||
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:])
|
||||
b.value = le.Load64(b.in, b.off-8)
|
||||
b.bitsRead = 0
|
||||
b.off -= 8
|
||||
}
|
||||
@@ -86,9 +86,8 @@ func (b *bitReaderBytes) fill() {
|
||||
if b.bitsRead < 32 {
|
||||
return
|
||||
}
|
||||
if b.off > 4 {
|
||||
v := b.in[b.off-4 : b.off]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
if b.off >= 4 {
|
||||
low := le.Load32(b.in, b.off-4)
|
||||
b.value |= uint64(low) << (b.bitsRead - 32)
|
||||
b.bitsRead -= 32
|
||||
b.off -= 4
|
||||
@@ -175,9 +174,7 @@ func (b *bitReaderShifted) fillFast() {
|
||||
return
|
||||
}
|
||||
|
||||
// 2 bounds checks.
|
||||
v := b.in[b.off-4 : b.off]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
low := le.Load32(b.in, b.off-4)
|
||||
b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
|
||||
b.bitsRead -= 32
|
||||
b.off -= 4
|
||||
@@ -185,8 +182,7 @@ func (b *bitReaderShifted) fillFast() {
|
||||
|
||||
// fillFastStart() assumes the bitReaderShifted is empty and there is at least 8 bytes to read.
|
||||
func (b *bitReaderShifted) fillFastStart() {
|
||||
// Do single re-slice to avoid bounds checks.
|
||||
b.value = binary.LittleEndian.Uint64(b.in[b.off-8:])
|
||||
b.value = le.Load64(b.in, b.off-8)
|
||||
b.bitsRead = 0
|
||||
b.off -= 8
|
||||
}
|
||||
@@ -197,8 +193,7 @@ func (b *bitReaderShifted) fill() {
|
||||
return
|
||||
}
|
||||
if b.off > 4 {
|
||||
v := b.in[b.off-4 : b.off]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
low := le.Load32(b.in, b.off-4)
|
||||
b.value |= uint64(low) << ((b.bitsRead - 32) & 63)
|
||||
b.bitsRead -= 32
|
||||
b.off -= 4
|
||||
|
||||
2
vendor/github.com/klauspost/compress/huff0/bitwriter.go
generated
vendored
2
vendor/github.com/klauspost/compress/huff0/bitwriter.go
generated
vendored
@@ -85,7 +85,7 @@ func (b *bitWriter) flush32() {
|
||||
// flushAlign will flush remaining full bytes and align to next byte boundary.
|
||||
func (b *bitWriter) flushAlign() {
|
||||
nbBytes := (b.nBits + 7) >> 3
|
||||
for i := uint8(0); i < nbBytes; i++ {
|
||||
for i := range nbBytes {
|
||||
b.out = append(b.out, byte(b.bitContainer>>(i*8)))
|
||||
}
|
||||
b.nBits = 0
|
||||
|
||||
6
vendor/github.com/klauspost/compress/huff0/compress.go
generated
vendored
6
vendor/github.com/klauspost/compress/huff0/compress.go
generated
vendored
@@ -276,7 +276,7 @@ func (s *Scratch) compress4X(src []byte) ([]byte, error) {
|
||||
offsetIdx := len(s.Out)
|
||||
s.Out = append(s.Out, sixZeros[:]...)
|
||||
|
||||
for i := 0; i < 4; i++ {
|
||||
for i := range 4 {
|
||||
toDo := src
|
||||
if len(toDo) > segmentSize {
|
||||
toDo = toDo[:segmentSize]
|
||||
@@ -312,7 +312,7 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) {
|
||||
segmentSize := (len(src) + 3) / 4
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(4)
|
||||
for i := 0; i < 4; i++ {
|
||||
for i := range 4 {
|
||||
toDo := src
|
||||
if len(toDo) > segmentSize {
|
||||
toDo = toDo[:segmentSize]
|
||||
@@ -326,7 +326,7 @@ func (s *Scratch) compress4Xp(src []byte) ([]byte, error) {
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
for i := 0; i < 4; i++ {
|
||||
for i := range 4 {
|
||||
o := s.tmpOut[i]
|
||||
if len(o) > math.MaxUint16 {
|
||||
// We cannot store the size in the jump table
|
||||
|
||||
14
vendor/github.com/klauspost/compress/huff0/decompress.go
generated
vendored
14
vendor/github.com/klauspost/compress/huff0/decompress.go
generated
vendored
@@ -626,7 +626,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
|
||||
|
||||
var br [4]bitReaderBytes
|
||||
start := 6
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
length := int(src[i*2]) | (int(src[i*2+1]) << 8)
|
||||
if start+length >= len(src) {
|
||||
return nil, errors.New("truncated input (or invalid offset)")
|
||||
@@ -798,10 +798,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
|
||||
remainBytes := dstEvery - (decoded / 4)
|
||||
for i := range br {
|
||||
offset := dstEvery * i
|
||||
endsAt := offset + remainBytes
|
||||
if endsAt > len(out) {
|
||||
endsAt = len(out)
|
||||
}
|
||||
endsAt := min(offset+remainBytes, len(out))
|
||||
br := &br[i]
|
||||
bitsLeft := br.remaining()
|
||||
for bitsLeft > 0 {
|
||||
@@ -864,7 +861,7 @@ func (d *Decoder) decompress4X8bit(dst, src []byte) ([]byte, error) {
|
||||
func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
|
||||
var br [4]bitReaderBytes
|
||||
start := 6
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
length := int(src[i*2]) | (int(src[i*2+1]) << 8)
|
||||
if start+length >= len(src) {
|
||||
return nil, errors.New("truncated input (or invalid offset)")
|
||||
@@ -1035,10 +1032,7 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) {
|
||||
remainBytes := dstEvery - (decoded / 4)
|
||||
for i := range br {
|
||||
offset := dstEvery * i
|
||||
endsAt := offset + remainBytes
|
||||
if endsAt > len(out) {
|
||||
endsAt = len(out)
|
||||
}
|
||||
endsAt := min(offset+remainBytes, len(out))
|
||||
br := &br[i]
|
||||
bitsLeft := br.remaining()
|
||||
for bitsLeft > 0 {
|
||||
|
||||
7
vendor/github.com/klauspost/compress/huff0/decompress_amd64.go
generated
vendored
7
vendor/github.com/klauspost/compress/huff0/decompress_amd64.go
generated
vendored
@@ -58,7 +58,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
|
||||
var br [4]bitReaderShifted
|
||||
// Decode "jump table"
|
||||
start := 6
|
||||
for i := 0; i < 3; i++ {
|
||||
for i := range 3 {
|
||||
length := int(src[i*2]) | (int(src[i*2+1]) << 8)
|
||||
if start+length >= len(src) {
|
||||
return nil, errors.New("truncated input (or invalid offset)")
|
||||
@@ -109,10 +109,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) {
|
||||
remainBytes := dstEvery - (decoded / 4)
|
||||
for i := range br {
|
||||
offset := dstEvery * i
|
||||
endsAt := offset + remainBytes
|
||||
if endsAt > len(out) {
|
||||
endsAt = len(out)
|
||||
}
|
||||
endsAt := min(offset+remainBytes, len(out))
|
||||
br := &br[i]
|
||||
bitsLeft := br.remaining()
|
||||
for bitsLeft > 0 {
|
||||
|
||||
4
vendor/github.com/klauspost/compress/huff0/huff0.go
generated
vendored
4
vendor/github.com/klauspost/compress/huff0/huff0.go
generated
vendored
@@ -201,7 +201,7 @@ func (c cTable) write(s *Scratch) error {
|
||||
for i := range hist[:16] {
|
||||
hist[i] = 0
|
||||
}
|
||||
for n := uint8(0); n < maxSymbolValue; n++ {
|
||||
for n := range maxSymbolValue {
|
||||
v := bitsToWeight[c[n].nBits] & 15
|
||||
huffWeight[n] = v
|
||||
hist[v]++
|
||||
@@ -271,7 +271,7 @@ func (c cTable) estTableSize(s *Scratch) (sz int, err error) {
|
||||
for i := range hist[:16] {
|
||||
hist[i] = 0
|
||||
}
|
||||
for n := uint8(0); n < maxSymbolValue; n++ {
|
||||
for n := range maxSymbolValue {
|
||||
v := bitsToWeight[c[n].nBits] & 15
|
||||
huffWeight[n] = v
|
||||
hist[v]++
|
||||
|
||||
5
vendor/github.com/klauspost/compress/internal/le/le.go
generated
vendored
Normal file
5
vendor/github.com/klauspost/compress/internal/le/le.go
generated
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
package le
|
||||
|
||||
type Indexer interface {
|
||||
int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64
|
||||
}
|
||||
42
vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go
generated
vendored
Normal file
42
vendor/github.com/klauspost/compress/internal/le/unsafe_disabled.go
generated
vendored
Normal file
@@ -0,0 +1,42 @@
|
||||
//go:build !(amd64 || arm64 || ppc64le || riscv64) || nounsafe || purego || appengine
|
||||
|
||||
package le
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
)
|
||||
|
||||
// Load8 will load from b at index i.
|
||||
func Load8[I Indexer](b []byte, i I) byte {
|
||||
return b[i]
|
||||
}
|
||||
|
||||
// Load16 will load from b at index i.
|
||||
func Load16[I Indexer](b []byte, i I) uint16 {
|
||||
return binary.LittleEndian.Uint16(b[i:])
|
||||
}
|
||||
|
||||
// Load32 will load from b at index i.
|
||||
func Load32[I Indexer](b []byte, i I) uint32 {
|
||||
return binary.LittleEndian.Uint32(b[i:])
|
||||
}
|
||||
|
||||
// Load64 will load from b at index i.
|
||||
func Load64[I Indexer](b []byte, i I) uint64 {
|
||||
return binary.LittleEndian.Uint64(b[i:])
|
||||
}
|
||||
|
||||
// Store16 will store v at b.
|
||||
func Store16(b []byte, v uint16) {
|
||||
binary.LittleEndian.PutUint16(b, v)
|
||||
}
|
||||
|
||||
// Store32 will store v at b.
|
||||
func Store32(b []byte, v uint32) {
|
||||
binary.LittleEndian.PutUint32(b, v)
|
||||
}
|
||||
|
||||
// Store64 will store v at b.
|
||||
func Store64[I Indexer](b []byte, i I, v uint64) {
|
||||
binary.LittleEndian.PutUint64(b[i:], v)
|
||||
}
|
||||
52
vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go
generated
vendored
Normal file
52
vendor/github.com/klauspost/compress/internal/le/unsafe_enabled.go
generated
vendored
Normal file
@@ -0,0 +1,52 @@
|
||||
// We enable 64 bit LE platforms:
|
||||
|
||||
//go:build (amd64 || arm64 || ppc64le || riscv64) && !nounsafe && !purego && !appengine
|
||||
|
||||
package le
|
||||
|
||||
import (
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// Load8 will load from b at index i.
|
||||
func Load8[I Indexer](b []byte, i I) byte {
|
||||
//return binary.LittleEndian.Uint16(b[i:])
|
||||
//return *(*uint16)(unsafe.Pointer(&b[i]))
|
||||
return *(*byte)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Load16 will load from b at index i.
|
||||
func Load16[I Indexer](b []byte, i I) uint16 {
|
||||
//return binary.LittleEndian.Uint16(b[i:])
|
||||
//return *(*uint16)(unsafe.Pointer(&b[i]))
|
||||
return *(*uint16)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Load32 will load from b at index i.
|
||||
func Load32[I Indexer](b []byte, i I) uint32 {
|
||||
//return binary.LittleEndian.Uint32(b[i:])
|
||||
//return *(*uint32)(unsafe.Pointer(&b[i]))
|
||||
return *(*uint32)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Load64 will load from b at index i.
|
||||
func Load64[I Indexer](b []byte, i I) uint64 {
|
||||
//return binary.LittleEndian.Uint64(b[i:])
|
||||
//return *(*uint64)(unsafe.Pointer(&b[i]))
|
||||
return *(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i))
|
||||
}
|
||||
|
||||
// Store16 will store v at b.
|
||||
func Store16(b []byte, v uint16) {
|
||||
*(*uint16)(unsafe.Pointer(unsafe.SliceData(b))) = v
|
||||
}
|
||||
|
||||
// Store32 will store v at b.
|
||||
func Store32(b []byte, v uint32) {
|
||||
*(*uint32)(unsafe.Pointer(unsafe.SliceData(b))) = v
|
||||
}
|
||||
|
||||
// Store64 will store v at b[i:].
|
||||
func Store64[I Indexer](b []byte, i I, v uint64) {
|
||||
*(*uint64)(unsafe.Add(unsafe.Pointer(unsafe.SliceData(b)), i)) = v
|
||||
}
|
||||
2
vendor/github.com/klauspost/compress/internal/snapref/decode.go
generated
vendored
2
vendor/github.com/klauspost/compress/internal/snapref/decode.go
generated
vendored
@@ -209,7 +209,7 @@ func (r *Reader) fill() error {
|
||||
if !r.readFull(r.buf[:len(magicBody)], false) {
|
||||
return r.err
|
||||
}
|
||||
for i := 0; i < len(magicBody); i++ {
|
||||
for i := range len(magicBody) {
|
||||
if r.buf[i] != magicBody[i] {
|
||||
r.err = ErrCorrupt
|
||||
return r.err
|
||||
|
||||
4
vendor/github.com/klauspost/compress/internal/snapref/encode.go
generated
vendored
4
vendor/github.com/klauspost/compress/internal/snapref/encode.go
generated
vendored
@@ -20,8 +20,10 @@ import (
|
||||
func Encode(dst, src []byte) []byte {
|
||||
if n := MaxEncodedLen(len(src)); n < 0 {
|
||||
panic(ErrTooLarge)
|
||||
} else if len(dst) < n {
|
||||
} else if cap(dst) < n {
|
||||
dst = make([]byte, n)
|
||||
} else {
|
||||
dst = dst[:n]
|
||||
}
|
||||
|
||||
// The block starts with the varint-encoded length of the decompressed bytes.
|
||||
|
||||
3
vendor/github.com/klauspost/compress/s2sx.mod
generated
vendored
3
vendor/github.com/klauspost/compress/s2sx.mod
generated
vendored
@@ -1,4 +1,3 @@
|
||||
module github.com/klauspost/compress
|
||||
|
||||
go 1.19
|
||||
|
||||
go 1.22
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/README.md
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/README.md
generated
vendored
@@ -6,7 +6,7 @@ A high performance compression algorithm is implemented. For now focused on spee
|
||||
|
||||
This package provides [compression](#Compressor) to and [decompression](#Decompressor) of Zstandard content.
|
||||
|
||||
This package is pure Go and without use of "unsafe".
|
||||
This package is pure Go. Use `noasm` and `nounsafe` to disable relevant features.
|
||||
|
||||
The `zstd` package is provided as open source software using a Go standard license.
|
||||
|
||||
|
||||
37
vendor/github.com/klauspost/compress/zstd/bitreader.go
generated
vendored
37
vendor/github.com/klauspost/compress/zstd/bitreader.go
generated
vendored
@@ -5,11 +5,12 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/bits"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// bitReader reads a bitstream in reverse.
|
||||
@@ -18,6 +19,7 @@ import (
|
||||
type bitReader struct {
|
||||
in []byte
|
||||
value uint64 // Maybe use [16]byte, but shifting is awkward.
|
||||
cursor int // offset where next read should end
|
||||
bitsRead uint8
|
||||
}
|
||||
|
||||
@@ -32,6 +34,7 @@ func (b *bitReader) init(in []byte) error {
|
||||
if v == 0 {
|
||||
return errors.New("corrupt stream, did not find end of stream")
|
||||
}
|
||||
b.cursor = len(in)
|
||||
b.bitsRead = 64
|
||||
b.value = 0
|
||||
if len(in) >= 8 {
|
||||
@@ -67,18 +70,15 @@ func (b *bitReader) fillFast() {
|
||||
if b.bitsRead < 32 {
|
||||
return
|
||||
}
|
||||
v := b.in[len(b.in)-4:]
|
||||
b.in = b.in[:len(b.in)-4]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value = (b.value << 32) | uint64(low)
|
||||
b.cursor -= 4
|
||||
b.value = (b.value << 32) | uint64(le.Load32(b.in, b.cursor))
|
||||
b.bitsRead -= 32
|
||||
}
|
||||
|
||||
// fillFastStart() assumes the bitreader is empty and there is at least 8 bytes to read.
|
||||
func (b *bitReader) fillFastStart() {
|
||||
v := b.in[len(b.in)-8:]
|
||||
b.in = b.in[:len(b.in)-8]
|
||||
b.value = binary.LittleEndian.Uint64(v)
|
||||
b.cursor -= 8
|
||||
b.value = le.Load64(b.in, b.cursor)
|
||||
b.bitsRead = 0
|
||||
}
|
||||
|
||||
@@ -87,25 +87,23 @@ func (b *bitReader) fill() {
|
||||
if b.bitsRead < 32 {
|
||||
return
|
||||
}
|
||||
if len(b.in) >= 4 {
|
||||
v := b.in[len(b.in)-4:]
|
||||
b.in = b.in[:len(b.in)-4]
|
||||
low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24)
|
||||
b.value = (b.value << 32) | uint64(low)
|
||||
if b.cursor >= 4 {
|
||||
b.cursor -= 4
|
||||
b.value = (b.value << 32) | uint64(le.Load32(b.in, b.cursor))
|
||||
b.bitsRead -= 32
|
||||
return
|
||||
}
|
||||
|
||||
b.bitsRead -= uint8(8 * len(b.in))
|
||||
for len(b.in) > 0 {
|
||||
b.value = (b.value << 8) | uint64(b.in[len(b.in)-1])
|
||||
b.in = b.in[:len(b.in)-1]
|
||||
b.bitsRead -= uint8(8 * b.cursor)
|
||||
for b.cursor > 0 {
|
||||
b.cursor -= 1
|
||||
b.value = (b.value << 8) | uint64(b.in[b.cursor])
|
||||
}
|
||||
}
|
||||
|
||||
// finished returns true if all bits have been read from the bit stream.
|
||||
func (b *bitReader) finished() bool {
|
||||
return len(b.in) == 0 && b.bitsRead >= 64
|
||||
return b.cursor == 0 && b.bitsRead >= 64
|
||||
}
|
||||
|
||||
// overread returns true if more bits have been requested than is on the stream.
|
||||
@@ -115,13 +113,14 @@ func (b *bitReader) overread() bool {
|
||||
|
||||
// remain returns the number of bits remaining.
|
||||
func (b *bitReader) remain() uint {
|
||||
return 8*uint(len(b.in)) + 64 - uint(b.bitsRead)
|
||||
return 8*uint(b.cursor) + 64 - uint(b.bitsRead)
|
||||
}
|
||||
|
||||
// close the bitstream and returns an error if out-of-buffer reads occurred.
|
||||
func (b *bitReader) close() error {
|
||||
// Release reference.
|
||||
b.in = nil
|
||||
b.cursor = 0
|
||||
if !b.finished() {
|
||||
return fmt.Errorf("%d extra bits on block, should be 0", b.remain())
|
||||
}
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/bitwriter.go
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/bitwriter.go
generated
vendored
@@ -88,7 +88,7 @@ func (b *bitWriter) flush32() {
|
||||
// flushAlign will flush remaining full bytes and align to next byte boundary.
|
||||
func (b *bitWriter) flushAlign() {
|
||||
nbBytes := (b.nBits + 7) >> 3
|
||||
for i := uint8(0); i < nbBytes; i++ {
|
||||
for i := range nbBytes {
|
||||
b.out = append(b.out, byte(b.bitContainer>>(i*8)))
|
||||
}
|
||||
b.nBits = 0
|
||||
|
||||
25
vendor/github.com/klauspost/compress/zstd/blockdec.go
generated
vendored
25
vendor/github.com/klauspost/compress/zstd/blockdec.go
generated
vendored
@@ -5,14 +5,10 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/klauspost/compress/huff0"
|
||||
@@ -58,11 +54,11 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
huffDecoderPool = sync.Pool{New: func() interface{} {
|
||||
huffDecoderPool = sync.Pool{New: func() any {
|
||||
return &huff0.Scratch{}
|
||||
}}
|
||||
|
||||
fseDecoderPool = sync.Pool{New: func() interface{} {
|
||||
fseDecoderPool = sync.Pool{New: func() any {
|
||||
return &fseDecoder{}
|
||||
}}
|
||||
)
|
||||
@@ -557,7 +553,7 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
|
||||
if compMode&3 != 0 {
|
||||
return errors.New("corrupt block: reserved bits not zero")
|
||||
}
|
||||
for i := uint(0); i < 3; i++ {
|
||||
for i := range uint(3) {
|
||||
mode := seqCompMode((compMode >> (6 - i*2)) & 3)
|
||||
if debugDecoder {
|
||||
println("Table", tableIndex(i), "is", mode)
|
||||
@@ -648,21 +644,6 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) {
|
||||
println("initializing sequences:", err)
|
||||
return err
|
||||
}
|
||||
// Extract blocks...
|
||||
if false && hist.dict == nil {
|
||||
fatalErr := func(err error) {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
fn := fmt.Sprintf("n-%d-lits-%d-prev-%d-%d-%d-win-%d.blk", hist.decoders.nSeqs, len(hist.decoders.literals), hist.recentOffsets[0], hist.recentOffsets[1], hist.recentOffsets[2], hist.windowSize)
|
||||
var buf bytes.Buffer
|
||||
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.litLengths.fse))
|
||||
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.matchLengths.fse))
|
||||
fatalErr(binary.Write(&buf, binary.LittleEndian, hist.decoders.offsets.fse))
|
||||
buf.Write(in)
|
||||
os.WriteFile(filepath.Join("testdata", "seqs", fn), buf.Bytes(), os.ModePerm)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
27
vendor/github.com/klauspost/compress/zstd/blockenc.go
generated
vendored
27
vendor/github.com/klauspost/compress/zstd/blockenc.go
generated
vendored
@@ -9,6 +9,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/bits"
|
||||
"slices"
|
||||
|
||||
"github.com/klauspost/compress/huff0"
|
||||
)
|
||||
@@ -457,16 +458,7 @@ func fuzzFseEncoder(data []byte) int {
|
||||
// All 0
|
||||
return 0
|
||||
}
|
||||
maxCount := func(a []uint32) int {
|
||||
var max uint32
|
||||
for _, v := range a {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return int(max)
|
||||
}
|
||||
cnt := maxCount(hist[:maxSym])
|
||||
cnt := int(slices.Max(hist[:maxSym]))
|
||||
if cnt == len(data) {
|
||||
// RLE
|
||||
return 0
|
||||
@@ -884,15 +876,6 @@ func (b *blockEnc) genCodes() {
|
||||
}
|
||||
}
|
||||
}
|
||||
maxCount := func(a []uint32) int {
|
||||
var max uint32
|
||||
for _, v := range a {
|
||||
if v > max {
|
||||
max = v
|
||||
}
|
||||
}
|
||||
return int(max)
|
||||
}
|
||||
if debugAsserts && mlMax > maxMatchLengthSymbol {
|
||||
panic(fmt.Errorf("mlMax > maxMatchLengthSymbol (%d)", mlMax))
|
||||
}
|
||||
@@ -903,7 +886,7 @@ func (b *blockEnc) genCodes() {
|
||||
panic(fmt.Errorf("llMax > maxLiteralLengthSymbol (%d)", llMax))
|
||||
}
|
||||
|
||||
b.coders.mlEnc.HistogramFinished(mlMax, maxCount(mlH[:mlMax+1]))
|
||||
b.coders.ofEnc.HistogramFinished(ofMax, maxCount(ofH[:ofMax+1]))
|
||||
b.coders.llEnc.HistogramFinished(llMax, maxCount(llH[:llMax+1]))
|
||||
b.coders.mlEnc.HistogramFinished(mlMax, int(slices.Max(mlH[:mlMax+1])))
|
||||
b.coders.ofEnc.HistogramFinished(ofMax, int(slices.Max(ofH[:ofMax+1])))
|
||||
b.coders.llEnc.HistogramFinished(llMax, int(slices.Max(llH[:llMax+1])))
|
||||
}
|
||||
|
||||
11
vendor/github.com/klauspost/compress/zstd/decoder.go
generated
vendored
11
vendor/github.com/klauspost/compress/zstd/decoder.go
generated
vendored
@@ -123,7 +123,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
|
||||
}
|
||||
|
||||
// Read bytes from the decompressed stream into p.
|
||||
// Returns the number of bytes written and any error that occurred.
|
||||
// Returns the number of bytes read and any error that occurred.
|
||||
// When the stream is done, io.EOF will be returned.
|
||||
func (d *Decoder) Read(p []byte) (int, error) {
|
||||
var n int
|
||||
@@ -323,6 +323,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
|
||||
frame.bBuf = nil
|
||||
if frame.history.decoders.br != nil {
|
||||
frame.history.decoders.br.in = nil
|
||||
frame.history.decoders.br.cursor = 0
|
||||
}
|
||||
d.decoders <- block
|
||||
}()
|
||||
@@ -372,11 +373,9 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) {
|
||||
if cap(dst) == 0 && !d.o.limitToCap {
|
||||
// Allocate len(input) * 2 by default if nothing is provided
|
||||
// and we didn't get frame content size.
|
||||
size := len(input) * 2
|
||||
// Cap to 1 MB.
|
||||
if size > 1<<20 {
|
||||
size = 1 << 20
|
||||
}
|
||||
size := min(
|
||||
// Cap to 1 MB.
|
||||
len(input)*2, 1<<20)
|
||||
if uint64(size) > d.o.maxDecodedSize {
|
||||
size = int(d.o.maxDecodedSize)
|
||||
}
|
||||
|
||||
20
vendor/github.com/klauspost/compress/zstd/dict.go
generated
vendored
20
vendor/github.com/klauspost/compress/zstd/dict.go
generated
vendored
@@ -194,17 +194,17 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
|
||||
hist := o.History
|
||||
contents := o.Contents
|
||||
debug := o.DebugOut != nil
|
||||
println := func(args ...interface{}) {
|
||||
println := func(args ...any) {
|
||||
if o.DebugOut != nil {
|
||||
fmt.Fprintln(o.DebugOut, args...)
|
||||
}
|
||||
}
|
||||
printf := func(s string, args ...interface{}) {
|
||||
printf := func(s string, args ...any) {
|
||||
if o.DebugOut != nil {
|
||||
fmt.Fprintf(o.DebugOut, s, args...)
|
||||
}
|
||||
}
|
||||
print := func(args ...interface{}) {
|
||||
print := func(args ...any) {
|
||||
if o.DebugOut != nil {
|
||||
fmt.Fprint(o.DebugOut, args...)
|
||||
}
|
||||
@@ -424,16 +424,10 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
|
||||
}
|
||||
|
||||
// Literal table
|
||||
avgSize := litTotal
|
||||
if avgSize > huff0.BlockSizeMax/2 {
|
||||
avgSize = huff0.BlockSizeMax / 2
|
||||
}
|
||||
avgSize := min(litTotal, huff0.BlockSizeMax/2)
|
||||
huffBuff := make([]byte, 0, avgSize)
|
||||
// Target size
|
||||
div := litTotal / avgSize
|
||||
if div < 1 {
|
||||
div = 1
|
||||
}
|
||||
div := max(litTotal/avgSize, 1)
|
||||
if debug {
|
||||
println("Huffman weights:")
|
||||
}
|
||||
@@ -454,7 +448,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
|
||||
huffBuff = append(huffBuff, 255)
|
||||
}
|
||||
scratch := &huff0.Scratch{TableLog: 11}
|
||||
for tries := 0; tries < 255; tries++ {
|
||||
for tries := range 255 {
|
||||
scratch = &huff0.Scratch{TableLog: 11}
|
||||
_, _, err = huff0.Compress1X(huffBuff, scratch)
|
||||
if err == nil {
|
||||
@@ -471,7 +465,7 @@ func BuildDict(o BuildDictOptions) ([]byte, error) {
|
||||
|
||||
// Bail out.... Just generate something
|
||||
huffBuff = append(huffBuff, bytes.Repeat([]byte{255}, 10000)...)
|
||||
for i := 0; i < 128; i++ {
|
||||
for i := range 128 {
|
||||
huffBuff = append(huffBuff, byte(i))
|
||||
}
|
||||
continue
|
||||
|
||||
12
vendor/github.com/klauspost/compress/zstd/enc_base.go
generated
vendored
12
vendor/github.com/klauspost/compress/zstd/enc_base.go
generated
vendored
@@ -8,7 +8,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
dictShardBits = 6
|
||||
dictShardBits = 7
|
||||
)
|
||||
|
||||
type fastBase struct {
|
||||
@@ -41,11 +41,9 @@ func (e *fastBase) AppendCRC(dst []byte) []byte {
|
||||
// or a window size small enough to contain the input size, if > 0.
|
||||
func (e *fastBase) WindowSize(size int64) int32 {
|
||||
if size > 0 && size < int64(e.maxMatchOff) {
|
||||
b := int32(1) << uint(bits.Len(uint(size)))
|
||||
// Keep minimum window.
|
||||
if b < 1024 {
|
||||
b = 1024
|
||||
}
|
||||
b := max(
|
||||
// Keep minimum window.
|
||||
int32(1)<<uint(bits.Len(uint(size))), 1024)
|
||||
return b
|
||||
}
|
||||
return e.maxMatchOff
|
||||
@@ -116,7 +114,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
|
||||
panic(err)
|
||||
}
|
||||
if t < 0 {
|
||||
err := fmt.Sprintf("s (%d) < 0", s)
|
||||
err := fmt.Sprintf("t (%d) < 0", t)
|
||||
panic(err)
|
||||
}
|
||||
if s-t > e.maxMatchOff {
|
||||
|
||||
23
vendor/github.com/klauspost/compress/zstd/enc_best.go
generated
vendored
23
vendor/github.com/klauspost/compress/zstd/enc_best.go
generated
vendored
@@ -158,11 +158,9 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) {
|
||||
|
||||
// Use this to estimate literal cost.
|
||||
// Scaled by 10 bits.
|
||||
bitsPerByte := int32((compress.ShannonEntropyBits(src) * 1024) / len(src))
|
||||
// Huffman can never go < 1 bit/byte
|
||||
if bitsPerByte < 1024 {
|
||||
bitsPerByte = 1024
|
||||
}
|
||||
bitsPerByte := max(
|
||||
// Huffman can never go < 1 bit/byte
|
||||
int32((compress.ShannonEntropyBits(src)*1024)/len(src)), 1024)
|
||||
|
||||
// Override src
|
||||
src = e.hist
|
||||
@@ -235,10 +233,7 @@ encodeLoop:
|
||||
// Extend candidate match backwards as far as possible.
|
||||
// Do not extend repeats as we can assume they are optimal
|
||||
// and offsets change if s == nextEmit.
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for offset > tMin && s > nextEmit && src[offset-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
offset--
|
||||
@@ -382,10 +377,7 @@ encodeLoop:
|
||||
nextEmit = s
|
||||
|
||||
// Index skipped...
|
||||
end := s
|
||||
if s > sLimit+4 {
|
||||
end = sLimit + 4
|
||||
}
|
||||
end := min(s, sLimit+4)
|
||||
off := index0 + e.cur
|
||||
for index0 < end {
|
||||
cv0 := load6432(src, index0)
|
||||
@@ -444,10 +436,7 @@ encodeLoop:
|
||||
nextEmit = s
|
||||
|
||||
// Index old s + 1 -> s - 1 or sLimit
|
||||
end := s
|
||||
if s > sLimit-4 {
|
||||
end = sLimit - 4
|
||||
}
|
||||
end := min(s, sLimit-4)
|
||||
|
||||
off := index0 + e.cur
|
||||
for index0 < end {
|
||||
|
||||
30
vendor/github.com/klauspost/compress/zstd/enc_better.go
generated
vendored
30
vendor/github.com/klauspost/compress/zstd/enc_better.go
generated
vendored
@@ -190,10 +190,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -252,10 +249,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -480,10 +474,7 @@ encodeLoop:
|
||||
l := matched
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
t--
|
||||
@@ -719,10 +710,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -783,10 +771,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -1005,10 +990,7 @@ encodeLoop:
|
||||
l := matched
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
t--
|
||||
|
||||
32
vendor/github.com/klauspost/compress/zstd/enc_dfast.go
generated
vendored
32
vendor/github.com/klauspost/compress/zstd/enc_dfast.go
generated
vendored
@@ -13,7 +13,7 @@ const (
|
||||
dFastLongLen = 8 // Bytes used for table hash
|
||||
|
||||
dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table
|
||||
dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard
|
||||
dLongTableShardSize = dFastLongTableSize / dLongTableShardCnt // Size of an individual shard
|
||||
|
||||
dFastShortTableBits = tableBits // Bits used in the short match table
|
||||
dFastShortTableSize = 1 << dFastShortTableBits // Size of the table
|
||||
@@ -149,10 +149,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -266,10 +263,7 @@ encodeLoop:
|
||||
l := e.matchlen(s+4, t+4, src) + 4
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
t--
|
||||
@@ -462,10 +456,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -576,10 +567,7 @@ encodeLoop:
|
||||
l := int32(matchLen(src[s+4:], src[t+4:])) + 4
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
@@ -809,10 +797,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -927,10 +912,7 @@ encodeLoop:
|
||||
l := e.matchlen(s+4, t+4, src) + 4
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
t--
|
||||
|
||||
30
vendor/github.com/klauspost/compress/zstd/enc_fast.go
generated
vendored
30
vendor/github.com/klauspost/compress/zstd/enc_fast.go
generated
vendored
@@ -143,10 +143,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
sMin := s - e.maxMatchOff
|
||||
if sMin < 0 {
|
||||
sMin = 0
|
||||
}
|
||||
sMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -223,10 +220,7 @@ encodeLoop:
|
||||
l := e.matchlen(s+4, t+4, src) + 4
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
t--
|
||||
@@ -387,10 +381,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
sMin := s - e.maxMatchOff
|
||||
if sMin < 0 {
|
||||
sMin = 0
|
||||
}
|
||||
sMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -469,10 +460,7 @@ encodeLoop:
|
||||
l := e.matchlen(s+4, t+4, src) + 4
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] {
|
||||
s--
|
||||
t--
|
||||
@@ -655,10 +643,7 @@ encodeLoop:
|
||||
// and have to do special offset treatment.
|
||||
startLimit := nextEmit + 1
|
||||
|
||||
sMin := s - e.maxMatchOff
|
||||
if sMin < 0 {
|
||||
sMin = 0
|
||||
}
|
||||
sMin := max(s-e.maxMatchOff, 0)
|
||||
for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch {
|
||||
repIndex--
|
||||
start--
|
||||
@@ -735,10 +720,7 @@ encodeLoop:
|
||||
l := e.matchlen(s+4, t+4, src) + 4
|
||||
|
||||
// Extend backwards
|
||||
tMin := s - e.maxMatchOff
|
||||
if tMin < 0 {
|
||||
tMin = 0
|
||||
}
|
||||
tMin := max(s-e.maxMatchOff, 0)
|
||||
for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
|
||||
s--
|
||||
t--
|
||||
|
||||
5
vendor/github.com/klauspost/compress/zstd/framedec.go
generated
vendored
5
vendor/github.com/klauspost/compress/zstd/framedec.go
generated
vendored
@@ -238,10 +238,7 @@ func (d *frameDec) reset(br byteBuffer) error {
|
||||
|
||||
if d.WindowSize == 0 && d.SingleSegment {
|
||||
// We may not need window in this case.
|
||||
d.WindowSize = d.FrameContentSize
|
||||
if d.WindowSize < MinWindowSize {
|
||||
d.WindowSize = MinWindowSize
|
||||
}
|
||||
d.WindowSize = max(d.FrameContentSize, MinWindowSize)
|
||||
if d.WindowSize > d.o.maxDecodedSize {
|
||||
if debugDecoder {
|
||||
printf("window size %d > max %d\n", d.WindowSize, d.o.maxWindowSize)
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/fse_encoder.go
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/fse_encoder.go
generated
vendored
@@ -149,7 +149,7 @@ func (s *fseEncoder) buildCTable() error {
|
||||
if v > largeLimit {
|
||||
s.zeroBits = true
|
||||
}
|
||||
for nbOccurrences := int16(0); nbOccurrences < v; nbOccurrences++ {
|
||||
for range v {
|
||||
tableSymbol[position] = symbol
|
||||
position = (position + step) & tableMask
|
||||
for position > highThreshold {
|
||||
|
||||
11
vendor/github.com/klauspost/compress/zstd/matchlen_generic.go
generated
vendored
11
vendor/github.com/klauspost/compress/zstd/matchlen_generic.go
generated
vendored
@@ -7,20 +7,25 @@
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math/bits"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// matchLen returns the maximum common prefix length of a and b.
|
||||
// a must be the shortest of the two.
|
||||
func matchLen(a, b []byte) (n int) {
|
||||
for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] {
|
||||
diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b)
|
||||
left := len(a)
|
||||
for left >= 8 {
|
||||
diff := le.Load64(a, n) ^ le.Load64(b, n)
|
||||
if diff != 0 {
|
||||
return n + bits.TrailingZeros64(diff)>>3
|
||||
}
|
||||
n += 8
|
||||
left -= 8
|
||||
}
|
||||
a = a[n:]
|
||||
b = b[n:]
|
||||
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
|
||||
7
vendor/github.com/klauspost/compress/zstd/seqdec.go
generated
vendored
7
vendor/github.com/klauspost/compress/zstd/seqdec.go
generated
vendored
@@ -231,10 +231,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error {
|
||||
llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize]
|
||||
llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state
|
||||
out := s.out
|
||||
maxBlockSize := maxCompressedBlockSize
|
||||
if s.windowSize < maxBlockSize {
|
||||
maxBlockSize = s.windowSize
|
||||
}
|
||||
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
|
||||
|
||||
if debugDecoder {
|
||||
println("decodeSync: decoding", seqs, "sequences", br.remain(), "bits remain on stream")
|
||||
@@ -245,7 +242,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
var ll, mo, ml int
|
||||
if len(br.in) > 4+((maxOffsetBits+16+16)>>3) {
|
||||
if br.cursor > 4+((maxOffsetBits+16+16)>>3) {
|
||||
// inlined function:
|
||||
// ll, mo, ml = s.nextFast(br, llState, mlState, ofState)
|
||||
|
||||
|
||||
10
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go
generated
vendored
10
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go
generated
vendored
@@ -79,10 +79,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) {
|
||||
|
||||
br := s.br
|
||||
|
||||
maxBlockSize := maxCompressedBlockSize
|
||||
if s.windowSize < maxBlockSize {
|
||||
maxBlockSize = s.windowSize
|
||||
}
|
||||
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
|
||||
|
||||
ctx := decodeSyncAsmContext{
|
||||
llTable: s.litLengths.fse.dt[:maxTablesize],
|
||||
@@ -237,10 +234,7 @@ func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmC
|
||||
func (s *sequenceDecs) decode(seqs []seqVals) error {
|
||||
br := s.br
|
||||
|
||||
maxBlockSize := maxCompressedBlockSize
|
||||
if s.windowSize < maxBlockSize {
|
||||
maxBlockSize = s.windowSize
|
||||
}
|
||||
maxBlockSize := min(s.windowSize, maxCompressedBlockSize)
|
||||
|
||||
ctx := decodeAsmContext{
|
||||
llTable: s.litLengths.fse.dt[:maxTablesize],
|
||||
|
||||
64
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s
generated
vendored
64
vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s
generated
vendored
@@ -7,9 +7,9 @@
|
||||
TEXT ·sequenceDecs_decode_amd64(SB), $8-32
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ 24(CX), DX
|
||||
MOVBQZX 32(CX), BX
|
||||
MOVBQZX 40(CX), BX
|
||||
MOVQ (CX), AX
|
||||
MOVQ 8(CX), SI
|
||||
MOVQ 32(CX), SI
|
||||
ADDQ SI, AX
|
||||
MOVQ AX, (SP)
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -299,8 +299,8 @@ sequenceDecs_decode_amd64_match_len_ofs_ok:
|
||||
MOVQ R13, 160(AX)
|
||||
MOVQ br+8(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVB BL, 32(AX)
|
||||
MOVQ SI, 8(AX)
|
||||
MOVB BL, 40(AX)
|
||||
MOVQ SI, 32(AX)
|
||||
|
||||
// Return success
|
||||
MOVQ $0x00000000, ret+24(FP)
|
||||
@@ -335,9 +335,9 @@ error_overread:
|
||||
TEXT ·sequenceDecs_decode_56_amd64(SB), $8-32
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ 24(CX), DX
|
||||
MOVBQZX 32(CX), BX
|
||||
MOVBQZX 40(CX), BX
|
||||
MOVQ (CX), AX
|
||||
MOVQ 8(CX), SI
|
||||
MOVQ 32(CX), SI
|
||||
ADDQ SI, AX
|
||||
MOVQ AX, (SP)
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -598,8 +598,8 @@ sequenceDecs_decode_56_amd64_match_len_ofs_ok:
|
||||
MOVQ R13, 160(AX)
|
||||
MOVQ br+8(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVB BL, 32(AX)
|
||||
MOVQ SI, 8(AX)
|
||||
MOVB BL, 40(AX)
|
||||
MOVQ SI, 32(AX)
|
||||
|
||||
// Return success
|
||||
MOVQ $0x00000000, ret+24(FP)
|
||||
@@ -634,9 +634,9 @@ error_overread:
|
||||
TEXT ·sequenceDecs_decode_bmi2(SB), $8-32
|
||||
MOVQ br+8(FP), BX
|
||||
MOVQ 24(BX), AX
|
||||
MOVBQZX 32(BX), DX
|
||||
MOVBQZX 40(BX), DX
|
||||
MOVQ (BX), CX
|
||||
MOVQ 8(BX), BX
|
||||
MOVQ 32(BX), BX
|
||||
ADDQ BX, CX
|
||||
MOVQ CX, (SP)
|
||||
MOVQ ctx+16(FP), CX
|
||||
@@ -884,8 +884,8 @@ sequenceDecs_decode_bmi2_match_len_ofs_ok:
|
||||
MOVQ R12, 160(CX)
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ AX, 24(CX)
|
||||
MOVB DL, 32(CX)
|
||||
MOVQ BX, 8(CX)
|
||||
MOVB DL, 40(CX)
|
||||
MOVQ BX, 32(CX)
|
||||
|
||||
// Return success
|
||||
MOVQ $0x00000000, ret+24(FP)
|
||||
@@ -920,9 +920,9 @@ error_overread:
|
||||
TEXT ·sequenceDecs_decode_56_bmi2(SB), $8-32
|
||||
MOVQ br+8(FP), BX
|
||||
MOVQ 24(BX), AX
|
||||
MOVBQZX 32(BX), DX
|
||||
MOVBQZX 40(BX), DX
|
||||
MOVQ (BX), CX
|
||||
MOVQ 8(BX), BX
|
||||
MOVQ 32(BX), BX
|
||||
ADDQ BX, CX
|
||||
MOVQ CX, (SP)
|
||||
MOVQ ctx+16(FP), CX
|
||||
@@ -1141,8 +1141,8 @@ sequenceDecs_decode_56_bmi2_match_len_ofs_ok:
|
||||
MOVQ R12, 160(CX)
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ AX, 24(CX)
|
||||
MOVB DL, 32(CX)
|
||||
MOVQ BX, 8(CX)
|
||||
MOVB DL, 40(CX)
|
||||
MOVQ BX, 32(CX)
|
||||
|
||||
// Return success
|
||||
MOVQ $0x00000000, ret+24(FP)
|
||||
@@ -1787,9 +1787,9 @@ empty_seqs:
|
||||
TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ 24(CX), DX
|
||||
MOVBQZX 32(CX), BX
|
||||
MOVBQZX 40(CX), BX
|
||||
MOVQ (CX), AX
|
||||
MOVQ 8(CX), SI
|
||||
MOVQ 32(CX), SI
|
||||
ADDQ SI, AX
|
||||
MOVQ AX, (SP)
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -2281,8 +2281,8 @@ handle_loop:
|
||||
loop_finished:
|
||||
MOVQ br+8(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVB BL, 32(AX)
|
||||
MOVQ SI, 8(AX)
|
||||
MOVB BL, 40(AX)
|
||||
MOVQ SI, 32(AX)
|
||||
|
||||
// Update the context
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -2349,9 +2349,9 @@ error_not_enough_space:
|
||||
TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32
|
||||
MOVQ br+8(FP), BX
|
||||
MOVQ 24(BX), AX
|
||||
MOVBQZX 32(BX), DX
|
||||
MOVBQZX 40(BX), DX
|
||||
MOVQ (BX), CX
|
||||
MOVQ 8(BX), BX
|
||||
MOVQ 32(BX), BX
|
||||
ADDQ BX, CX
|
||||
MOVQ CX, (SP)
|
||||
MOVQ ctx+16(FP), CX
|
||||
@@ -2801,8 +2801,8 @@ handle_loop:
|
||||
loop_finished:
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ AX, 24(CX)
|
||||
MOVB DL, 32(CX)
|
||||
MOVQ BX, 8(CX)
|
||||
MOVB DL, 40(CX)
|
||||
MOVQ BX, 32(CX)
|
||||
|
||||
// Update the context
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -2869,9 +2869,9 @@ error_not_enough_space:
|
||||
TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ 24(CX), DX
|
||||
MOVBQZX 32(CX), BX
|
||||
MOVBQZX 40(CX), BX
|
||||
MOVQ (CX), AX
|
||||
MOVQ 8(CX), SI
|
||||
MOVQ 32(CX), SI
|
||||
ADDQ SI, AX
|
||||
MOVQ AX, (SP)
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -3465,8 +3465,8 @@ handle_loop:
|
||||
loop_finished:
|
||||
MOVQ br+8(FP), AX
|
||||
MOVQ DX, 24(AX)
|
||||
MOVB BL, 32(AX)
|
||||
MOVQ SI, 8(AX)
|
||||
MOVB BL, 40(AX)
|
||||
MOVQ SI, 32(AX)
|
||||
|
||||
// Update the context
|
||||
MOVQ ctx+16(FP), AX
|
||||
@@ -3533,9 +3533,9 @@ error_not_enough_space:
|
||||
TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32
|
||||
MOVQ br+8(FP), BX
|
||||
MOVQ 24(BX), AX
|
||||
MOVBQZX 32(BX), DX
|
||||
MOVBQZX 40(BX), DX
|
||||
MOVQ (BX), CX
|
||||
MOVQ 8(BX), BX
|
||||
MOVQ 32(BX), BX
|
||||
ADDQ BX, CX
|
||||
MOVQ CX, (SP)
|
||||
MOVQ ctx+16(FP), CX
|
||||
@@ -4087,8 +4087,8 @@ handle_loop:
|
||||
loop_finished:
|
||||
MOVQ br+8(FP), CX
|
||||
MOVQ AX, 24(CX)
|
||||
MOVB DL, 32(CX)
|
||||
MOVQ BX, 8(CX)
|
||||
MOVB DL, 40(CX)
|
||||
MOVQ BX, 32(CX)
|
||||
|
||||
// Update the context
|
||||
MOVQ ctx+16(FP), AX
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/seqdec_generic.go
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/seqdec_generic.go
generated
vendored
@@ -29,7 +29,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error {
|
||||
}
|
||||
for i := range seqs {
|
||||
var ll, mo, ml int
|
||||
if len(br.in) > 4+((maxOffsetBits+16+16)>>3) {
|
||||
if br.cursor > 4+((maxOffsetBits+16+16)>>3) {
|
||||
// inlined function:
|
||||
// ll, mo, ml = s.nextFast(br, llState, mlState, ofState)
|
||||
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/seqenc.go
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/seqenc.go
generated
vendored
@@ -69,7 +69,6 @@ var llBitsTable = [maxLLCode + 1]byte{
|
||||
func llCode(litLength uint32) uint8 {
|
||||
const llDeltaCode = 19
|
||||
if litLength <= 63 {
|
||||
// Compiler insists on bounds check (Go 1.12)
|
||||
return llCodeTable[litLength&63]
|
||||
}
|
||||
return uint8(highBit(litLength)) + llDeltaCode
|
||||
@@ -102,7 +101,6 @@ var mlBitsTable = [maxMLCode + 1]byte{
|
||||
func mlCode(mlBase uint32) uint8 {
|
||||
const mlDeltaCode = 36
|
||||
if mlBase <= 127 {
|
||||
// Compiler insists on bounds check (Go 1.12)
|
||||
return mlCodeTable[mlBase&127]
|
||||
}
|
||||
return uint8(highBit(mlBase)) + mlDeltaCode
|
||||
|
||||
56
vendor/github.com/klauspost/compress/zstd/simple_go124.go
generated
vendored
Normal file
56
vendor/github.com/klauspost/compress/zstd/simple_go124.go
generated
vendored
Normal file
@@ -0,0 +1,56 @@
|
||||
// Copyright 2025+ Klaus Post. All rights reserved.
|
||||
// License information can be found in the LICENSE file.
|
||||
|
||||
//go:build go1.24
|
||||
|
||||
package zstd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"runtime"
|
||||
"sync"
|
||||
"weak"
|
||||
)
|
||||
|
||||
var weakMu sync.Mutex
|
||||
var simpleEnc weak.Pointer[Encoder]
|
||||
var simpleDec weak.Pointer[Decoder]
|
||||
|
||||
// EncodeTo appends the encoded data from src to dst.
|
||||
func EncodeTo(dst []byte, src []byte) []byte {
|
||||
weakMu.Lock()
|
||||
enc := simpleEnc.Value()
|
||||
if enc == nil {
|
||||
var err error
|
||||
enc, err = NewWriter(nil, WithEncoderConcurrency(runtime.NumCPU()), WithWindowSize(1<<20), WithLowerEncoderMem(true), WithZeroFrames(true))
|
||||
if err != nil {
|
||||
panic("failed to create simple encoder: " + err.Error())
|
||||
}
|
||||
simpleEnc = weak.Make(enc)
|
||||
}
|
||||
weakMu.Unlock()
|
||||
|
||||
return enc.EncodeAll(src, dst)
|
||||
}
|
||||
|
||||
// DecodeTo appends the decoded data from src to dst.
|
||||
// The maximum decoded size is 1GiB,
|
||||
// not including what may already be in dst.
|
||||
func DecodeTo(dst []byte, src []byte) ([]byte, error) {
|
||||
weakMu.Lock()
|
||||
dec := simpleDec.Value()
|
||||
if dec == nil {
|
||||
var err error
|
||||
dec, err = NewReader(nil, WithDecoderConcurrency(runtime.NumCPU()), WithDecoderLowmem(true), WithDecoderMaxMemory(1<<30))
|
||||
if err != nil {
|
||||
weakMu.Unlock()
|
||||
return nil, errors.New("failed to create simple decoder: " + err.Error())
|
||||
}
|
||||
runtime.SetFinalizer(dec, func(d *Decoder) {
|
||||
d.Close()
|
||||
})
|
||||
simpleDec = weak.Make(dec)
|
||||
}
|
||||
weakMu.Unlock()
|
||||
return dec.DecodeAll(src, dst)
|
||||
}
|
||||
6
vendor/github.com/klauspost/compress/zstd/snappy.go
generated
vendored
6
vendor/github.com/klauspost/compress/zstd/snappy.go
generated
vendored
@@ -197,7 +197,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) {
|
||||
|
||||
n, r.err = w.Write(r.block.output)
|
||||
if r.err != nil {
|
||||
return written, err
|
||||
return written, r.err
|
||||
}
|
||||
written += int64(n)
|
||||
continue
|
||||
@@ -239,7 +239,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) {
|
||||
}
|
||||
n, r.err = w.Write(r.block.output)
|
||||
if r.err != nil {
|
||||
return written, err
|
||||
return written, r.err
|
||||
}
|
||||
written += int64(n)
|
||||
continue
|
||||
@@ -257,7 +257,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) {
|
||||
if !r.readFull(r.buf[:len(snappyMagicBody)], false) {
|
||||
return written, r.err
|
||||
}
|
||||
for i := 0; i < len(snappyMagicBody); i++ {
|
||||
for i := range len(snappyMagicBody) {
|
||||
if r.buf[i] != snappyMagicBody[i] {
|
||||
println("r.buf[i] != snappyMagicBody[i]", r.buf[i], snappyMagicBody[i], i)
|
||||
r.err = ErrSnappyCorrupt
|
||||
|
||||
2
vendor/github.com/klauspost/compress/zstd/zip.go
generated
vendored
2
vendor/github.com/klauspost/compress/zstd/zip.go
generated
vendored
@@ -19,7 +19,7 @@ const ZipMethodWinZip = 93
|
||||
const ZipMethodPKWare = 20
|
||||
|
||||
// zipReaderPool is the default reader pool.
|
||||
var zipReaderPool = sync.Pool{New: func() interface{} {
|
||||
var zipReaderPool = sync.Pool{New: func() any {
|
||||
z, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderMaxWindow(128<<20), WithDecoderConcurrency(1))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
||||
11
vendor/github.com/klauspost/compress/zstd/zstd.go
generated
vendored
11
vendor/github.com/klauspost/compress/zstd/zstd.go
generated
vendored
@@ -5,10 +5,11 @@ package zstd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"log"
|
||||
"math"
|
||||
|
||||
"github.com/klauspost/compress/internal/le"
|
||||
)
|
||||
|
||||
// enable debug printing
|
||||
@@ -97,24 +98,24 @@ var (
|
||||
ErrDecoderNilInput = errors.New("nil input provided as reader")
|
||||
)
|
||||
|
||||
func println(a ...interface{}) {
|
||||
func println(a ...any) {
|
||||
if debug || debugDecoder || debugEncoder {
|
||||
log.Println(a...)
|
||||
}
|
||||
}
|
||||
|
||||
func printf(format string, a ...interface{}) {
|
||||
func printf(format string, a ...any) {
|
||||
if debug || debugDecoder || debugEncoder {
|
||||
log.Printf(format, a...)
|
||||
}
|
||||
}
|
||||
|
||||
func load3232(b []byte, i int32) uint32 {
|
||||
return binary.LittleEndian.Uint32(b[:len(b):len(b)][i:])
|
||||
return le.Load32(b, i)
|
||||
}
|
||||
|
||||
func load6432(b []byte, i int32) uint64 {
|
||||
return binary.LittleEndian.Uint64(b[:len(b):len(b)][i:])
|
||||
return le.Load64(b, i)
|
||||
}
|
||||
|
||||
type byter interface {
|
||||
|
||||
Reference in New Issue
Block a user