This commit is contained in:
bluepython508 committed 2026-10-22 12:27:19 +01:00
1 parent 3530afb854
commit 6903fcff27
2146 files changed
+112256 -190703

No files matched your search

+13 -36
View File
@@ -39,7 +39,7 @@ func ConsumeWhitespace(b []byte) (n int) {
}
// ConsumeNull consumes the next JSON null literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case consumeLiteral should be used.
// It returns 0 if it is invalid, in which case ConsumeLiteral should be used.
func ConsumeNull(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "null"
@@ -50,7 +50,7 @@ func ConsumeNull(b []byte) int {
}
// ConsumeFalse consumes the next JSON false literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case consumeLiteral should be used.
// It returns 0 if it is invalid, in which case ConsumeLiteral should be used.
func ConsumeFalse(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "false"
@@ -61,7 +61,7 @@ func ConsumeFalse(b []byte) int {
}
// ConsumeTrue consumes the next JSON true literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case consumeLiteral should be used.
// It returns 0 if it is invalid, in which case ConsumeLiteral should be used.
func ConsumeTrue(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "true"
@@ -88,7 +88,7 @@ func ConsumeLiteral(b []byte, lit string) (n int, err error) {
// ConsumeSimpleString consumes the next JSON string per RFC 7159, section 7
// but is limited to the grammar for an ASCII string without escape sequences.
// It returns 0 if it is invalid or more complicated than a simple string,
// in which case consumeString should be called.
// in which case [ConsumeString] should be called.
//
// It rejects '<', '>', and '&' for compatibility reasons since these were
// always escaped in the v1 implementation. Thus, if this function reports
@@ -118,7 +118,7 @@ func ConsumeString(flags *ValueFlags, b []byte, validateUTF8 bool) (n int, err e
return ConsumeStringResumable(flags, b, 0, validateUTF8)
}
// ConsumeStringResumable is identical to consumeString but supports resuming
// ConsumeStringResumable is identical to [ConsumeString] but supports resuming
// from a previous call that returned io.ErrUnexpectedEOF.
func ConsumeStringResumable(flags *ValueFlags, b []byte, resumeOffset int, validateUTF8 bool) (n int, err error) {
// Consume the leading double quote.
@@ -254,7 +254,7 @@ func ConsumeStringResumable(flags *ValueFlags, b []byte, resumeOffset int, valid
// Any invalid UTF-8 within the string will be replaced with utf8.RuneError,
// but the error will be specified as having encountered such an error.
// The input must be an entire JSON string with no surrounding whitespace.
func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) (v []byte, err error) {
func AppendUnquote(dst, src []byte) (v []byte, err error) {
dst = slices.Grow(dst, len(src))
// Consume the leading double quote.
@@ -292,7 +292,7 @@ func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) (v []byte, er
return dst, err
}
switch r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[n:]))); {
switch r, rn := utf8.DecodeRune(src[n:]); {
// Handle UTF-8 encoded byte sequence.
// Due to specialized handling of ASCII above, we know that
// all normal sequences at this point must be 2 bytes or larger.
@@ -364,7 +364,7 @@ func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) (v []byte, er
// Handle invalid UTF-8.
case r == utf8.RuneError:
dst = append(dst, src[i:n]...)
if !utf8.FullRuneInString(string(truncateMaxUTF8(src[n:]))) {
if !utf8.FullRune(src[n:]) {
return dst, io.ErrUnexpectedEOF
}
// NOTE: An unescaped string may be longer than the escaped string
@@ -387,7 +387,7 @@ func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) (v []byte, er
// hasEscapedUTF16Prefix reports whether b is possibly
// the truncated prefix of a \uFFFF escape sequence.
func hasEscapedUTF16Prefix[Bytes ~[]byte | ~string](b Bytes, lowerSurrogateHalf bool) bool {
func hasEscapedUTF16Prefix(b []byte, lowerSurrogateHalf bool) bool {
for i := range len(b) {
switch c := b[i]; {
case i == 0 && c != '\\':
@@ -420,9 +420,9 @@ func UnquoteMayCopy(b []byte, isVerbatim bool) []byte {
}
// ConsumeSimpleNumber consumes the next JSON number per RFC 7159, section 6
// but is limited to the grammar for a positive integer.
// but is limited to the grammar for a non-negative integer.
// It returns 0 if it is invalid or more complicated than a simple integer,
// in which case consumeNumber should be called.
// in which case ConsumeNumber should be called.
func ConsumeSimpleNumber(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 {
@@ -467,7 +467,7 @@ func ConsumeNumber(b []byte) (n int, err error) {
return n, err
}
// ConsumeNumberResumable is identical to consumeNumber but supports resuming
// ConsumeNumberResumable is identical to ConsumeNumber but supports resuming
// from a previous call that returned io.ErrUnexpectedEOF.
func ConsumeNumberResumable(b []byte, resumeOffset int, state ConsumeNumberState) (n int, _ ConsumeNumberState, err error) {
// Jump to the right state when resuming from a partial consumption.
@@ -565,7 +565,7 @@ beforeExponent:
// parseHexUint16 is similar to strconv.ParseUint,
// but operates directly on []byte and is optimized for base-16.
// See https://go.dev/issue/42429.
func parseHexUint16[Bytes ~[]byte | ~string](b Bytes) (v uint16, ok bool) {
func parseHexUint16(b []byte) (v uint16, ok bool) {
if len(b) != 4 {
return 0, false
}
@@ -604,26 +604,3 @@ func ParseUint(b []byte) (v uint64, ok bool) {
}
return v, true
}
// ParseFloat parses a floating point number according to the Go float grammar.
// Note that the JSON number grammar is a strict subset.
//
// If the number overflows the finite representation of a float,
// then we return MaxFloat since any finite value will always be infinitely
// more accurate at representing another finite value than an infinite value.
func ParseFloat(b []byte, bits int) (v float64, ok bool) {
fv, err := strconv.ParseFloat(string(b), bits)
if math.IsInf(fv, 0) {
switch {
case bits == 32 && math.IsInf(fv, +1):
fv = +math.MaxFloat32
case bits == 64 && math.IsInf(fv, +1):
fv = +math.MaxFloat64
case bits == 32 && math.IsInf(fv, -1):
fv = -math.MaxFloat32
case bits == 64 && math.IsInf(fv, -1):
fv = -math.MaxFloat64
}
}
return fv, err == nil
}
+7 -7
View File
@@ -32,7 +32,7 @@ var escapeASCII = [...]uint8{
// NeedEscape reports whether src needs escaping of any characters.
// It conservatively assumes EscapeForHTML and EscapeForJS.
// It reports true for inputs with invalid UTF-8.
func NeedEscape[Bytes ~[]byte | ~string](src Bytes) bool {
func NeedEscape(src []byte) bool {
var i int
for uint(len(src)) > uint(i) {
if c := src[i]; c < utf8.RuneSelf {
@@ -41,7 +41,7 @@ func NeedEscape[Bytes ~[]byte | ~string](src Bytes) bool {
}
i++
} else {
r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[i:])))
r, rn := utf8.DecodeRune(src[i:])
if r == utf8.RuneError || r == '\u2028' || r == '\u2029' {
return true
}
@@ -62,7 +62,7 @@ func NeedEscape[Bytes ~[]byte | ~string](src Bytes) bool {
// invalid bytes are replaced with the Unicode replacement character ('\ufffd').
// If no escape flags are set, then the shortest representable form is used,
// which is also the canonical form for strings (RFC 8785, section 3.2.2.2).
func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes, flags *jsonflags.Flags) ([]byte, error) {
func AppendQuote(dst, src []byte, flags *jsonflags.Flags) ([]byte, error) {
var i, n int
var hasInvalidUTF8 bool
dst = slices.Grow(dst, len(`"`)+len(src)+len(`"`))
@@ -82,7 +82,7 @@ func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes, flags *jsonflag
}
} else {
// Handle multi-byte Unicode.
r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[n:])))
r, rn := utf8.DecodeRune(src[n:])
n += rn
if r != utf8.RuneError && r != '\u2028' && r != '\u2029' {
continue // no escaping possibly needed
@@ -178,7 +178,7 @@ func ReformatString(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error
}
i++
} else {
r, rn := utf8.DecodeRune(truncateMaxUTF8(src[i:]))
r, rn := utf8.DecodeRune(src[i:])
if (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS) {
dst = append(dst, src[lastAppendIndex:i]...)
dst = appendEscapedUnicode(dst, r)
@@ -193,7 +193,7 @@ func ReformatString(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error
// The input contains characters that might need escaping,
// unnecessary escape sequences, or invalid UTF-8.
// Perform a round-trip unquote and quote to properly reformat
// these sequences according the current flags.
// these sequences according to the current flags.
b, _ := AppendUnquote(nil, src[:n])
dst, _ = AppendQuote(dst, b, flags)
return dst, n, nil
@@ -235,7 +235,7 @@ func AppendFloat(dst []byte, src float64, bits int) []byte {
return dst
}
// ReformatNumber consumes a JSON string from src and appends it to dst,
// ReformatNumber consumes a JSON number from src and appends it to dst,
// canonicalizing it if specified.
// It returns the appended output and the number of consumed input bytes.
func ReformatNumber(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error) {
+30 -38
View File
@@ -17,7 +17,7 @@ import (
"unicode/utf8"
)
// TrimSuffixWhitespace trims JSON from the end of b.
// TrimSuffixWhitespace trims JSON whitespace from the end of b.
func TrimSuffixWhitespace(b []byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
n := len(b) - 1
@@ -59,8 +59,8 @@ func TrimSuffixByte(b []byte, c byte) []byte {
}
// QuoteRune quotes the first rune in the input.
func QuoteRune[Bytes ~[]byte | ~string](b Bytes) string {
r, n := utf8.DecodeRuneInString(string(truncateMaxUTF8(b)))
func QuoteRune(b []byte) string {
r, n := utf8.DecodeRune(b)
if r == utf8.RuneError && n == 1 {
return `'\x` + strconv.FormatUint(uint64(b[0]), 16) + `'`
}
@@ -70,7 +70,7 @@ func QuoteRune[Bytes ~[]byte | ~string](b Bytes) string {
// CompareUTF16 lexicographically compares x to y according
// to the UTF-16 codepoints of the UTF-8 encoded input strings.
// This implements the ordering specified in RFC 8785, section 3.2.3.
func CompareUTF16[Bytes ~[]byte | ~string](x, y Bytes) int {
func CompareUTF16(x, y []byte) int {
// NOTE: This is an optimized, mostly allocation-free implementation
// of CompareUTF16Simple in wire_test.go. FuzzCompareUTF16 verifies that the
// two implementations agree on the result of comparing any two strings.
@@ -93,8 +93,8 @@ func CompareUTF16[Bytes ~[]byte | ~string](x, y Bytes) int {
}
// Decode next pair of runes as UTF-8.
rx, nx := utf8.DecodeRuneInString(string(truncateMaxUTF8(x)))
ry, ny := utf8.DecodeRuneInString(string(truncateMaxUTF8(y)))
rx, nx := utf8.DecodeRune(x)
ry, ny := utf8.DecodeRune(y)
selfx := isUTF16Self(rx)
selfy := isUTF16Self(ry)
@@ -123,49 +123,41 @@ func CompareUTF16[Bytes ~[]byte | ~string](x, y Bytes) int {
}
}
// truncateMaxUTF8 truncates b such it contains at least one rune.
//
// The utf8 package currently lacks generic variants, which complicates
// generic functions that operates on either []byte or string.
// As a hack, we always call the utf8 function operating on strings,
// but always truncate the input such that the result is identical.
//
// Example usage:
//
// utf8.DecodeRuneInString(string(truncateMaxUTF8(b)))
//
// Converting a []byte to a string is stack allocated since
// truncateMaxUTF8 guarantees that the []byte is short.
func truncateMaxUTF8[Bytes ~[]byte | ~string](b Bytes) Bytes {
// TODO(https://go.dev/issue/56948): Remove this function and
// instead directly call generic utf8 functions wherever used.
if len(b) > utf8.UTFMax {
return b[:utf8.UTFMax]
}
return b
}
// TODO(https://go.dev/issue/70547): Use utf8.ErrInvalid instead.
var ErrInvalidUTF8 = errors.New("invalid UTF-8")
func NewInvalidCharacterError[Bytes ~[]byte | ~string](prefix Bytes, where string) error {
what := QuoteRune(prefix)
return errors.New("invalid character " + what + " " + where)
_, n := utf8.DecodeRune([]byte(prefix))
return &InvalidTextError{"character", string(prefix[:n]), where}
}
func NewInvalidEscapeSequenceError[Bytes ~[]byte | ~string](what Bytes) error {
label := "escape sequence"
if len(what) > 6 {
label = "surrogate pair"
return &InvalidTextError{"surrogate pair", string(what), "in string"}
}
needEscape := strings.IndexFunc(string(what), func(r rune) bool {
return &InvalidTextError{"escape sequence", string(what), "in string"}
}
type InvalidTextError struct {
Label string // e.g., "character" | "escape sequence" | "surrogate pair"
What string // raw invalid text
Where string // e.g., "in string" | "at start of value"
}
func (e *InvalidTextError) Error() string {
var what string
needEscape := strings.ContainsFunc(e.What, func(r rune) bool {
return r == '`' || r == utf8.RuneError || unicode.IsSpace(r) || !unicode.IsPrint(r)
}) >= 0
if needEscape {
return errors.New("invalid " + label + " " + strconv.Quote(string(what)) + " in string")
} else {
return errors.New("invalid " + label + " `" + string(what) + "` in string")
})
switch {
case utf8.RuneCount([]byte(e.What)) == 1:
what = QuoteRune([]byte(e.What))
case needEscape:
what = strconv.Quote(e.What)
default:
what = "`" + e.What + "`"
}
return strings.TrimSuffix("invalid "+e.Label+" "+what+" "+e.Where, " ")
}
// TruncatePointer optionally truncates the JSON pointer,