Update

vendor/golang.org/x/text/unicode/norm/composition.go

@@ -449,6 +449,16 @@ func (rb *reorderBuffer) combineHangul(s, i, k int) {
 				// ACxx plus 11Ax to LVT
 				rb.assignRune(s, l+v-jamoTBase)
 			default:
+				// Not a Hangul composition. The segment may still
+				// contain regular canonical compositions, such as
+				// combining marks following a Hangul syllable, so
+				// fall back to the composition table.
+				if b[i].combinesBackward() {
+					if combined := combine(l, v); combined != 0 {
+						rb.assignRune(s, combined)
+						continue
+					}
+				}
 				b[k] = b[i]
 				k++
 			}
@@ -484,25 +494,27 @@ func (rb *reorderBuffer) compose() {
 			return
 		}
 		ii := b[i]
+		// Track the last starter unconditionally: b[i] must be blocked by
+		// any starter between it and s, even one that b[i] itself cannot
+		// combine with, and even if the runes in between never enter the
+		// combinesBackward branch below.
+		cccB := b[k-1].ccc
+		cccC := ii.ccc
+		blocked := false // b[i] blocked by starter or greater or equal CCC?
+		if cccB == 0 {
+			s = k - 1
+		} else {
+			blocked = s != k-1 && cccB >= cccC
+		}
 		// We can only use combineForward as a filter if we later
 		// get the info for the combined character. This is more
 		// expensive than using the filter. Using combinesBackward()
 		// is safe.
-		if ii.combinesBackward() {
-			cccB := b[k-1].ccc
-			cccC := ii.ccc
-			blocked := false // b[i] blocked by starter or greater or equal CCC?
-			if cccB == 0 {
-				s = k - 1
-			} else {
-				blocked = s != k-1 && cccB >= cccC
-			}
-			if !blocked {
-				combined := combine(rb.runeAt(s), rb.runeAt(i))
-				if combined != 0 {
-					rb.assignRune(s, combined)
-					continue
-				}
+		if ii.combinesBackward() && !blocked {
+			combined := combine(rb.runeAt(s), rb.runeAt(i))
+			if combined != 0 {
+				rb.assignRune(s, combined)
+				continue
 			}
 		}
 		b[k] = b[i]

vendor/golang.org/x/text/unicode/norm/forminfo.go

@@ -13,15 +13,18 @@ import "encoding/binary"
 // a rune to a uint16. The values take two forms.  For v >= 0x8000:
 //   bits
 //   15:    1 (inverse of NFD_QC bit of qcInfo)
-//   13..7: qcInfo (see below). isYesD is always true (no decomposition).
+//   12..7: qcInfo (see below). isYesD is always true (no decomposition).
 //    6..0: ccc (compressed CCC value).
 // For v < 0x8000, the respective rune has a decomposition and v is an index
 // into a byte array of UTF-8 decomposition sequences and additional info and
 // has the form:
 //    <header> <decomp_byte>* [<tccc> [<lccc>]]
 // The header contains the number of bytes in the decomposition (excluding this
-// length byte). The two most significant bits of this length byte correspond
-// to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
+// length byte), with 33 mapped to 31 to fit in 5 bits.
+// (If any 31- or 32-byte decompositions come along, we could switch to using
+// use a general lookup table as long as there are at most 32 distinct lengths.)
+// The three most significant bits of this length byte correspond
+// to bit 5, 4, and 3 of qcInfo (see below).  The byte sequence itself starts at v+1.
 // The byte sequence is followed by a trailing and leading CCC if the values
 // for these are not zero.  The value of v determines which ccc are appended
 // to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
@@ -32,8 +35,8 @@ import "encoding/binary"
 
 const (
 	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
-	headerLenMask   = 0x3F // extract the length value from the header byte
-	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
+	headerLenMask   = 0x1F // extract the length value from the header byte (31 => 33)
+	headerFlagsMask = 0xE0 // extract the qcInfo bits from the header byte
 )
 
 // Properties provides access to normalization properties of a rune.
@@ -109,17 +112,21 @@ func (p Properties) BoundaryAfter() bool {
 	return p.isInert()
 }
 
-// We pack quick check data in 4 bits:
+// We pack quick check data in 6 bits:
 //
 //	5:    Combines forward  (0 == false, 1 == true)
 //	4..3: NFC_QC Yes(00), No (10), or Maybe (11)
 //	2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
 //	1..0: Number of trailing non-starters.
 //
-// When all 4 bits are zero, the character is inert, meaning it is never
+// When all 6 bits are zero, the character is inert, meaning it is never
 // influenced by normalization.
+//
+// We set flags to 0x80 (high bit 7 unused in quick check data) to indicate an invalid rune.
 type qcInfo uint8
 
+func (p Properties) isInvalid() bool { return p.flags == 0x80 }
+
 func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
 func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
 
@@ -152,6 +159,9 @@ func (p Properties) Decomposition() []byte {
 	}
 	i := p.index
 	n := decomps[i] & headerLenMask
+	if n == 31 {
+		n = 33
+	}
 	i++
 	return decomps[i : i+uint16(n)]
 }
@@ -181,30 +191,30 @@ func (p Properties) TrailCCC() uint8 {
 	return ccc[p.tccc]
 }
 
+// Recomposition
+// Each entry of recompMapPacked holds the two runes of the key and the
+// composed rune of the value packed into a big-endian uint64 as three
+// 21-bit fields, which is wide enough for any rune.
+// Note that the recomposition map for NFC and NFKC are identical.
+
+const recompShift = 21
+
 func buildRecompMap() {
-	recompMap = make(map[uint32]rune, len(recompMapPacked)/8)
+	recompMap = make(map[uint64]rune, len(recompMapPacked)/8)
 	var buf [8]byte
 	for i := 0; i < len(recompMapPacked); i += 8 {
 		copy(buf[:], recompMapPacked[i:i+8])
-		key := binary.BigEndian.Uint32(buf[:4])
-		val := binary.BigEndian.Uint32(buf[4:])
-		recompMap[key] = rune(val)
+		e := binary.BigEndian.Uint64(buf[:])
+		recompMap[e>>recompShift] = rune(e & (1<<recompShift - 1))
 	}
 }
 
-// Recomposition
-// We use 32-bit keys instead of 64-bit for the two codepoint keys.
-// This clips off the bits of three entries, but we know this will not
-// result in a collision. In the unlikely event that changes to
-// UnicodeData.txt introduce collisions, the compiler will catch it.
-// Note that the recomposition map for NFC and NFKC are identical.
-
 // combine returns the combined rune or 0 if it doesn't exist.
 //
 // The caller is responsible for calling
 // recompMapOnce.Do(buildRecompMap) sometime before this is called.
 func combine(a, b rune) rune {
-	key := uint32(uint16(a))<<16 + uint32(uint16(b))
+	key := uint64(a)<<recompShift | uint64(b)
 	if recompMap == nil {
 		panic("caller error") // see func comment
 	}
@@ -241,6 +251,9 @@ func (f Form) PropertiesString(s string) Properties {
 // to a Properties.  See the comment at the top of the file
 // for more information on the format.
 func compInfo(v uint16, sz int) Properties {
+	if sz == 0 {
+		return Properties{flags: 0x80, size: 1}
+	}
 	if v == 0 {
 		return Properties{size: uint8(sz)}
 	} else if v >= 0x8000 {
@@ -248,7 +261,7 @@ func compInfo(v uint16, sz int) Properties {
 			size:  uint8(sz),
 			ccc:   uint8(v),
 			tccc:  uint8(v),
-			flags: qcInfo(v >> 8),
+			flags: qcInfo(v>>8) & 0x3f,
 		}
 		if p.ccc > 0 || p.combinesBackward() {
 			p.nLead = uint8(p.flags & 0x3)
@@ -260,7 +273,11 @@ func compInfo(v uint16, sz int) Properties {
 	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
 	p := Properties{size: uint8(sz), flags: f, index: v}
 	if v >= firstCCC {
-		v += uint16(h&headerLenMask) + 1
+		n := uint16(h & headerLenMask)
+		if n == 31 {
+			n = 33
+		}
+		v += n + 1
 		c := decomps[v]
 		p.tccc = c >> 2
 		p.flags |= qcInfo(c & 0x3)

vendor/golang.org/x/text/unicode/norm/iter.go

@@ -376,16 +376,12 @@ func nextComposed(i *Iter) []byte {
 			goto doNorm
 		}
 		prevCC = i.info.tccc
-		sz := int(i.info.size)
-		if sz == 0 {
-			sz = 1 // illegal rune: copy byte-by-byte
-		}
-		p := outp + sz
+		p := outp + int(i.info.size)
 		if p > len(i.buf) {
 			break
 		}
 		outp = p
-		i.p += sz
+		i.p += int(i.info.size)
 		if i.p >= i.rb.nsrc {
 			i.setDone()
 			break

vendor/golang.org/x/text/unicode/norm/normalize.go

@@ -148,7 +148,7 @@ func (f Form) IsNormalString(s string) bool {
 // patched buffer and whether the decomposition is still in progress.
 func patchTail(rb *reorderBuffer) bool {
 	info, p := lastRuneStart(&rb.f, rb.out)
-	if p == -1 || info.size == 0 {
+	if p == -1 || info.isInvalid() {
 		return true
 	}
 	end := p + int(info.size)
@@ -225,7 +225,7 @@ func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
 	}
 	fd := &rb.f
 	if doMerge {
-		var info Properties
+		info := Properties{flags: 0x80, size: 1} // invalid rune
 		if p < n {
 			info = fd.info(src, p)
 			if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
@@ -235,7 +235,7 @@ func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
 				p = decomposeSegment(rb, p, true)
 			}
 		}
-		if info.size == 0 {
+		if info.isInvalid() {
 			rb.doFlush()
 			// Append incomplete UTF-8 encoding.
 			return src.appendSlice(rb.out, p, n)
@@ -314,7 +314,7 @@ func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool)
 			continue
 		}
 		info := f.info(src, i)
-		if info.size == 0 {
+		if info.isInvalid() {
 			if atEOF {
 				// include incomplete runes
 				return n, true
@@ -379,7 +379,7 @@ func (f Form) firstBoundary(src input, nsrc int) int {
 	// CGJ insertion points correctly. Luckily it doesn't have to.
 	for {
 		info := fd.info(src, i)
-		if info.size == 0 {
+		if info.isInvalid() {
 			return -1
 		}
 		if s := ss.next(info); s != ssSuccess {
@@ -424,7 +424,7 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
 	}
 	fd := formTable[f]
 	info := fd.info(src, 0)
-	if info.size == 0 {
+	if info.isInvalid() {
 		if atEOF {
 			return 1
 		}
@@ -435,7 +435,7 @@ func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
 
 	for i := int(info.size); i < nsrc; i += int(info.size) {
 		info = fd.info(src, i)
-		if info.size == 0 {
+		if info.isInvalid() {
 			if atEOF {
 				return i
 			}
@@ -465,7 +465,7 @@ func lastBoundary(fd *formInfo, b []byte) int {
 	if p == -1 {
 		return -1
 	}
-	if info.size == 0 { // ends with incomplete rune
+	if info.isInvalid() { // ends with incomplete rune
 		if p == 0 { // starts with incomplete rune
 			return -1
 		}
@@ -504,8 +504,11 @@ func lastBoundary(fd *formInfo, b []byte) int {
 func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
 	// Force one character to be consumed.
 	info := rb.f.info(rb.src, sp)
-	if info.size == 0 {
-		return 0
+	if info.isInvalid() {
+		// Consume the invalid character.
+		// Perhaps this should return iShortSrc instead?
+		// May not matter, this path does not appear to be reachable in practice.
+		return sp + 1
 	}
 	if s := rb.ss.next(info); s == ssStarter {
 		// TODO: this could be removed if we don't support merging.
@@ -528,7 +531,7 @@ func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
 			break
 		}
 		info = rb.f.info(rb.src, sp)
-		if info.size == 0 {
+		if info.isInvalid() {
 			if !atEOF {
 				return int(iShortSrc)
 			}

vendor/golang.org/x/text/unicode/norm/transform.go

@@ -33,7 +33,7 @@ func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
 		return nDst + n, nSrc + n, err
 	}
 
-	if err == nil && n < len(src) && !atEOF {
+	if err == nil && n < len(src) {
 		err = transform.ErrShortSrc
 	}
 	return n, n, err
@@ -67,6 +67,7 @@ func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
 		nSrc = end
 
 		// Next quickSpan.
+		var err error
 		end = rb.nsrc
 		eof := atEOF
 		if n := nSrc + len(dst) - nDst; n < end {
@@ -79,7 +80,7 @@ func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error)
 		nSrc += n
 		nDst += n
 		if ok {
-			if err == nil && n < rb.nsrc && !atEOF {
+			if err == nil && nSrc < rb.nsrc {
 				err = transform.ErrShortSrc
 			}
 			return nDst, nSrc, err