Initial

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

vendor/golang.org/x/text/unicode/bidi/bidi.go

@@ -0,0 +1,198 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run gen.go gen_trieval.go gen_ranges.go
+
+// Package bidi contains functionality for bidirectional text support.
+//
+// See https://www.unicode.org/reports/tr9.
+//
+// NOTE: UNDER CONSTRUCTION. This API may change in backwards incompatible ways
+// and without notice.
+package bidi // import "golang.org/x/text/unicode/bidi"
+
+// TODO:
+// The following functionality would not be hard to implement, but hinges on
+// the definition of a Segmenter interface. For now this is up to the user.
+// - Iterate over paragraphs
+// - Segmenter to iterate over runs directly from a given text.
+// Also:
+// - Transformer for reordering?
+// - Transformer (validator, really) for Bidi Rule.
+
+// This API tries to avoid dealing with embedding levels for now. Under the hood
+// these will be computed, but the question is to which extent the user should
+// know they exist. We should at some point allow the user to specify an
+// embedding hierarchy, though.
+
+// A Direction indicates the overall flow of text.
+type Direction int
+
+const (
+	// LeftToRight indicates the text contains no right-to-left characters and
+	// that either there are some left-to-right characters or the option
+	// DefaultDirection(LeftToRight) was passed.
+	LeftToRight Direction = iota
+
+	// RightToLeft indicates the text contains no left-to-right characters and
+	// that either there are some right-to-left characters or the option
+	// DefaultDirection(RightToLeft) was passed.
+	RightToLeft
+
+	// Mixed indicates text contains both left-to-right and right-to-left
+	// characters.
+	Mixed
+
+	// Neutral means that text contains no left-to-right and right-to-left
+	// characters and that no default direction has been set.
+	Neutral
+)
+
+type options struct{}
+
+// An Option is an option for Bidi processing.
+type Option func(*options)
+
+// ICU allows the user to define embedding levels. This may be used, for example,
+// to use hierarchical structure of markup languages to define embeddings.
+// The following option may be a way to expose this functionality in this API.
+// // LevelFunc sets a function that associates nesting levels with the given text.
+// // The levels function will be called with monotonically increasing values for p.
+// func LevelFunc(levels func(p int) int) Option {
+// 	panic("unimplemented")
+// }
+
+// DefaultDirection sets the default direction for a Paragraph. The direction is
+// overridden if the text contains directional characters.
+func DefaultDirection(d Direction) Option {
+	panic("unimplemented")
+}
+
+// A Paragraph holds a single Paragraph for Bidi processing.
+type Paragraph struct {
+	// buffers
+}
+
+// SetBytes configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetBytes(b []byte, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// SetString configures p for the given paragraph text. It replaces text
+// previously set by SetBytes or SetString. If b contains a paragraph separator
+// it will only process the first paragraph and report the number of bytes
+// consumed from b including this separator. Error may be non-nil if options are
+// given.
+func (p *Paragraph) SetString(s string, opts ...Option) (n int, err error) {
+	panic("unimplemented")
+}
+
+// IsLeftToRight reports whether the principle direction of rendering for this
+// paragraphs is left-to-right. If this returns false, the principle direction
+// of rendering is right-to-left.
+func (p *Paragraph) IsLeftToRight() bool {
+	panic("unimplemented")
+}
+
+// Direction returns the direction of the text of this paragraph.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (p *Paragraph) Direction() Direction {
+	panic("unimplemented")
+}
+
+// RunAt reports the Run at the given position of the input text.
+//
+// This method can be used for computing line breaks on paragraphs.
+func (p *Paragraph) RunAt(pos int) Run {
+	panic("unimplemented")
+}
+
+// Order computes the visual ordering of all the runs in a Paragraph.
+func (p *Paragraph) Order() (Ordering, error) {
+	panic("unimplemented")
+}
+
+// Line computes the visual ordering of runs for a single line starting and
+// ending at the given positions in the original text.
+func (p *Paragraph) Line(start, end int) (Ordering, error) {
+	panic("unimplemented")
+}
+
+// An Ordering holds the computed visual order of runs of a Paragraph. Calling
+// SetBytes or SetString on the originating Paragraph invalidates an Ordering.
+// The methods of an Ordering should only be called by one goroutine at a time.
+type Ordering struct{}
+
+// Direction reports the directionality of the runs.
+//
+// The direction may be LeftToRight, RightToLeft, Mixed, or Neutral.
+func (o *Ordering) Direction() Direction {
+	panic("unimplemented")
+}
+
+// NumRuns returns the number of runs.
+func (o *Ordering) NumRuns() int {
+	panic("unimplemented")
+}
+
+// Run returns the ith run within the ordering.
+func (o *Ordering) Run(i int) Run {
+	panic("unimplemented")
+}
+
+// TODO: perhaps with options.
+// // Reorder creates a reader that reads the runes in visual order per character.
+// // Modifiers remain after the runes they modify.
+// func (l *Runs) Reorder() io.Reader {
+// 	panic("unimplemented")
+// }
+
+// A Run is a continuous sequence of characters of a single direction.
+type Run struct {
+}
+
+// String returns the text of the run in its original order.
+func (r *Run) String() string {
+	panic("unimplemented")
+}
+
+// Bytes returns the text of the run in its original order.
+func (r *Run) Bytes() []byte {
+	panic("unimplemented")
+}
+
+// TODO: methods for
+// - Display order
+// - headers and footers
+// - bracket replacement.
+
+// Direction reports the direction of the run.
+func (r *Run) Direction() Direction {
+	panic("unimplemented")
+}
+
+// Position of the Run within the text passed to SetBytes or SetString of the
+// originating Paragraph value.
+func (r *Run) Pos() (start, end int) {
+	panic("unimplemented")
+}
+
+// AppendReverse reverses the order of characters of in, appends them to out,
+// and returns the result. Modifiers will still follow the runes they modify.
+// Brackets are replaced with their counterparts.
+func AppendReverse(out, in []byte) []byte {
+	panic("unimplemented")
+}
+
+// ReverseString reverses the order of characters in s and returns a new string.
+// Modifiers will still follow the runes they modify. Brackets are replaced with
+// their counterparts.
+func ReverseString(s string) string {
+	panic("unimplemented")
+}

vendor/golang.org/x/text/unicode/bidi/bracket.go

@@ -0,0 +1,335 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import (
+	"container/list"
+	"fmt"
+	"sort"
+)
+
+// This file contains a port of the reference implementation of the
+// Bidi Parentheses Algorithm:
+// https://www.unicode.org/Public/PROGRAMS/BidiReferenceJava/BidiPBAReference.java
+//
+// The implementation in this file covers definitions BD14-BD16 and rule N0
+// of UAX#9.
+//
+// Some preprocessing is done for each rune before data is passed to this
+// algorithm:
+//  - opening and closing brackets are identified
+//  - a bracket pair type, like '(' and ')' is assigned a unique identifier that
+//    is identical for the opening and closing bracket. It is left to do these
+//    mappings.
+//  - The BPA algorithm requires that bracket characters that are canonical
+//    equivalents of each other be able to be substituted for each other.
+//    It is the responsibility of the caller to do this canonicalization.
+//
+// In implementing BD16, this implementation departs slightly from the "logical"
+// algorithm defined in UAX#9. In particular, the stack referenced there
+// supports operations that go beyond a "basic" stack. An equivalent
+// implementation based on a linked list is used here.
+
+// Bidi_Paired_Bracket_Type
+// BD14. An opening paired bracket is a character whose
+// Bidi_Paired_Bracket_Type property value is Open.
+//
+// BD15. A closing paired bracket is a character whose
+// Bidi_Paired_Bracket_Type property value is Close.
+type bracketType byte
+
+const (
+	bpNone bracketType = iota
+	bpOpen
+	bpClose
+)
+
+// bracketPair holds a pair of index values for opening and closing bracket
+// location of a bracket pair.
+type bracketPair struct {
+	opener int
+	closer int
+}
+
+func (b *bracketPair) String() string {
+	return fmt.Sprintf("(%v, %v)", b.opener, b.closer)
+}
+
+// bracketPairs is a slice of bracketPairs with a sort.Interface implementation.
+type bracketPairs []bracketPair
+
+func (b bracketPairs) Len() int           { return len(b) }
+func (b bracketPairs) Swap(i, j int)      { b[i], b[j] = b[j], b[i] }
+func (b bracketPairs) Less(i, j int) bool { return b[i].opener < b[j].opener }
+
+// resolvePairedBrackets runs the paired bracket part of the UBA algorithm.
+//
+// For each rune, it takes the indexes into the original string, the class the
+// bracket type (in pairTypes) and the bracket identifier (pairValues). It also
+// takes the direction type for the start-of-sentence and the embedding level.
+//
+// The identifiers for bracket types are the rune of the canonicalized opening
+// bracket for brackets (open or close) or 0 for runes that are not brackets.
+func resolvePairedBrackets(s *isolatingRunSequence) {
+	p := bracketPairer{
+		sos:              s.sos,
+		openers:          list.New(),
+		codesIsolatedRun: s.types,
+		indexes:          s.indexes,
+	}
+	dirEmbed := L
+	if s.level&1 != 0 {
+		dirEmbed = R
+	}
+	p.locateBrackets(s.p.pairTypes, s.p.pairValues)
+	p.resolveBrackets(dirEmbed, s.p.initialTypes)
+}
+
+type bracketPairer struct {
+	sos Class // direction corresponding to start of sequence
+
+	// The following is a restatement of BD 16 using non-algorithmic language.
+	//
+	// A bracket pair is a pair of characters consisting of an opening
+	// paired bracket and a closing paired bracket such that the
+	// Bidi_Paired_Bracket property value of the former equals the latter,
+	// subject to the following constraints.
+	// - both characters of a pair occur in the same isolating run sequence
+	// - the closing character of a pair follows the opening character
+	// - any bracket character can belong at most to one pair, the earliest possible one
+	// - any bracket character not part of a pair is treated like an ordinary character
+	// - pairs may nest properly, but their spans may not overlap otherwise
+
+	// Bracket characters with canonical decompositions are supposed to be
+	// treated as if they had been normalized, to allow normalized and non-
+	// normalized text to give the same result. In this implementation that step
+	// is pushed out to the caller. The caller has to ensure that the pairValue
+	// slices contain the rune of the opening bracket after normalization for
+	// any opening or closing bracket.
+
+	openers *list.List // list of positions for opening brackets
+
+	// bracket pair positions sorted by location of opening bracket
+	pairPositions bracketPairs
+
+	codesIsolatedRun []Class // directional bidi codes for an isolated run
+	indexes          []int   // array of index values into the original string
+
+}
+
+// matchOpener reports whether characters at given positions form a matching
+// bracket pair.
+func (p *bracketPairer) matchOpener(pairValues []rune, opener, closer int) bool {
+	return pairValues[p.indexes[opener]] == pairValues[p.indexes[closer]]
+}
+
+const maxPairingDepth = 63
+
+// locateBrackets locates matching bracket pairs according to BD16.
+//
+// This implementation uses a linked list instead of a stack, because, while
+// elements are added at the front (like a push) they are not generally removed
+// in atomic 'pop' operations, reducing the benefit of the stack archetype.
+func (p *bracketPairer) locateBrackets(pairTypes []bracketType, pairValues []rune) {
+	// traverse the run
+	// do that explicitly (not in a for-each) so we can record position
+	for i, index := range p.indexes {
+
+		// look at the bracket type for each character
+		if pairTypes[index] == bpNone || p.codesIsolatedRun[i] != ON {
+			// continue scanning
+			continue
+		}
+		switch pairTypes[index] {
+		case bpOpen:
+			// check if maximum pairing depth reached
+			if p.openers.Len() == maxPairingDepth {
+				p.openers.Init()
+				return
+			}
+			// remember opener location, most recent first
+			p.openers.PushFront(i)
+
+		case bpClose:
+			// see if there is a match
+			count := 0
+			for elem := p.openers.Front(); elem != nil; elem = elem.Next() {
+				count++
+				opener := elem.Value.(int)
+				if p.matchOpener(pairValues, opener, i) {
+					// if the opener matches, add nested pair to the ordered list
+					p.pairPositions = append(p.pairPositions, bracketPair{opener, i})
+					// remove up to and including matched opener
+					for ; count > 0; count-- {
+						p.openers.Remove(p.openers.Front())
+					}
+					break
+				}
+			}
+			sort.Sort(p.pairPositions)
+			// if we get here, the closing bracket matched no openers
+			// and gets ignored
+		}
+	}
+}
+
+// Bracket pairs within an isolating run sequence are processed as units so
+// that both the opening and the closing paired bracket in a pair resolve to
+// the same direction.
+//
+// N0. Process bracket pairs in an isolating run sequence sequentially in
+// the logical order of the text positions of the opening paired brackets
+// using the logic given below. Within this scope, bidirectional types EN
+// and AN are treated as R.
+//
+// Identify the bracket pairs in the current isolating run sequence
+// according to BD16. For each bracket-pair element in the list of pairs of
+// text positions:
+//
+// a Inspect the bidirectional types of the characters enclosed within the
+// bracket pair.
+//
+// b If any strong type (either L or R) matching the embedding direction is
+// found, set the type for both brackets in the pair to match the embedding
+// direction.
+//
+// o [ e ] o -> o e e e o
+//
+// o [ o e ] -> o e o e e
+//
+// o [ NI e ] -> o e NI e e
+//
+// c Otherwise, if a strong type (opposite the embedding direction) is
+// found, test for adjacent strong types as follows: 1 First, check
+// backwards before the opening paired bracket until the first strong type
+// (L, R, or sos) is found. If that first preceding strong type is opposite
+// the embedding direction, then set the type for both brackets in the pair
+// to that type. 2 Otherwise, set the type for both brackets in the pair to
+// the embedding direction.
+//
+// o [ o ] e -> o o o o e
+//
+// o [ o NI ] o -> o o o NI o o
+//
+// e [ o ] o -> e e o e o
+//
+// e [ o ] e -> e e o e e
+//
+// e ( o [ o ] NI ) e -> e e o o o o NI e e
+//
+// d Otherwise, do not set the type for the current bracket pair. Note that
+// if the enclosed text contains no strong types the paired brackets will
+// both resolve to the same level when resolved individually using rules N1
+// and N2.
+//
+// e ( NI ) o -> e ( NI ) o
+
+// getStrongTypeN0 maps character's directional code to strong type as required
+// by rule N0.
+//
+// TODO: have separate type for "strong" directionality.
+func (p *bracketPairer) getStrongTypeN0(index int) Class {
+	switch p.codesIsolatedRun[index] {
+	// in the scope of N0, number types are treated as R
+	case EN, AN, AL, R:
+		return R
+	case L:
+		return L
+	default:
+		return ON
+	}
+}
+
+// classifyPairContent reports the strong types contained inside a Bracket Pair,
+// assuming the given embedding direction.
+//
+// It returns ON if no strong type is found. If a single strong type is found,
+// it returns this type. Otherwise it returns the embedding direction.
+//
+// TODO: use separate type for "strong" directionality.
+func (p *bracketPairer) classifyPairContent(loc bracketPair, dirEmbed Class) Class {
+	dirOpposite := ON
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		dir := p.getStrongTypeN0(i)
+		if dir == ON {
+			continue
+		}
+		if dir == dirEmbed {
+			return dir // type matching embedding direction found
+		}
+		dirOpposite = dir
+	}
+	// return ON if no strong type found, or class opposite to dirEmbed
+	return dirOpposite
+}
+
+// classBeforePair determines which strong types are present before a Bracket
+// Pair. Return R or L if strong type found, otherwise ON.
+func (p *bracketPairer) classBeforePair(loc bracketPair) Class {
+	for i := loc.opener - 1; i >= 0; i-- {
+		if dir := p.getStrongTypeN0(i); dir != ON {
+			return dir
+		}
+	}
+	// no strong types found, return sos
+	return p.sos
+}
+
+// assignBracketType implements rule N0 for a single bracket pair.
+func (p *bracketPairer) assignBracketType(loc bracketPair, dirEmbed Class, initialTypes []Class) {
+	// rule "N0, a", inspect contents of pair
+	dirPair := p.classifyPairContent(loc, dirEmbed)
+
+	// dirPair is now L, R, or N (no strong type found)
+
+	// the following logical tests are performed out of order compared to
+	// the statement of the rules but yield the same results
+	if dirPair == ON {
+		return // case "d" - nothing to do
+	}
+
+	if dirPair != dirEmbed {
+		// case "c": strong type found, opposite - check before (c.1)
+		dirPair = p.classBeforePair(loc)
+		if dirPair == dirEmbed || dirPair == ON {
+			// no strong opposite type found before - use embedding (c.2)
+			dirPair = dirEmbed
+		}
+	}
+	// else: case "b", strong type found matching embedding,
+	// no explicit action needed, as dirPair is already set to embedding
+	// direction
+
+	// set the bracket types to the type found
+	p.setBracketsToType(loc, dirPair, initialTypes)
+}
+
+func (p *bracketPairer) setBracketsToType(loc bracketPair, dirPair Class, initialTypes []Class) {
+	p.codesIsolatedRun[loc.opener] = dirPair
+	p.codesIsolatedRun[loc.closer] = dirPair
+
+	for i := loc.opener + 1; i < loc.closer; i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+
+	for i := loc.closer + 1; i < len(p.indexes); i++ {
+		index := p.indexes[i]
+		if initialTypes[index] != NSM {
+			break
+		}
+		p.codesIsolatedRun[i] = dirPair
+	}
+}
+
+// resolveBrackets implements rule N0 for a list of pairs.
+func (p *bracketPairer) resolveBrackets(dirEmbed Class, initialTypes []Class) {
+	for _, loc := range p.pairPositions {
+		p.assignBracketType(loc, dirEmbed, initialTypes)
+	}
+}

vendor/golang.org/x/text/unicode/bidi/gen.go

@@ -0,0 +1,133 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"flag"
+	"log"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/triegen"
+	"golang.org/x/text/internal/ucd"
+)
+
+var outputFile = flag.String("out", "tables.go", "output file")
+
+func main() {
+	gen.Init()
+	gen.Repackage("gen_trieval.go", "trieval.go", "bidi")
+	gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi")
+
+	genTables()
+}
+
+// bidiClass names and codes taken from class "bc" in
+// https://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt
+var bidiClass = map[string]Class{
+	"AL":  AL,  // ArabicLetter
+	"AN":  AN,  // ArabicNumber
+	"B":   B,   // ParagraphSeparator
+	"BN":  BN,  // BoundaryNeutral
+	"CS":  CS,  // CommonSeparator
+	"EN":  EN,  // EuropeanNumber
+	"ES":  ES,  // EuropeanSeparator
+	"ET":  ET,  // EuropeanTerminator
+	"L":   L,   // LeftToRight
+	"NSM": NSM, // NonspacingMark
+	"ON":  ON,  // OtherNeutral
+	"R":   R,   // RightToLeft
+	"S":   S,   // SegmentSeparator
+	"WS":  WS,  // WhiteSpace
+
+	"FSI": Control,
+	"PDF": Control,
+	"PDI": Control,
+	"LRE": Control,
+	"LRI": Control,
+	"LRO": Control,
+	"RLE": Control,
+	"RLI": Control,
+	"RLO": Control,
+}
+
+func genTables() {
+	if numClass > 0x0F {
+		log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass)
+	}
+	w := gen.NewCodeWriter()
+	defer w.WriteVersionedGoFile(*outputFile, "bidi")
+
+	gen.WriteUnicodeVersion(w)
+
+	t := triegen.NewTrie("bidi")
+
+	// Build data about bracket mapping. These bits need to be or-ed with
+	// any other bits.
+	orMask := map[rune]uint64{}
+
+	xorMap := map[rune]int{}
+	xorMasks := []rune{0} // First value is no-op.
+
+	ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) {
+		r1 := p.Rune(0)
+		r2 := p.Rune(1)
+		xor := r1 ^ r2
+		if _, ok := xorMap[xor]; !ok {
+			xorMap[xor] = len(xorMasks)
+			xorMasks = append(xorMasks, xor)
+		}
+		entry := uint64(xorMap[xor]) << xorMaskShift
+		switch p.String(2) {
+		case "o":
+			entry |= openMask
+		case "c", "n":
+		default:
+			log.Fatalf("Unknown bracket class %q.", p.String(2))
+		}
+		orMask[r1] = entry
+	})
+
+	w.WriteComment(`
+	xorMasks contains masks to be xor-ed with brackets to get the reverse
+	version.`)
+	w.WriteVar("xorMasks", xorMasks)
+
+	done := map[rune]bool{}
+
+	insert := func(r rune, c Class) {
+		if !done[r] {
+			t.Insert(r, orMask[r]|uint64(c))
+			done[r] = true
+		}
+	}
+
+	// Insert the derived BiDi properties.
+	ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) {
+		r := p.Rune(0)
+		class, ok := bidiClass[p.String(1)]
+		if !ok {
+			log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1))
+		}
+		insert(r, class)
+	})
+	visitDefaults(insert)
+
+	// TODO: use sparse blocks. This would reduce table size considerably
+	// from the looks of it.
+
+	sz, err := t.Gen(w)
+	if err != nil {
+		log.Fatal(err)
+	}
+	w.Size += sz
+}
+
+// dummy values to make methods in gen_common compile. The real versions
+// will be generated by this file to tables.go.
+var (
+	xorMasks []rune
+)

vendor/golang.org/x/text/unicode/bidi/gen_ranges.go

@@ -0,0 +1,57 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"unicode"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/ucd"
+	"golang.org/x/text/unicode/rangetable"
+)
+
+// These tables are hand-extracted from:
+// https://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt
+func visitDefaults(fn func(r rune, c Class)) {
+	// first write default values for ranges listed above.
+	visitRunes(fn, AL, []rune{
+		0x0600, 0x07BF, // Arabic
+		0x08A0, 0x08FF, // Arabic Extended-A
+		0xFB50, 0xFDCF, // Arabic Presentation Forms
+		0xFDF0, 0xFDFF,
+		0xFE70, 0xFEFF,
+		0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols
+	})
+	visitRunes(fn, R, []rune{
+		0x0590, 0x05FF, // Hebrew
+		0x07C0, 0x089F, // Nko et al.
+		0xFB1D, 0xFB4F,
+		0x00010800, 0x00010FFF, // Cypriot Syllabary et. al.
+		0x0001E800, 0x0001EDFF,
+		0x0001EF00, 0x0001EFFF,
+	})
+	visitRunes(fn, ET, []rune{ // European Terminator
+		0x20A0, 0x20Cf, // Currency symbols
+	})
+	rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) {
+		fn(r, BN) // Boundary Neutral
+	})
+	ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) {
+		if p.String(1) == "Default_Ignorable_Code_Point" {
+			fn(p.Rune(0), BN) // Boundary Neutral
+		}
+	})
+}
+
+func visitRunes(fn func(r rune, c Class), c Class, runes []rune) {
+	for i := 0; i < len(runes); i += 2 {
+		lo, hi := runes[i], runes[i+1]
+		for j := lo; j <= hi; j++ {
+			fn(j, c)
+		}
+	}
+}

vendor/golang.org/x/text/unicode/bidi/gen_trieval.go

@@ -0,0 +1,64 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+// Class is the Unicode BiDi class. Each rune has a single class.
+type Class uint
+
+const (
+	L       Class = iota // LeftToRight
+	R                    // RightToLeft
+	EN                   // EuropeanNumber
+	ES                   // EuropeanSeparator
+	ET                   // EuropeanTerminator
+	AN                   // ArabicNumber
+	CS                   // CommonSeparator
+	B                    // ParagraphSeparator
+	S                    // SegmentSeparator
+	WS                   // WhiteSpace
+	ON                   // OtherNeutral
+	BN                   // BoundaryNeutral
+	NSM                  // NonspacingMark
+	AL                   // ArabicLetter
+	Control              // Control LRO - PDI
+
+	numClass
+
+	LRO // LeftToRightOverride
+	RLO // RightToLeftOverride
+	LRE // LeftToRightEmbedding
+	RLE // RightToLeftEmbedding
+	PDF // PopDirectionalFormat
+	LRI // LeftToRightIsolate
+	RLI // RightToLeftIsolate
+	FSI // FirstStrongIsolate
+	PDI // PopDirectionalIsolate
+
+	unknownClass = ^Class(0)
+)
+
+var controlToClass = map[rune]Class{
+	0x202D: LRO, // LeftToRightOverride,
+	0x202E: RLO, // RightToLeftOverride,
+	0x202A: LRE, // LeftToRightEmbedding,
+	0x202B: RLE, // RightToLeftEmbedding,
+	0x202C: PDF, // PopDirectionalFormat,
+	0x2066: LRI, // LeftToRightIsolate,
+	0x2067: RLI, // RightToLeftIsolate,
+	0x2068: FSI, // FirstStrongIsolate,
+	0x2069: PDI, // PopDirectionalIsolate,
+}
+
+// A trie entry has the following bits:
+// 7..5  XOR mask for brackets
+// 4     1: Bracket open, 0: Bracket close
+// 3..0  Class type
+
+const (
+	openMask     = 0x10
+	xorMaskShift = 5
+)

vendor/golang.org/x/text/unicode/bidi/prop.go

@@ -0,0 +1,206 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package bidi
+
+import "unicode/utf8"
+
+// Properties provides access to BiDi properties of runes.
+type Properties struct {
+	entry uint8
+	last  uint8
+}
+
+var trie = newBidiTrie(0)
+
+// TODO: using this for bidirule reduces the running time by about 5%. Consider
+// if this is worth exposing or if we can find a way to speed up the Class
+// method.
+//
+// // CompactClass is like Class, but maps all of the BiDi control classes
+// // (LRO, RLO, LRE, RLE, PDF, LRI, RLI, FSI, PDI) to the class Control.
+// func (p Properties) CompactClass() Class {
+// 	return Class(p.entry & 0x0F)
+// }
+
+// Class returns the Bidi class for p.
+func (p Properties) Class() Class {
+	c := Class(p.entry & 0x0F)
+	if c == Control {
+		c = controlByteToClass[p.last&0xF]
+	}
+	return c
+}
+
+// IsBracket reports whether the rune is a bracket.
+func (p Properties) IsBracket() bool { return p.entry&0xF0 != 0 }
+
+// IsOpeningBracket reports whether the rune is an opening bracket.
+// IsBracket must return true.
+func (p Properties) IsOpeningBracket() bool { return p.entry&openMask != 0 }
+
+// TODO: find a better API and expose.
+func (p Properties) reverseBracket(r rune) rune {
+	return xorMasks[p.entry>>xorMaskShift] ^ r
+}
+
+var controlByteToClass = [16]Class{
+	0xD: LRO, // U+202D LeftToRightOverride,
+	0xE: RLO, // U+202E RightToLeftOverride,
+	0xA: LRE, // U+202A LeftToRightEmbedding,
+	0xB: RLE, // U+202B RightToLeftEmbedding,
+	0xC: PDF, // U+202C PopDirectionalFormat,
+	0x6: LRI, // U+2066 LeftToRightIsolate,
+	0x7: RLI, // U+2067 RightToLeftIsolate,
+	0x8: FSI, // U+2068 FirstStrongIsolate,
+	0x9: PDI, // U+2069 PopDirectionalIsolate,
+}
+
+// LookupRune returns properties for r.
+func LookupRune(r rune) (p Properties, size int) {
+	var buf [4]byte
+	n := utf8.EncodeRune(buf[:], r)
+	return Lookup(buf[:n])
+}
+
+// TODO: these lookup methods are based on the generated trie code. The returned
+// sizes have slightly different semantics from the generated code, in that it
+// always returns size==1 for an illegal UTF-8 byte (instead of the length
+// of the maximum invalid subsequence). Most Transformers, like unicode/norm,
+// leave invalid UTF-8 untouched, in which case it has performance benefits to
+// do so (without changing the semantics). Bidi requires the semantics used here
+// for the bidirule implementation to be compatible with the Go semantics.
+//  They ultimately should perhaps be adopted by all trie implementations, for
+// convenience sake.
+// This unrolled code also boosts performance of the secure/bidirule package by
+// about 30%.
+// So, to remove this code:
+//   - add option to trie generator to define return type.
+//   - always return 1 byte size for ill-formed UTF-8 runes.
+
+// Lookup returns properties for the first rune in s and the width in bytes of
+// its encoding. The size will be 0 if s does not hold enough bytes to complete
+// the encoding.
+func Lookup(s []byte) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}
+
+// LookupString returns properties for the first rune in s and the width in
+// bytes of its encoding. The size will be 0 if s does not hold enough bytes to
+// complete the encoding.
+func LookupString(s string) (p Properties, sz int) {
+	c0 := s[0]
+	switch {
+	case c0 < 0x80: // is ASCII
+		return Properties{entry: bidiValues[c0]}, 1
+	case c0 < 0xC2:
+		return Properties{}, 1
+	case c0 < 0xE0: // 2-byte UTF-8
+		if len(s) < 2 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c1)}, 2
+	case c0 < 0xF0: // 3-byte UTF-8
+		if len(s) < 3 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c2), last: c2}, 3
+	case c0 < 0xF8: // 4-byte UTF-8
+		if len(s) < 4 {
+			return Properties{}, 0
+		}
+		i := bidiIndex[c0]
+		c1 := s[1]
+		if c1 < 0x80 || 0xC0 <= c1 {
+			return Properties{}, 1
+		}
+		o := uint32(i)<<6 + uint32(c1)
+		i = bidiIndex[o]
+		c2 := s[2]
+		if c2 < 0x80 || 0xC0 <= c2 {
+			return Properties{}, 1
+		}
+		o = uint32(i)<<6 + uint32(c2)
+		i = bidiIndex[o]
+		c3 := s[3]
+		if c3 < 0x80 || 0xC0 <= c3 {
+			return Properties{}, 1
+		}
+		return Properties{entry: trie.lookupValue(uint32(i), c3)}, 4
+	}
+	// Illegal rune
+	return Properties{}, 1
+}

vendor/golang.org/x/text/unicode/bidi/trieval.go

@@ -0,0 +1,60 @@
+// Code generated by running "go generate" in golang.org/x/text. DO NOT EDIT.
+
+package bidi
+
+// Class is the Unicode BiDi class. Each rune has a single class.
+type Class uint
+
+const (
+	L       Class = iota // LeftToRight
+	R                    // RightToLeft
+	EN                   // EuropeanNumber
+	ES                   // EuropeanSeparator
+	ET                   // EuropeanTerminator
+	AN                   // ArabicNumber
+	CS                   // CommonSeparator
+	B                    // ParagraphSeparator
+	S                    // SegmentSeparator
+	WS                   // WhiteSpace
+	ON                   // OtherNeutral
+	BN                   // BoundaryNeutral
+	NSM                  // NonspacingMark
+	AL                   // ArabicLetter
+	Control              // Control LRO - PDI
+
+	numClass
+
+	LRO // LeftToRightOverride
+	RLO // RightToLeftOverride
+	LRE // LeftToRightEmbedding
+	RLE // RightToLeftEmbedding
+	PDF // PopDirectionalFormat
+	LRI // LeftToRightIsolate
+	RLI // RightToLeftIsolate
+	FSI // FirstStrongIsolate
+	PDI // PopDirectionalIsolate
+
+	unknownClass = ^Class(0)
+)
+
+var controlToClass = map[rune]Class{
+	0x202D: LRO, // LeftToRightOverride,
+	0x202E: RLO, // RightToLeftOverride,
+	0x202A: LRE, // LeftToRightEmbedding,
+	0x202B: RLE, // RightToLeftEmbedding,
+	0x202C: PDF, // PopDirectionalFormat,
+	0x2066: LRI, // LeftToRightIsolate,
+	0x2067: RLI, // RightToLeftIsolate,
+	0x2068: FSI, // FirstStrongIsolate,
+	0x2069: PDI, // PopDirectionalIsolate,
+}
+
+// A trie entry has the following bits:
+// 7..5  XOR mask for brackets
+// 4     1: Bracket open, 0: Bracket close
+// 3..0  Class type
+
+const (
+	openMask     = 0x10
+	xorMaskShift = 5
+)

vendor/golang.org/x/text/unicode/cldr/base.go

@@ -0,0 +1,105 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"encoding/xml"
+	"regexp"
+	"strconv"
+)
+
+// Elem is implemented by every XML element.
+type Elem interface {
+	setEnclosing(Elem)
+	setName(string)
+	enclosing() Elem
+
+	GetCommon() *Common
+}
+
+type hidden struct {
+	CharData string `xml:",chardata"`
+	Alias    *struct {
+		Common
+		Source string `xml:"source,attr"`
+		Path   string `xml:"path,attr"`
+	} `xml:"alias"`
+	Def *struct {
+		Common
+		Choice string `xml:"choice,attr,omitempty"`
+		Type   string `xml:"type,attr,omitempty"`
+	} `xml:"default"`
+}
+
+// Common holds several of the most common attributes and sub elements
+// of an XML element.
+type Common struct {
+	XMLName         xml.Name
+	name            string
+	enclElem        Elem
+	Type            string `xml:"type,attr,omitempty"`
+	Reference       string `xml:"reference,attr,omitempty"`
+	Alt             string `xml:"alt,attr,omitempty"`
+	ValidSubLocales string `xml:"validSubLocales,attr,omitempty"`
+	Draft           string `xml:"draft,attr,omitempty"`
+	hidden
+}
+
+// Default returns the default type to select from the enclosed list
+// or "" if no default value is specified.
+func (e *Common) Default() string {
+	if e.Def == nil {
+		return ""
+	}
+	if e.Def.Choice != "" {
+		return e.Def.Choice
+	} else if e.Def.Type != "" {
+		// Type is still used by the default element in collation.
+		return e.Def.Type
+	}
+	return ""
+}
+
+// Element returns the XML element name.
+func (e *Common) Element() string {
+	return e.name
+}
+
+// GetCommon returns e. It is provided such that Common implements Elem.
+func (e *Common) GetCommon() *Common {
+	return e
+}
+
+// Data returns the character data accumulated for this element.
+func (e *Common) Data() string {
+	e.CharData = charRe.ReplaceAllStringFunc(e.CharData, replaceUnicode)
+	return e.CharData
+}
+
+func (e *Common) setName(s string) {
+	e.name = s
+}
+
+func (e *Common) enclosing() Elem {
+	return e.enclElem
+}
+
+func (e *Common) setEnclosing(en Elem) {
+	e.enclElem = en
+}
+
+// Escape characters that can be escaped without further escaping the string.
+var charRe = regexp.MustCompile(`&#x[0-9a-fA-F]*;|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|\\x[0-9a-fA-F]{2}|\\[0-7]{3}|\\[abtnvfr]`)
+
+// replaceUnicode converts hexadecimal Unicode codepoint notations to a one-rune string.
+// It assumes the input string is correctly formatted.
+func replaceUnicode(s string) string {
+	if s[1] == '#' {
+		r, _ := strconv.ParseInt(s[3:len(s)-1], 16, 32)
+		return string(r)
+	}
+	r, _, _, _ := strconv.UnquoteChar(s, 0)
+	return string(r)
+}

vendor/golang.org/x/text/unicode/cldr/cldr.go

@@ -0,0 +1,137 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:generate go run makexml.go -output xml.go
+
+// Package cldr provides a parser for LDML and related XML formats.
+//
+// This package is intended to be used by the table generation tools for the
+// various packages in x/text and is not internal for historical reasons.
+//
+// As the XML types are generated from the CLDR DTD, and as the CLDR standard is
+// periodically amended, this package may change considerably over time. This
+// mostly means that data may appear and disappear between versions. That is,
+// old code should keep compiling for newer versions, but data may have moved or
+// changed. CLDR version 22 is the first version supported by this package.
+// Older versions may not work.
+package cldr // import "golang.org/x/text/unicode/cldr"
+
+import (
+	"fmt"
+	"sort"
+)
+
+// CLDR provides access to parsed data of the Unicode Common Locale Data Repository.
+type CLDR struct {
+	parent   map[string][]string
+	locale   map[string]*LDML
+	resolved map[string]*LDML
+	bcp47    *LDMLBCP47
+	supp     *SupplementalData
+}
+
+func makeCLDR() *CLDR {
+	return &CLDR{
+		parent:   make(map[string][]string),
+		locale:   make(map[string]*LDML),
+		resolved: make(map[string]*LDML),
+		bcp47:    &LDMLBCP47{},
+		supp:     &SupplementalData{},
+	}
+}
+
+// BCP47 returns the parsed BCP47 LDML data. If no such data was parsed, nil is returned.
+func (cldr *CLDR) BCP47() *LDMLBCP47 {
+	return nil
+}
+
+// Draft indicates the draft level of an element.
+type Draft int
+
+const (
+	Approved Draft = iota
+	Contributed
+	Provisional
+	Unconfirmed
+)
+
+var drafts = []string{"unconfirmed", "provisional", "contributed", "approved", ""}
+
+// ParseDraft returns the Draft value corresponding to the given string. The
+// empty string corresponds to Approved.
+func ParseDraft(level string) (Draft, error) {
+	if level == "" {
+		return Approved, nil
+	}
+	for i, s := range drafts {
+		if level == s {
+			return Unconfirmed - Draft(i), nil
+		}
+	}
+	return Approved, fmt.Errorf("cldr: unknown draft level %q", level)
+}
+
+func (d Draft) String() string {
+	return drafts[len(drafts)-1-int(d)]
+}
+
+// SetDraftLevel sets which draft levels to include in the evaluated LDML.
+// Any draft element for which the draft level is higher than lev will be excluded.
+// If multiple draft levels are available for a single element, the one with the
+// lowest draft level will be selected, unless preferDraft is true, in which case
+// the highest draft will be chosen.
+// It is assumed that the underlying LDML is canonicalized.
+func (cldr *CLDR) SetDraftLevel(lev Draft, preferDraft bool) {
+	// TODO: implement
+	cldr.resolved = make(map[string]*LDML)
+}
+
+// RawLDML returns the LDML XML for id in unresolved form.
+// id must be one of the strings returned by Locales.
+func (cldr *CLDR) RawLDML(loc string) *LDML {
+	return cldr.locale[loc]
+}
+
+// LDML returns the fully resolved LDML XML for loc, which must be one of
+// the strings returned by Locales.
+//
+// Deprecated: Use RawLDML and implement inheritance manually or using the
+// internal cldrtree package.
+// Inheritance has changed quite a bit since the onset of this package and in
+// practice data often represented in a way where knowledge of how it was
+// inherited is relevant.
+func (cldr *CLDR) LDML(loc string) (*LDML, error) {
+	return cldr.resolve(loc)
+}
+
+// Supplemental returns the parsed supplemental data. If no such data was parsed,
+// nil is returned.
+func (cldr *CLDR) Supplemental() *SupplementalData {
+	return cldr.supp
+}
+
+// Locales returns the locales for which there exist files.
+// Valid sublocales for which there is no file are not included.
+// The root locale is always sorted first.
+func (cldr *CLDR) Locales() []string {
+	loc := []string{"root"}
+	hasRoot := false
+	for l, _ := range cldr.locale {
+		if l == "root" {
+			hasRoot = true
+			continue
+		}
+		loc = append(loc, l)
+	}
+	sort.Strings(loc[1:])
+	if !hasRoot {
+		return loc[1:]
+	}
+	return loc
+}
+
+// Get fills in the fields of x based on the XPath path.
+func Get(e Elem, path string) (res Elem, err error) {
+	return walkXPath(e, path)
+}

vendor/golang.org/x/text/unicode/cldr/collate.go

@@ -0,0 +1,359 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"bufio"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+)
+
+// RuleProcessor can be passed to Collator's Process method, which
+// parses the rules and calls the respective method for each rule found.
+type RuleProcessor interface {
+	Reset(anchor string, before int) error
+	Insert(level int, str, context, extend string) error
+	Index(id string)
+}
+
+const (
+	// cldrIndex is a Unicode-reserved sentinel value used to mark the start
+	// of a grouping within an index.
+	// We ignore any rule that starts with this rune.
+	// See https://unicode.org/reports/tr35/#Collation_Elements for details.
+	cldrIndex = "\uFDD0"
+
+	// specialAnchor is the format in which to represent logical reset positions,
+	// such as "first tertiary ignorable".
+	specialAnchor = "<%s/>"
+)
+
+// Process parses the rules for the tailorings of this collation
+// and calls the respective methods of p for each rule found.
+func (c Collation) Process(p RuleProcessor) (err error) {
+	if len(c.Cr) > 0 {
+		if len(c.Cr) > 1 {
+			return fmt.Errorf("multiple cr elements, want 0 or 1")
+		}
+		return processRules(p, c.Cr[0].Data())
+	}
+	if c.Rules.Any != nil {
+		return c.processXML(p)
+	}
+	return errors.New("no tailoring data")
+}
+
+// processRules parses rules in the Collation Rule Syntax defined in
+// https://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Tailorings.
+func processRules(p RuleProcessor, s string) (err error) {
+	chk := func(s string, e error) string {
+		if err == nil {
+			err = e
+		}
+		return s
+	}
+	i := 0 // Save the line number for use after the loop.
+	scanner := bufio.NewScanner(strings.NewReader(s))
+	for ; scanner.Scan() && err == nil; i++ {
+		for s := skipSpace(scanner.Text()); s != "" && s[0] != '#'; s = skipSpace(s) {
+			level := 5
+			var ch byte
+			switch ch, s = s[0], s[1:]; ch {
+			case '&': // followed by <anchor> or '[' <key> ']'
+				if s = skipSpace(s); consume(&s, '[') {
+					s = chk(parseSpecialAnchor(p, s))
+				} else {
+					s = chk(parseAnchor(p, 0, s))
+				}
+			case '<': // sort relation '<'{1,4}, optionally followed by '*'.
+				for level = 1; consume(&s, '<'); level++ {
+				}
+				if level > 4 {
+					err = fmt.Errorf("level %d > 4", level)
+				}
+				fallthrough
+			case '=': // identity relation, optionally followed by *.
+				if consume(&s, '*') {
+					s = chk(parseSequence(p, level, s))
+				} else {
+					s = chk(parseOrder(p, level, s))
+				}
+			default:
+				chk("", fmt.Errorf("illegal operator %q", ch))
+				break
+			}
+		}
+	}
+	if chk("", scanner.Err()); err != nil {
+		return fmt.Errorf("%d: %v", i, err)
+	}
+	return nil
+}
+
+// parseSpecialAnchor parses the anchor syntax which is either of the form
+//    ['before' <level>] <anchor>
+// or
+//    [<label>]
+// The starting should already be consumed.
+func parseSpecialAnchor(p RuleProcessor, s string) (tail string, err error) {
+	i := strings.IndexByte(s, ']')
+	if i == -1 {
+		return "", errors.New("unmatched bracket")
+	}
+	a := strings.TrimSpace(s[:i])
+	s = s[i+1:]
+	if strings.HasPrefix(a, "before ") {
+		l, err := strconv.ParseUint(skipSpace(a[len("before "):]), 10, 3)
+		if err != nil {
+			return s, err
+		}
+		return parseAnchor(p, int(l), s)
+	}
+	return s, p.Reset(fmt.Sprintf(specialAnchor, a), 0)
+}
+
+func parseAnchor(p RuleProcessor, level int, s string) (tail string, err error) {
+	anchor, s, err := scanString(s)
+	if err != nil {
+		return s, err
+	}
+	return s, p.Reset(anchor, level)
+}
+
+func parseOrder(p RuleProcessor, level int, s string) (tail string, err error) {
+	var value, context, extend string
+	if value, s, err = scanString(s); err != nil {
+		return s, err
+	}
+	if strings.HasPrefix(value, cldrIndex) {
+		p.Index(value[len(cldrIndex):])
+		return
+	}
+	if consume(&s, '|') {
+		if context, s, err = scanString(s); err != nil {
+			return s, errors.New("missing string after context")
+		}
+	}
+	if consume(&s, '/') {
+		if extend, s, err = scanString(s); err != nil {
+			return s, errors.New("missing string after extension")
+		}
+	}
+	return s, p.Insert(level, value, context, extend)
+}
+
+// scanString scans a single input string.
+func scanString(s string) (str, tail string, err error) {
+	if s = skipSpace(s); s == "" {
+		return s, s, errors.New("missing string")
+	}
+	buf := [16]byte{} // small but enough to hold most cases.
+	value := buf[:0]
+	for s != "" {
+		if consume(&s, '\'') {
+			i := strings.IndexByte(s, '\'')
+			if i == -1 {
+				return "", "", errors.New(`unmatched single quote`)
+			}
+			if i == 0 {
+				value = append(value, '\'')
+			} else {
+				value = append(value, s[:i]...)
+			}
+			s = s[i+1:]
+			continue
+		}
+		r, sz := utf8.DecodeRuneInString(s)
+		if unicode.IsSpace(r) || strings.ContainsRune("&<=#", r) {
+			break
+		}
+		value = append(value, s[:sz]...)
+		s = s[sz:]
+	}
+	return string(value), skipSpace(s), nil
+}
+
+func parseSequence(p RuleProcessor, level int, s string) (tail string, err error) {
+	if s = skipSpace(s); s == "" {
+		return s, errors.New("empty sequence")
+	}
+	last := rune(0)
+	for s != "" {
+		r, sz := utf8.DecodeRuneInString(s)
+		s = s[sz:]
+
+		if r == '-' {
+			// We have a range. The first element was already written.
+			if last == 0 {
+				return s, errors.New("range without starter value")
+			}
+			r, sz = utf8.DecodeRuneInString(s)
+			s = s[sz:]
+			if r == utf8.RuneError || r < last {
+				return s, fmt.Errorf("invalid range %q-%q", last, r)
+			}
+			for i := last + 1; i <= r; i++ {
+				if err := p.Insert(level, string(i), "", ""); err != nil {
+					return s, err
+				}
+			}
+			last = 0
+			continue
+		}
+
+		if unicode.IsSpace(r) || unicode.IsPunct(r) {
+			break
+		}
+
+		// normal case
+		if err := p.Insert(level, string(r), "", ""); err != nil {
+			return s, err
+		}
+		last = r
+	}
+	return s, nil
+}
+
+func skipSpace(s string) string {
+	return strings.TrimLeftFunc(s, unicode.IsSpace)
+}
+
+// consumes returns whether the next byte is ch. If so, it gobbles it by
+// updating s.
+func consume(s *string, ch byte) (ok bool) {
+	if *s == "" || (*s)[0] != ch {
+		return false
+	}
+	*s = (*s)[1:]
+	return true
+}
+
+// The following code parses Collation rules of CLDR version 24 and before.
+
+var lmap = map[byte]int{
+	'p': 1,
+	's': 2,
+	't': 3,
+	'i': 5,
+}
+
+type rulesElem struct {
+	Rules struct {
+		Common
+		Any []*struct {
+			XMLName xml.Name
+			rule
+		} `xml:",any"`
+	} `xml:"rules"`
+}
+
+type rule struct {
+	Value  string `xml:",chardata"`
+	Before string `xml:"before,attr"`
+	Any    []*struct {
+		XMLName xml.Name
+		rule
+	} `xml:",any"`
+}
+
+var emptyValueError = errors.New("cldr: empty rule value")
+
+func (r *rule) value() (string, error) {
+	// Convert hexadecimal Unicode codepoint notation to a string.
+	s := charRe.ReplaceAllStringFunc(r.Value, replaceUnicode)
+	r.Value = s
+	if s == "" {
+		if len(r.Any) != 1 {
+			return "", emptyValueError
+		}
+		r.Value = fmt.Sprintf(specialAnchor, r.Any[0].XMLName.Local)
+		r.Any = nil
+	} else if len(r.Any) != 0 {
+		return "", fmt.Errorf("cldr: XML elements found in collation rule: %v", r.Any)
+	}
+	return r.Value, nil
+}
+
+func (r rule) process(p RuleProcessor, name, context, extend string) error {
+	v, err := r.value()
+	if err != nil {
+		return err
+	}
+	switch name {
+	case "p", "s", "t", "i":
+		if strings.HasPrefix(v, cldrIndex) {
+			p.Index(v[len(cldrIndex):])
+			return nil
+		}
+		if err := p.Insert(lmap[name[0]], v, context, extend); err != nil {
+			return err
+		}
+	case "pc", "sc", "tc", "ic":
+		level := lmap[name[0]]
+		for _, s := range v {
+			if err := p.Insert(level, string(s), context, extend); err != nil {
+				return err
+			}
+		}
+	default:
+		return fmt.Errorf("cldr: unsupported tag: %q", name)
+	}
+	return nil
+}
+
+// processXML parses the format of CLDR versions 24 and older.
+func (c Collation) processXML(p RuleProcessor) (err error) {
+	// Collation is generated and defined in xml.go.
+	var v string
+	for _, r := range c.Rules.Any {
+		switch r.XMLName.Local {
+		case "reset":
+			level := 0
+			switch r.Before {
+			case "primary", "1":
+				level = 1
+			case "secondary", "2":
+				level = 2
+			case "tertiary", "3":
+				level = 3
+			case "":
+			default:
+				return fmt.Errorf("cldr: unknown level %q", r.Before)
+			}
+			v, err = r.value()
+			if err == nil {
+				err = p.Reset(v, level)
+			}
+		case "x":
+			var context, extend string
+			for _, r1 := range r.Any {
+				v, err = r1.value()
+				switch r1.XMLName.Local {
+				case "context":
+					context = v
+				case "extend":
+					extend = v
+				}
+			}
+			for _, r1 := range r.Any {
+				if t := r1.XMLName.Local; t == "context" || t == "extend" {
+					continue
+				}
+				r1.rule.process(p, r1.XMLName.Local, context, extend)
+			}
+		default:
+			err = r.rule.process(p, r.XMLName.Local, "", "")
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

vendor/golang.org/x/text/unicode/cldr/decode.go

@@ -0,0 +1,172 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+)
+
+// A Decoder loads an archive of CLDR data.
+type Decoder struct {
+	dirFilter     []string
+	sectionFilter []string
+	loader        Loader
+	cldr          *CLDR
+	curLocale     string
+}
+
+// SetSectionFilter takes a list top-level LDML element names to which
+// evaluation of LDML should be limited.  It automatically calls SetDirFilter.
+func (d *Decoder) SetSectionFilter(filter ...string) {
+	d.sectionFilter = filter
+	// TODO: automatically set dir filter
+}
+
+// SetDirFilter limits the loading of LDML XML files of the specied directories.
+// Note that sections may be split across directories differently for different CLDR versions.
+// For more robust code, use SetSectionFilter.
+func (d *Decoder) SetDirFilter(dir ...string) {
+	d.dirFilter = dir
+}
+
+// A Loader provides access to the files of a CLDR archive.
+type Loader interface {
+	Len() int
+	Path(i int) string
+	Reader(i int) (io.ReadCloser, error)
+}
+
+var fileRe = regexp.MustCompile(`.*[/\\](.*)[/\\](.*)\.xml`)
+
+// Decode loads and decodes the files represented by l.
+func (d *Decoder) Decode(l Loader) (cldr *CLDR, err error) {
+	d.cldr = makeCLDR()
+	for i := 0; i < l.Len(); i++ {
+		fname := l.Path(i)
+		if m := fileRe.FindStringSubmatch(fname); m != nil {
+			if len(d.dirFilter) > 0 && !in(d.dirFilter, m[1]) {
+				continue
+			}
+			var r io.ReadCloser
+			if r, err = l.Reader(i); err == nil {
+				err = d.decode(m[1], m[2], r)
+				r.Close()
+			}
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+	d.cldr.finalize(d.sectionFilter)
+	return d.cldr, nil
+}
+
+func (d *Decoder) decode(dir, id string, r io.Reader) error {
+	var v interface{}
+	var l *LDML
+	cldr := d.cldr
+	switch {
+	case dir == "supplemental":
+		v = cldr.supp
+	case dir == "transforms":
+		return nil
+	case dir == "bcp47":
+		v = cldr.bcp47
+	case dir == "validity":
+		return nil
+	default:
+		ok := false
+		if v, ok = cldr.locale[id]; !ok {
+			l = &LDML{}
+			v, cldr.locale[id] = l, l
+		}
+	}
+	x := xml.NewDecoder(r)
+	if err := x.Decode(v); err != nil {
+		log.Printf("%s/%s: %v", dir, id, err)
+		return err
+	}
+	if l != nil {
+		if l.Identity == nil {
+			return fmt.Errorf("%s/%s: missing identity element", dir, id)
+		}
+		// TODO: verify when CLDR bug https://unicode.org/cldr/trac/ticket/8970
+		// is resolved.
+		// path := strings.Split(id, "_")
+		// if lang := l.Identity.Language.Type; lang != path[0] {
+		// 	return fmt.Errorf("%s/%s: language was %s; want %s", dir, id, lang, path[0])
+		// }
+	}
+	return nil
+}
+
+type pathLoader []string
+
+func makePathLoader(path string) (pl pathLoader, err error) {
+	err = filepath.Walk(path, func(path string, _ os.FileInfo, err error) error {
+		pl = append(pl, path)
+		return err
+	})
+	return pl, err
+}
+
+func (pl pathLoader) Len() int {
+	return len(pl)
+}
+
+func (pl pathLoader) Path(i int) string {
+	return pl[i]
+}
+
+func (pl pathLoader) Reader(i int) (io.ReadCloser, error) {
+	return os.Open(pl[i])
+}
+
+// DecodePath loads CLDR data from the given path.
+func (d *Decoder) DecodePath(path string) (cldr *CLDR, err error) {
+	loader, err := makePathLoader(path)
+	if err != nil {
+		return nil, err
+	}
+	return d.Decode(loader)
+}
+
+type zipLoader struct {
+	r *zip.Reader
+}
+
+func (zl zipLoader) Len() int {
+	return len(zl.r.File)
+}
+
+func (zl zipLoader) Path(i int) string {
+	return zl.r.File[i].Name
+}
+
+func (zl zipLoader) Reader(i int) (io.ReadCloser, error) {
+	return zl.r.File[i].Open()
+}
+
+// DecodeZip loads CLDR data from the zip archive for which r is the source.
+func (d *Decoder) DecodeZip(r io.Reader) (cldr *CLDR, err error) {
+	buffer, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	if err != nil {
+		return nil, err
+	}
+	return d.Decode(zipLoader{archive})
+}

vendor/golang.org/x/text/unicode/cldr/makexml.go

@@ -0,0 +1,400 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// This tool generates types for the various XML formats of CLDR.
+package main
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/xml"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"strings"
+
+	"golang.org/x/text/internal/gen"
+)
+
+var outputFile = flag.String("output", "xml.go", "output file name")
+
+func main() {
+	flag.Parse()
+
+	r := gen.OpenCLDRCoreZip()
+	buffer, err := ioutil.ReadAll(r)
+	if err != nil {
+		log.Fatal("Could not read zip file")
+	}
+	r.Close()
+	z, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	if err != nil {
+		log.Fatalf("Could not read zip archive: %v", err)
+	}
+
+	var buf bytes.Buffer
+
+	version := gen.CLDRVersion()
+
+	for _, dtd := range files {
+		for _, f := range z.File {
+			if strings.HasSuffix(f.Name, dtd.file+".dtd") {
+				r, err := f.Open()
+				failOnError(err)
+
+				b := makeBuilder(&buf, dtd)
+				b.parseDTD(r)
+				b.resolve(b.index[dtd.top[0]])
+				b.write()
+				if b.version != "" && version != b.version {
+					println(f.Name)
+					log.Fatalf("main: inconsistent versions: found %s; want %s", b.version, version)
+				}
+				break
+			}
+		}
+	}
+	fmt.Fprintln(&buf, "// Version is the version of CLDR from which the XML definitions are generated.")
+	fmt.Fprintf(&buf, "const Version = %q\n", version)
+
+	gen.WriteGoFile(*outputFile, "cldr", buf.Bytes())
+}
+
+func failOnError(err error) {
+	if err != nil {
+		log.New(os.Stderr, "", log.Lshortfile).Output(2, err.Error())
+		os.Exit(1)
+	}
+}
+
+// configuration data per DTD type
+type dtd struct {
+	file string   // base file name
+	root string   // Go name of the root XML element
+	top  []string // create a different type for this section
+
+	skipElem    []string // hard-coded or deprecated elements
+	skipAttr    []string // attributes to exclude
+	predefined  []string // hard-coded elements exist of the form <name>Elem
+	forceRepeat []string // elements to make slices despite DTD
+}
+
+var files = []dtd{
+	{
+		file: "ldmlBCP47",
+		root: "LDMLBCP47",
+		top:  []string{"ldmlBCP47"},
+		skipElem: []string{
+			"cldrVersion", // deprecated, not used
+		},
+	},
+	{
+		file: "ldmlSupplemental",
+		root: "SupplementalData",
+		top:  []string{"supplementalData"},
+		skipElem: []string{
+			"cldrVersion", // deprecated, not used
+		},
+		forceRepeat: []string{
+			"plurals", // data defined in plurals.xml and ordinals.xml
+		},
+	},
+	{
+		file: "ldml",
+		root: "LDML",
+		top: []string{
+			"ldml", "collation", "calendar", "timeZoneNames", "localeDisplayNames", "numbers",
+		},
+		skipElem: []string{
+			"cp",       // not used anywhere
+			"special",  // not used anywhere
+			"fallback", // deprecated, not used
+			"alias",    // in Common
+			"default",  // in Common
+		},
+		skipAttr: []string{
+			"hiraganaQuarternary", // typo in DTD, correct version included as well
+		},
+		predefined: []string{"rules"},
+	},
+}
+
+var comments = map[string]string{
+	"ldmlBCP47": `
+// LDMLBCP47 holds information on allowable values for various variables in LDML.
+`,
+	"supplementalData": `
+// SupplementalData holds information relevant for internationalization
+// and proper use of CLDR, but that is not contained in the locale hierarchy.
+`,
+	"ldml": `
+// LDML is the top-level type for locale-specific data.
+`,
+	"collation": `
+// Collation contains rules that specify a certain sort-order,
+// as a tailoring of the root order. 
+// The parsed rules are obtained by passing a RuleProcessor to Collation's
+// Process method.
+`,
+	"calendar": `
+// Calendar specifies the fields used for formatting and parsing dates and times.
+// The month and quarter names are identified numerically, starting at 1.
+// The day (of the week) names are identified with short strings, since there is
+// no universally-accepted numeric designation.
+`,
+	"dates": `
+// Dates contains information regarding the format and parsing of dates and times.
+`,
+	"localeDisplayNames": `
+// LocaleDisplayNames specifies localized display names for scripts, languages,
+// countries, currencies, and variants.
+`,
+	"numbers": `
+// Numbers supplies information for formatting and parsing numbers and currencies.
+`,
+}
+
+type element struct {
+	name      string // XML element name
+	category  string // elements contained by this element
+	signature string // category + attrKey*
+
+	attr []*attribute // attributes supported by this element.
+	sub  []struct {   // parsed and evaluated sub elements of this element.
+		e      *element
+		repeat bool // true if the element needs to be a slice
+	}
+
+	resolved bool // prevent multiple resolutions of this element.
+}
+
+type attribute struct {
+	name string
+	key  string
+	list []string
+
+	tag string // Go tag
+}
+
+var (
+	reHead  = regexp.MustCompile(` *(\w+) +([\w\-]+)`)
+	reAttr  = regexp.MustCompile(` *(\w+) *(?:(\w+)|\(([\w\- \|]+)\)) *(?:#([A-Z]*) *(?:\"([\.\d+])\")?)? *("[\w\-:]*")?`)
+	reElem  = regexp.MustCompile(`^ *(EMPTY|ANY|\(.*\)[\*\+\?]?) *$`)
+	reToken = regexp.MustCompile(`\w\-`)
+)
+
+// builder is used to read in the DTD files from CLDR and generate Go code
+// to be used with the encoding/xml package.
+type builder struct {
+	w       io.Writer
+	index   map[string]*element
+	elem    []*element
+	info    dtd
+	version string
+}
+
+func makeBuilder(w io.Writer, d dtd) builder {
+	return builder{
+		w:     w,
+		index: make(map[string]*element),
+		elem:  []*element{},
+		info:  d,
+	}
+}
+
+// parseDTD parses a DTD file.
+func (b *builder) parseDTD(r io.Reader) {
+	for d := xml.NewDecoder(r); ; {
+		t, err := d.Token()
+		if t == nil {
+			break
+		}
+		failOnError(err)
+		dir, ok := t.(xml.Directive)
+		if !ok {
+			continue
+		}
+		m := reHead.FindSubmatch(dir)
+		dir = dir[len(m[0]):]
+		ename := string(m[2])
+		el, elementFound := b.index[ename]
+		switch string(m[1]) {
+		case "ELEMENT":
+			if elementFound {
+				log.Fatal("parseDTD: duplicate entry for element %q", ename)
+			}
+			m := reElem.FindSubmatch(dir)
+			if m == nil {
+				log.Fatalf("parseDTD: invalid element %q", string(dir))
+			}
+			if len(m[0]) != len(dir) {
+				log.Fatal("parseDTD: invalid element %q", string(dir), len(dir), len(m[0]), string(m[0]))
+			}
+			s := string(m[1])
+			el = &element{
+				name:     ename,
+				category: s,
+			}
+			b.index[ename] = el
+		case "ATTLIST":
+			if !elementFound {
+				log.Fatalf("parseDTD: unknown element %q", ename)
+			}
+			s := string(dir)
+			m := reAttr.FindStringSubmatch(s)
+			if m == nil {
+				log.Fatal(fmt.Errorf("parseDTD: invalid attribute %q", string(dir)))
+			}
+			if m[4] == "FIXED" {
+				b.version = m[5]
+			} else {
+				switch m[1] {
+				case "draft", "references", "alt", "validSubLocales", "standard" /* in Common */ :
+				case "type", "choice":
+				default:
+					el.attr = append(el.attr, &attribute{
+						name: m[1],
+						key:  s,
+						list: reToken.FindAllString(m[3], -1),
+					})
+					el.signature = fmt.Sprintf("%s=%s+%s", el.signature, m[1], m[2])
+				}
+			}
+		}
+	}
+}
+
+var reCat = regexp.MustCompile(`[ ,\|]*(?:(\(|\)|\#?[\w_-]+)([\*\+\?]?))?`)
+
+// resolve takes a parsed element and converts it into structured data
+// that can be used to generate the XML code.
+func (b *builder) resolve(e *element) {
+	if e.resolved {
+		return
+	}
+	b.elem = append(b.elem, e)
+	e.resolved = true
+	s := e.category
+	found := make(map[string]bool)
+	sequenceStart := []int{}
+	for len(s) > 0 {
+		m := reCat.FindStringSubmatch(s)
+		if m == nil {
+			log.Fatalf("%s: invalid category string %q", e.name, s)
+		}
+		repeat := m[2] == "*" || m[2] == "+" || in(b.info.forceRepeat, m[1])
+		switch m[1] {
+		case "":
+		case "(":
+			sequenceStart = append(sequenceStart, len(e.sub))
+		case ")":
+			if len(sequenceStart) == 0 {
+				log.Fatalf("%s: unmatched closing parenthesis", e.name)
+			}
+			for i := sequenceStart[len(sequenceStart)-1]; i < len(e.sub); i++ {
+				e.sub[i].repeat = e.sub[i].repeat || repeat
+			}
+			sequenceStart = sequenceStart[:len(sequenceStart)-1]
+		default:
+			if in(b.info.skipElem, m[1]) {
+			} else if sub, ok := b.index[m[1]]; ok {
+				if !found[sub.name] {
+					e.sub = append(e.sub, struct {
+						e      *element
+						repeat bool
+					}{sub, repeat})
+					found[sub.name] = true
+					b.resolve(sub)
+				}
+			} else if m[1] == "#PCDATA" || m[1] == "ANY" {
+			} else if m[1] != "EMPTY" {
+				log.Fatalf("resolve:%s: element %q not found", e.name, m[1])
+			}
+		}
+		s = s[len(m[0]):]
+	}
+}
+
+// return true if s is contained in set.
+func in(set []string, s string) bool {
+	for _, v := range set {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+var repl = strings.NewReplacer("-", " ", "_", " ")
+
+// title puts the first character or each character following '_' in title case and
+// removes all occurrences of '_'.
+func title(s string) string {
+	return strings.Replace(strings.Title(repl.Replace(s)), " ", "", -1)
+}
+
+// writeElem generates Go code for a single element, recursively.
+func (b *builder) writeElem(tab int, e *element) {
+	p := func(f string, x ...interface{}) {
+		f = strings.Replace(f, "\n", "\n"+strings.Repeat("\t", tab), -1)
+		fmt.Fprintf(b.w, f, x...)
+	}
+	if len(e.sub) == 0 && len(e.attr) == 0 {
+		p("Common")
+		return
+	}
+	p("struct {")
+	tab++
+	p("\nCommon")
+	for _, attr := range e.attr {
+		if !in(b.info.skipAttr, attr.name) {
+			p("\n%s string `xml:\"%s,attr\"`", title(attr.name), attr.name)
+		}
+	}
+	for _, sub := range e.sub {
+		if in(b.info.predefined, sub.e.name) {
+			p("\n%sElem", sub.e.name)
+			continue
+		}
+		if in(b.info.skipElem, sub.e.name) {
+			continue
+		}
+		p("\n%s ", title(sub.e.name))
+		if sub.repeat {
+			p("[]")
+		}
+		p("*")
+		if in(b.info.top, sub.e.name) {
+			p(title(sub.e.name))
+		} else {
+			b.writeElem(tab, sub.e)
+		}
+		p(" `xml:\"%s\"`", sub.e.name)
+	}
+	tab--
+	p("\n}")
+}
+
+// write generates the Go XML code.
+func (b *builder) write() {
+	for i, name := range b.info.top {
+		e := b.index[name]
+		if e != nil {
+			fmt.Fprintf(b.w, comments[name])
+			name := title(e.name)
+			if i == 0 {
+				name = b.info.root
+			}
+			fmt.Fprintf(b.w, "type %s ", name)
+			b.writeElem(0, e)
+			fmt.Fprint(b.w, "\n")
+		}
+	}
+}

vendor/golang.org/x/text/unicode/cldr/resolve.go

@@ -0,0 +1,602 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+// This file implements the various inheritance constructs defined by LDML.
+// See https://www.unicode.org/reports/tr35/#Inheritance_and_Validity
+// for more details.
+
+import (
+	"fmt"
+	"log"
+	"reflect"
+	"regexp"
+	"sort"
+	"strings"
+)
+
+// fieldIter iterates over fields in a struct. It includes
+// fields of embedded structs.
+type fieldIter struct {
+	v        reflect.Value
+	index, n []int
+}
+
+func iter(v reflect.Value) fieldIter {
+	if v.Kind() != reflect.Struct {
+		log.Panicf("value %v must be a struct", v)
+	}
+	i := fieldIter{
+		v:     v,
+		index: []int{0},
+		n:     []int{v.NumField()},
+	}
+	i.descent()
+	return i
+}
+
+func (i *fieldIter) descent() {
+	for f := i.field(); f.Anonymous && f.Type.NumField() > 0; f = i.field() {
+		i.index = append(i.index, 0)
+		i.n = append(i.n, f.Type.NumField())
+	}
+}
+
+func (i *fieldIter) done() bool {
+	return len(i.index) == 1 && i.index[0] >= i.n[0]
+}
+
+func skip(f reflect.StructField) bool {
+	return !f.Anonymous && (f.Name[0] < 'A' || f.Name[0] > 'Z')
+}
+
+func (i *fieldIter) next() {
+	for {
+		k := len(i.index) - 1
+		i.index[k]++
+		if i.index[k] < i.n[k] {
+			if !skip(i.field()) {
+				break
+			}
+		} else {
+			if k == 0 {
+				return
+			}
+			i.index = i.index[:k]
+			i.n = i.n[:k]
+		}
+	}
+	i.descent()
+}
+
+func (i *fieldIter) value() reflect.Value {
+	return i.v.FieldByIndex(i.index)
+}
+
+func (i *fieldIter) field() reflect.StructField {
+	return i.v.Type().FieldByIndex(i.index)
+}
+
+type visitor func(v reflect.Value) error
+
+var stopDescent = fmt.Errorf("do not recurse")
+
+func (f visitor) visit(x interface{}) error {
+	return f.visitRec(reflect.ValueOf(x))
+}
+
+// visit recursively calls f on all nodes in v.
+func (f visitor) visitRec(v reflect.Value) error {
+	if v.Kind() == reflect.Ptr {
+		if v.IsNil() {
+			return nil
+		}
+		return f.visitRec(v.Elem())
+	}
+	if err := f(v); err != nil {
+		if err == stopDescent {
+			return nil
+		}
+		return err
+	}
+	switch v.Kind() {
+	case reflect.Struct:
+		for i := iter(v); !i.done(); i.next() {
+			if err := f.visitRec(i.value()); err != nil {
+				return err
+			}
+		}
+	case reflect.Slice:
+		for i := 0; i < v.Len(); i++ {
+			if err := f.visitRec(v.Index(i)); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// getPath is used for error reporting purposes only.
+func getPath(e Elem) string {
+	if e == nil {
+		return "<nil>"
+	}
+	if e.enclosing() == nil {
+		return e.GetCommon().name
+	}
+	if e.GetCommon().Type == "" {
+		return fmt.Sprintf("%s.%s", getPath(e.enclosing()), e.GetCommon().name)
+	}
+	return fmt.Sprintf("%s.%s[type=%s]", getPath(e.enclosing()), e.GetCommon().name, e.GetCommon().Type)
+}
+
+// xmlName returns the xml name of the element or attribute
+func xmlName(f reflect.StructField) (name string, attr bool) {
+	tags := strings.Split(f.Tag.Get("xml"), ",")
+	for _, s := range tags {
+		attr = attr || s == "attr"
+	}
+	return tags[0], attr
+}
+
+func findField(v reflect.Value, key string) (reflect.Value, error) {
+	v = reflect.Indirect(v)
+	for i := iter(v); !i.done(); i.next() {
+		if n, _ := xmlName(i.field()); n == key {
+			return i.value(), nil
+		}
+	}
+	return reflect.Value{}, fmt.Errorf("cldr: no field %q in element %#v", key, v.Interface())
+}
+
+var xpathPart = regexp.MustCompile(`(\pL+)(?:\[@(\pL+)='([\w-]+)'\])?`)
+
+func walkXPath(e Elem, path string) (res Elem, err error) {
+	for _, c := range strings.Split(path, "/") {
+		if c == ".." {
+			if e = e.enclosing(); e == nil {
+				panic("path ..")
+				return nil, fmt.Errorf(`cldr: ".." moves past root in path %q`, path)
+			}
+			continue
+		} else if c == "" {
+			continue
+		}
+		m := xpathPart.FindStringSubmatch(c)
+		if len(m) == 0 || len(m[0]) != len(c) {
+			return nil, fmt.Errorf("cldr: syntax error in path component %q", c)
+		}
+		v, err := findField(reflect.ValueOf(e), m[1])
+		if err != nil {
+			return nil, err
+		}
+		switch v.Kind() {
+		case reflect.Slice:
+			i := 0
+			if m[2] != "" || v.Len() > 1 {
+				if m[2] == "" {
+					m[2] = "type"
+					if m[3] = e.GetCommon().Default(); m[3] == "" {
+						return nil, fmt.Errorf("cldr: type selector or default value needed for element %s", m[1])
+					}
+				}
+				for ; i < v.Len(); i++ {
+					vi := v.Index(i)
+					key, err := findField(vi.Elem(), m[2])
+					if err != nil {
+						return nil, err
+					}
+					key = reflect.Indirect(key)
+					if key.Kind() == reflect.String && key.String() == m[3] {
+						break
+					}
+				}
+			}
+			if i == v.Len() || v.Index(i).IsNil() {
+				return nil, fmt.Errorf("no %s found with %s==%s", m[1], m[2], m[3])
+			}
+			e = v.Index(i).Interface().(Elem)
+		case reflect.Ptr:
+			if v.IsNil() {
+				return nil, fmt.Errorf("cldr: element %q not found within element %q", m[1], e.GetCommon().name)
+			}
+			var ok bool
+			if e, ok = v.Interface().(Elem); !ok {
+				return nil, fmt.Errorf("cldr: %q is not an XML element", m[1])
+			} else if m[2] != "" || m[3] != "" {
+				return nil, fmt.Errorf("cldr: no type selector allowed for element %s", m[1])
+			}
+		default:
+			return nil, fmt.Errorf("cldr: %q is not an XML element", m[1])
+		}
+	}
+	return e, nil
+}
+
+const absPrefix = "//ldml/"
+
+func (cldr *CLDR) resolveAlias(e Elem, src, path string) (res Elem, err error) {
+	if src != "locale" {
+		if !strings.HasPrefix(path, absPrefix) {
+			return nil, fmt.Errorf("cldr: expected absolute path, found %q", path)
+		}
+		path = path[len(absPrefix):]
+		if e, err = cldr.resolve(src); err != nil {
+			return nil, err
+		}
+	}
+	return walkXPath(e, path)
+}
+
+func (cldr *CLDR) resolveAndMergeAlias(e Elem) error {
+	alias := e.GetCommon().Alias
+	if alias == nil {
+		return nil
+	}
+	a, err := cldr.resolveAlias(e, alias.Source, alias.Path)
+	if err != nil {
+		return fmt.Errorf("%v: error evaluating path %q: %v", getPath(e), alias.Path, err)
+	}
+	// Ensure alias node was already evaluated. TODO: avoid double evaluation.
+	err = cldr.resolveAndMergeAlias(a)
+	v := reflect.ValueOf(e).Elem()
+	for i := iter(reflect.ValueOf(a).Elem()); !i.done(); i.next() {
+		if vv := i.value(); vv.Kind() != reflect.Ptr || !vv.IsNil() {
+			if _, attr := xmlName(i.field()); !attr {
+				v.FieldByIndex(i.index).Set(vv)
+			}
+		}
+	}
+	return err
+}
+
+func (cldr *CLDR) aliasResolver() visitor {
+	return func(v reflect.Value) (err error) {
+		if e, ok := v.Addr().Interface().(Elem); ok {
+			err = cldr.resolveAndMergeAlias(e)
+			if err == nil && blocking[e.GetCommon().name] {
+				return stopDescent
+			}
+		}
+		return err
+	}
+}
+
+// elements within blocking elements do not inherit.
+// Taken from CLDR's supplementalMetaData.xml.
+var blocking = map[string]bool{
+	"identity":         true,
+	"supplementalData": true,
+	"cldrTest":         true,
+	"collation":        true,
+	"transform":        true,
+}
+
+// Distinguishing attributes affect inheritance; two elements with different
+// distinguishing attributes are treated as different for purposes of inheritance,
+// except when such attributes occur in the indicated elements.
+// Taken from CLDR's supplementalMetaData.xml.
+var distinguishing = map[string][]string{
+	"key":        nil,
+	"request_id": nil,
+	"id":         nil,
+	"registry":   nil,
+	"alt":        nil,
+	"iso4217":    nil,
+	"iso3166":    nil,
+	"mzone":      nil,
+	"from":       nil,
+	"to":         nil,
+	"type": []string{
+		"abbreviationFallback",
+		"default",
+		"mapping",
+		"measurementSystem",
+		"preferenceOrdering",
+	},
+	"numberSystem": nil,
+}
+
+func in(set []string, s string) bool {
+	for _, v := range set {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+// attrKey computes a key based on the distinguishable attributes of
+// an element and its values.
+func attrKey(v reflect.Value, exclude ...string) string {
+	parts := []string{}
+	ename := v.Interface().(Elem).GetCommon().name
+	v = v.Elem()
+	for i := iter(v); !i.done(); i.next() {
+		if name, attr := xmlName(i.field()); attr {
+			if except, ok := distinguishing[name]; ok && !in(exclude, name) && !in(except, ename) {
+				v := i.value()
+				if v.Kind() == reflect.Ptr {
+					v = v.Elem()
+				}
+				if v.IsValid() {
+					parts = append(parts, fmt.Sprintf("%s=%s", name, v.String()))
+				}
+			}
+		}
+	}
+	sort.Strings(parts)
+	return strings.Join(parts, ";")
+}
+
+// Key returns a key for e derived from all distinguishing attributes
+// except those specified by exclude.
+func Key(e Elem, exclude ...string) string {
+	return attrKey(reflect.ValueOf(e), exclude...)
+}
+
+// linkEnclosing sets the enclosing element as well as the name
+// for all sub-elements of child, recursively.
+func linkEnclosing(parent, child Elem) {
+	child.setEnclosing(parent)
+	v := reflect.ValueOf(child).Elem()
+	for i := iter(v); !i.done(); i.next() {
+		vf := i.value()
+		if vf.Kind() == reflect.Slice {
+			for j := 0; j < vf.Len(); j++ {
+				linkEnclosing(child, vf.Index(j).Interface().(Elem))
+			}
+		} else if vf.Kind() == reflect.Ptr && !vf.IsNil() && vf.Elem().Kind() == reflect.Struct {
+			linkEnclosing(child, vf.Interface().(Elem))
+		}
+	}
+}
+
+func setNames(e Elem, name string) {
+	e.setName(name)
+	v := reflect.ValueOf(e).Elem()
+	for i := iter(v); !i.done(); i.next() {
+		vf := i.value()
+		name, _ = xmlName(i.field())
+		if vf.Kind() == reflect.Slice {
+			for j := 0; j < vf.Len(); j++ {
+				setNames(vf.Index(j).Interface().(Elem), name)
+			}
+		} else if vf.Kind() == reflect.Ptr && !vf.IsNil() && vf.Elem().Kind() == reflect.Struct {
+			setNames(vf.Interface().(Elem), name)
+		}
+	}
+}
+
+// deepCopy copies elements of v recursively.  All elements of v that may
+// be modified by inheritance are explicitly copied.
+func deepCopy(v reflect.Value) reflect.Value {
+	switch v.Kind() {
+	case reflect.Ptr:
+		if v.IsNil() || v.Elem().Kind() != reflect.Struct {
+			return v
+		}
+		nv := reflect.New(v.Elem().Type())
+		nv.Elem().Set(v.Elem())
+		deepCopyRec(nv.Elem(), v.Elem())
+		return nv
+	case reflect.Slice:
+		nv := reflect.MakeSlice(v.Type(), v.Len(), v.Len())
+		for i := 0; i < v.Len(); i++ {
+			deepCopyRec(nv.Index(i), v.Index(i))
+		}
+		return nv
+	}
+	panic("deepCopy: must be called with pointer or slice")
+}
+
+// deepCopyRec is only called by deepCopy.
+func deepCopyRec(nv, v reflect.Value) {
+	if v.Kind() == reflect.Struct {
+		t := v.Type()
+		for i := 0; i < v.NumField(); i++ {
+			if name, attr := xmlName(t.Field(i)); name != "" && !attr {
+				deepCopyRec(nv.Field(i), v.Field(i))
+			}
+		}
+	} else {
+		nv.Set(deepCopy(v))
+	}
+}
+
+// newNode is used to insert a missing node during inheritance.
+func (cldr *CLDR) newNode(v, enc reflect.Value) reflect.Value {
+	n := reflect.New(v.Type())
+	for i := iter(v); !i.done(); i.next() {
+		if name, attr := xmlName(i.field()); name == "" || attr {
+			n.Elem().FieldByIndex(i.index).Set(i.value())
+		}
+	}
+	n.Interface().(Elem).GetCommon().setEnclosing(enc.Addr().Interface().(Elem))
+	return n
+}
+
+// v, parent must be pointers to struct
+func (cldr *CLDR) inheritFields(v, parent reflect.Value) (res reflect.Value, err error) {
+	t := v.Type()
+	nv := reflect.New(t)
+	nv.Elem().Set(v)
+	for i := iter(v); !i.done(); i.next() {
+		vf := i.value()
+		f := i.field()
+		name, attr := xmlName(f)
+		if name == "" || attr {
+			continue
+		}
+		pf := parent.FieldByIndex(i.index)
+		if blocking[name] {
+			if vf.IsNil() {
+				vf = pf
+			}
+			nv.Elem().FieldByIndex(i.index).Set(deepCopy(vf))
+			continue
+		}
+		switch f.Type.Kind() {
+		case reflect.Ptr:
+			if f.Type.Elem().Kind() == reflect.Struct {
+				if !vf.IsNil() {
+					if vf, err = cldr.inheritStructPtr(vf, pf); err != nil {
+						return reflect.Value{}, err
+					}
+					vf.Interface().(Elem).setEnclosing(nv.Interface().(Elem))
+					nv.Elem().FieldByIndex(i.index).Set(vf)
+				} else if !pf.IsNil() {
+					n := cldr.newNode(pf.Elem(), v)
+					if vf, err = cldr.inheritStructPtr(n, pf); err != nil {
+						return reflect.Value{}, err
+					}
+					vf.Interface().(Elem).setEnclosing(nv.Interface().(Elem))
+					nv.Elem().FieldByIndex(i.index).Set(vf)
+				}
+			}
+		case reflect.Slice:
+			vf, err := cldr.inheritSlice(nv.Elem(), vf, pf)
+			if err != nil {
+				return reflect.Zero(t), err
+			}
+			nv.Elem().FieldByIndex(i.index).Set(vf)
+		}
+	}
+	return nv, nil
+}
+
+func root(e Elem) *LDML {
+	for ; e.enclosing() != nil; e = e.enclosing() {
+	}
+	return e.(*LDML)
+}
+
+// inheritStructPtr first merges possible aliases in with v and then inherits
+// any underspecified elements from parent.
+func (cldr *CLDR) inheritStructPtr(v, parent reflect.Value) (r reflect.Value, err error) {
+	if !v.IsNil() {
+		e := v.Interface().(Elem).GetCommon()
+		alias := e.Alias
+		if alias == nil && !parent.IsNil() {
+			alias = parent.Interface().(Elem).GetCommon().Alias
+		}
+		if alias != nil {
+			a, err := cldr.resolveAlias(v.Interface().(Elem), alias.Source, alias.Path)
+			if a != nil {
+				if v, err = cldr.inheritFields(v.Elem(), reflect.ValueOf(a).Elem()); err != nil {
+					return reflect.Value{}, err
+				}
+			}
+		}
+		if !parent.IsNil() {
+			return cldr.inheritFields(v.Elem(), parent.Elem())
+		}
+	} else if parent.IsNil() {
+		panic("should not reach here")
+	}
+	return v, nil
+}
+
+// Must be slice of struct pointers.
+func (cldr *CLDR) inheritSlice(enc, v, parent reflect.Value) (res reflect.Value, err error) {
+	t := v.Type()
+	index := make(map[string]reflect.Value)
+	if !v.IsNil() {
+		for i := 0; i < v.Len(); i++ {
+			vi := v.Index(i)
+			key := attrKey(vi)
+			index[key] = vi
+		}
+	}
+	if !parent.IsNil() {
+		for i := 0; i < parent.Len(); i++ {
+			vi := parent.Index(i)
+			key := attrKey(vi)
+			if w, ok := index[key]; ok {
+				index[key], err = cldr.inheritStructPtr(w, vi)
+			} else {
+				n := cldr.newNode(vi.Elem(), enc)
+				index[key], err = cldr.inheritStructPtr(n, vi)
+			}
+			index[key].Interface().(Elem).setEnclosing(enc.Addr().Interface().(Elem))
+			if err != nil {
+				return v, err
+			}
+		}
+	}
+	keys := make([]string, 0, len(index))
+	for k, _ := range index {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	sl := reflect.MakeSlice(t, len(index), len(index))
+	for i, k := range keys {
+		sl.Index(i).Set(index[k])
+	}
+	return sl, nil
+}
+
+func parentLocale(loc string) string {
+	parts := strings.Split(loc, "_")
+	if len(parts) == 1 {
+		return "root"
+	}
+	parts = parts[:len(parts)-1]
+	key := strings.Join(parts, "_")
+	return key
+}
+
+func (cldr *CLDR) resolve(loc string) (res *LDML, err error) {
+	if r := cldr.resolved[loc]; r != nil {
+		return r, nil
+	}
+	x := cldr.RawLDML(loc)
+	if x == nil {
+		return nil, fmt.Errorf("cldr: unknown locale %q", loc)
+	}
+	var v reflect.Value
+	if loc == "root" {
+		x = deepCopy(reflect.ValueOf(x)).Interface().(*LDML)
+		linkEnclosing(nil, x)
+		err = cldr.aliasResolver().visit(x)
+	} else {
+		key := parentLocale(loc)
+		var parent *LDML
+		for ; cldr.locale[key] == nil; key = parentLocale(key) {
+		}
+		if parent, err = cldr.resolve(key); err != nil {
+			return nil, err
+		}
+		v, err = cldr.inheritFields(reflect.ValueOf(x).Elem(), reflect.ValueOf(parent).Elem())
+		x = v.Interface().(*LDML)
+		linkEnclosing(nil, x)
+	}
+	if err != nil {
+		return nil, err
+	}
+	cldr.resolved[loc] = x
+	return x, err
+}
+
+// finalize finalizes the initialization of the raw LDML structs.  It also
+// removed unwanted fields, as specified by filter, so that they will not
+// be unnecessarily evaluated.
+func (cldr *CLDR) finalize(filter []string) {
+	for _, x := range cldr.locale {
+		if filter != nil {
+			v := reflect.ValueOf(x).Elem()
+			t := v.Type()
+			for i := 0; i < v.NumField(); i++ {
+				f := t.Field(i)
+				name, _ := xmlName(f)
+				if name != "" && name != "identity" && !in(filter, name) {
+					v.Field(i).Set(reflect.Zero(f.Type))
+				}
+			}
+		}
+		linkEnclosing(nil, x) // for resolving aliases and paths
+		setNames(x, "ldml")
+	}
+}

vendor/golang.org/x/text/unicode/cldr/slice.go

@@ -0,0 +1,144 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cldr
+
+import (
+	"fmt"
+	"reflect"
+	"sort"
+)
+
+// Slice provides utilities for modifying slices of elements.
+// It can be wrapped around any slice of which the element type implements
+// interface Elem.
+type Slice struct {
+	ptr reflect.Value
+	typ reflect.Type
+}
+
+// Value returns the reflect.Value of the underlying slice.
+func (s *Slice) Value() reflect.Value {
+	return s.ptr.Elem()
+}
+
+// MakeSlice wraps a pointer to a slice of Elems.
+// It replaces the array pointed to by the slice so that subsequent modifications
+// do not alter the data in a CLDR type.
+// It panics if an incorrect type is passed.
+func MakeSlice(slicePtr interface{}) Slice {
+	ptr := reflect.ValueOf(slicePtr)
+	if ptr.Kind() != reflect.Ptr {
+		panic(fmt.Sprintf("MakeSlice: argument must be pointer to slice, found %v", ptr.Type()))
+	}
+	sl := ptr.Elem()
+	if sl.Kind() != reflect.Slice {
+		panic(fmt.Sprintf("MakeSlice: argument must point to a slice, found %v", sl.Type()))
+	}
+	intf := reflect.TypeOf((*Elem)(nil)).Elem()
+	if !sl.Type().Elem().Implements(intf) {
+		panic(fmt.Sprintf("MakeSlice: element type of slice (%v) does not implement Elem", sl.Type().Elem()))
+	}
+	nsl := reflect.MakeSlice(sl.Type(), sl.Len(), sl.Len())
+	reflect.Copy(nsl, sl)
+	sl.Set(nsl)
+	return Slice{
+		ptr: ptr,
+		typ: sl.Type().Elem().Elem(),
+	}
+}
+
+func (s Slice) indexForAttr(a string) []int {
+	for i := iter(reflect.Zero(s.typ)); !i.done(); i.next() {
+		if n, _ := xmlName(i.field()); n == a {
+			return i.index
+		}
+	}
+	panic(fmt.Sprintf("MakeSlice: no attribute %q for type %v", a, s.typ))
+}
+
+// Filter filters s to only include elements for which fn returns true.
+func (s Slice) Filter(fn func(e Elem) bool) {
+	k := 0
+	sl := s.Value()
+	for i := 0; i < sl.Len(); i++ {
+		vi := sl.Index(i)
+		if fn(vi.Interface().(Elem)) {
+			sl.Index(k).Set(vi)
+			k++
+		}
+	}
+	sl.Set(sl.Slice(0, k))
+}
+
+// Group finds elements in s for which fn returns the same value and groups
+// them in a new Slice.
+func (s Slice) Group(fn func(e Elem) string) []Slice {
+	m := make(map[string][]reflect.Value)
+	sl := s.Value()
+	for i := 0; i < sl.Len(); i++ {
+		vi := sl.Index(i)
+		key := fn(vi.Interface().(Elem))
+		m[key] = append(m[key], vi)
+	}
+	keys := []string{}
+	for k, _ := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	res := []Slice{}
+	for _, k := range keys {
+		nsl := reflect.New(sl.Type())
+		nsl.Elem().Set(reflect.Append(nsl.Elem(), m[k]...))
+		res = append(res, MakeSlice(nsl.Interface()))
+	}
+	return res
+}
+
+// SelectAnyOf filters s to contain only elements for which attr matches
+// any of the values.
+func (s Slice) SelectAnyOf(attr string, values ...string) {
+	index := s.indexForAttr(attr)
+	s.Filter(func(e Elem) bool {
+		vf := reflect.ValueOf(e).Elem().FieldByIndex(index)
+		return in(values, vf.String())
+	})
+}
+
+// SelectOnePerGroup filters s to include at most one element e per group of
+// elements matching Key(attr), where e has an attribute a that matches any
+// the values in v.
+// If more than one element in a group matches a value in v preference
+// is given to the element that matches the first value in v.
+func (s Slice) SelectOnePerGroup(a string, v []string) {
+	index := s.indexForAttr(a)
+	grouped := s.Group(func(e Elem) string { return Key(e, a) })
+	sl := s.Value()
+	sl.Set(sl.Slice(0, 0))
+	for _, g := range grouped {
+		e := reflect.Value{}
+		found := len(v)
+		gsl := g.Value()
+		for i := 0; i < gsl.Len(); i++ {
+			vi := gsl.Index(i).Elem().FieldByIndex(index)
+			j := 0
+			for ; j < len(v) && v[j] != vi.String(); j++ {
+			}
+			if j < found {
+				found = j
+				e = gsl.Index(i)
+			}
+		}
+		if found < len(v) {
+			sl.Set(reflect.Append(sl, e))
+		}
+	}
+}
+
+// SelectDraft drops all elements from the list with a draft level smaller than d
+// and selects the highest draft level of the remaining.
+// This method assumes that the input CLDR is canonicalized.
+func (s Slice) SelectDraft(d Draft) {
+	s.SelectOnePerGroup("draft", drafts[len(drafts)-2-int(d):])
+}

vendor/golang.org/x/text/unicode/norm/composition.go

@@ -0,0 +1,512 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+const (
+	maxNonStarters = 30
+	// The maximum number of characters needed for a buffer is
+	// maxNonStarters + 1 for the starter + 1 for the GCJ
+	maxBufferSize    = maxNonStarters + 2
+	maxNFCExpansion  = 3  // NFC(0x1D160)
+	maxNFKCExpansion = 18 // NFKC(0xFDFA)
+
+	maxByteBufferSize = utf8.UTFMax * maxBufferSize // 128
+)
+
+// ssState is used for reporting the segment state after inserting a rune.
+// It is returned by streamSafe.next.
+type ssState int
+
+const (
+	// Indicates a rune was successfully added to the segment.
+	ssSuccess ssState = iota
+	// Indicates a rune starts a new segment and should not be added.
+	ssStarter
+	// Indicates a rune caused a segment overflow and a CGJ should be inserted.
+	ssOverflow
+)
+
+// streamSafe implements the policy of when a CGJ should be inserted.
+type streamSafe uint8
+
+// first inserts the first rune of a segment. It is a faster version of next if
+// it is known p represents the first rune in a segment.
+func (ss *streamSafe) first(p Properties) {
+	*ss = streamSafe(p.nTrailingNonStarters())
+}
+
+// insert returns a ssState value to indicate whether a rune represented by p
+// can be inserted.
+func (ss *streamSafe) next(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	n := p.nLeadingNonStarters()
+	if *ss += streamSafe(n); *ss > maxNonStarters {
+		*ss = 0
+		return ssOverflow
+	}
+	// The Stream-Safe Text Processing prescribes that the counting can stop
+	// as soon as a starter is encountered. However, there are some starters,
+	// like Jamo V and T, that can combine with other runes, leaving their
+	// successive non-starters appended to the previous, possibly causing an
+	// overflow. We will therefore consider any rune with a non-zero nLead to
+	// be a non-starter. Note that it always hold that if nLead > 0 then
+	// nLead == nTrail.
+	if n == 0 {
+		*ss = streamSafe(p.nTrailingNonStarters())
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+// backwards is used for checking for overflow and segment starts
+// when traversing a string backwards. Users do not need to call first
+// for the first rune. The state of the streamSafe retains the count of
+// the non-starters loaded.
+func (ss *streamSafe) backwards(p Properties) ssState {
+	if *ss > maxNonStarters {
+		panic("streamSafe was not reset")
+	}
+	c := *ss + streamSafe(p.nTrailingNonStarters())
+	if c > maxNonStarters {
+		return ssOverflow
+	}
+	*ss = c
+	if p.nLeadingNonStarters() == 0 {
+		return ssStarter
+	}
+	return ssSuccess
+}
+
+func (ss streamSafe) isMax() bool {
+	return ss == maxNonStarters
+}
+
+// GraphemeJoiner is inserted after maxNonStarters non-starter runes.
+const GraphemeJoiner = "\u034F"
+
+// reorderBuffer is used to normalize a single segment.  Characters inserted with
+// insert are decomposed and reordered based on CCC. The compose method can
+// be used to recombine characters.  Note that the byte buffer does not hold
+// the UTF-8 characters in order.  Only the rune array is maintained in sorted
+// order. flush writes the resulting segment to a byte array.
+type reorderBuffer struct {
+	rune  [maxBufferSize]Properties // Per character info.
+	byte  [maxByteBufferSize]byte   // UTF-8 buffer. Referenced by runeInfo.pos.
+	nbyte uint8                     // Number or bytes.
+	ss    streamSafe                // For limiting length of non-starter sequence.
+	nrune int                       // Number of runeInfos.
+	f     formInfo
+
+	src      input
+	nsrc     int
+	tmpBytes input
+
+	out    []byte
+	flushF func(*reorderBuffer) bool
+}
+
+func (rb *reorderBuffer) init(f Form, src []byte) {
+	rb.f = *formTable[f]
+	rb.src.setBytes(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) initString(f Form, src string) {
+	rb.f = *formTable[f]
+	rb.src.setString(src)
+	rb.nsrc = len(src)
+	rb.ss = 0
+}
+
+func (rb *reorderBuffer) setFlusher(out []byte, f func(*reorderBuffer) bool) {
+	rb.out = out
+	rb.flushF = f
+}
+
+// reset discards all characters from the buffer.
+func (rb *reorderBuffer) reset() {
+	rb.nrune = 0
+	rb.nbyte = 0
+}
+
+func (rb *reorderBuffer) doFlush() bool {
+	if rb.f.composing {
+		rb.compose()
+	}
+	res := rb.flushF(rb)
+	rb.reset()
+	return res
+}
+
+// appendFlush appends the normalized segment to rb.out.
+func appendFlush(rb *reorderBuffer) bool {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		rb.out = append(rb.out, rb.byte[start:end]...)
+	}
+	return true
+}
+
+// flush appends the normalized segment to out and resets rb.
+func (rb *reorderBuffer) flush(out []byte) []byte {
+	for i := 0; i < rb.nrune; i++ {
+		start := rb.rune[i].pos
+		end := start + rb.rune[i].size
+		out = append(out, rb.byte[start:end]...)
+	}
+	rb.reset()
+	return out
+}
+
+// flushCopy copies the normalized segment to buf and resets rb.
+// It returns the number of bytes written to buf.
+func (rb *reorderBuffer) flushCopy(buf []byte) int {
+	p := 0
+	for i := 0; i < rb.nrune; i++ {
+		runep := rb.rune[i]
+		p += copy(buf[p:], rb.byte[runep.pos:runep.pos+runep.size])
+	}
+	rb.reset()
+	return p
+}
+
+// insertOrdered inserts a rune in the buffer, ordered by Canonical Combining Class.
+// It returns false if the buffer is not large enough to hold the rune.
+// It is used internally by insert and insertString only.
+func (rb *reorderBuffer) insertOrdered(info Properties) {
+	n := rb.nrune
+	b := rb.rune[:]
+	cc := info.ccc
+	if cc > 0 {
+		// Find insertion position + move elements to make room.
+		for ; n > 0; n-- {
+			if b[n-1].ccc <= cc {
+				break
+			}
+			b[n] = b[n-1]
+		}
+	}
+	rb.nrune += 1
+	pos := uint8(rb.nbyte)
+	rb.nbyte += utf8.UTFMax
+	info.pos = pos
+	b[n] = info
+}
+
+// insertErr is an error code returned by insert. Using this type instead
+// of error improves performance up to 20% for many of the benchmarks.
+type insertErr int
+
+const (
+	iSuccess insertErr = -iota
+	iShortDst
+	iShortSrc
+)
+
+// insertFlush inserts the given rune in the buffer ordered by CCC.
+// If a decomposition with multiple segments are encountered, they leading
+// ones are flushed.
+// It returns a non-zero error code if the rune was not inserted.
+func (rb *reorderBuffer) insertFlush(src input, i int, info Properties) insertErr {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+		return iSuccess
+	}
+	if info.hasDecomposition() {
+		return rb.insertDecomposed(info.Decomposition())
+	}
+	rb.insertSingle(src, i, info)
+	return iSuccess
+}
+
+// insertUnsafe inserts the given rune in the buffer ordered by CCC.
+// It is assumed there is sufficient space to hold the runes. It is the
+// responsibility of the caller to ensure this. This can be done by checking
+// the state returned by the streamSafe type.
+func (rb *reorderBuffer) insertUnsafe(src input, i int, info Properties) {
+	if rune := src.hangul(i); rune != 0 {
+		rb.decomposeHangul(rune)
+	}
+	if info.hasDecomposition() {
+		// TODO: inline.
+		rb.insertDecomposed(info.Decomposition())
+	} else {
+		rb.insertSingle(src, i, info)
+	}
+}
+
+// insertDecomposed inserts an entry in to the reorderBuffer for each rune
+// in dcomp. dcomp must be a sequence of decomposed UTF-8-encoded runes.
+// It flushes the buffer on each new segment start.
+func (rb *reorderBuffer) insertDecomposed(dcomp []byte) insertErr {
+	rb.tmpBytes.setBytes(dcomp)
+	// As the streamSafe accounting already handles the counting for modifiers,
+	// we don't have to call next. However, we do need to keep the accounting
+	// intact when flushing the buffer.
+	for i := 0; i < len(dcomp); {
+		info := rb.f.info(rb.tmpBytes, i)
+		if info.BoundaryBefore() && rb.nrune > 0 && !rb.doFlush() {
+			return iShortDst
+		}
+		i += copy(rb.byte[rb.nbyte:], dcomp[i:i+int(info.size)])
+		rb.insertOrdered(info)
+	}
+	return iSuccess
+}
+
+// insertSingle inserts an entry in the reorderBuffer for the rune at
+// position i. info is the runeInfo for the rune at position i.
+func (rb *reorderBuffer) insertSingle(src input, i int, info Properties) {
+	src.copySlice(rb.byte[rb.nbyte:], i, i+int(info.size))
+	rb.insertOrdered(info)
+}
+
+// insertCGJ inserts a Combining Grapheme Joiner (0x034f) into rb.
+func (rb *reorderBuffer) insertCGJ() {
+	rb.insertSingle(input{str: GraphemeJoiner}, 0, Properties{size: uint8(len(GraphemeJoiner))})
+}
+
+// appendRune inserts a rune at the end of the buffer. It is used for Hangul.
+func (rb *reorderBuffer) appendRune(r rune) {
+	bn := rb.nbyte
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.nbyte += utf8.UTFMax
+	rb.rune[rb.nrune] = Properties{pos: bn, size: uint8(sz)}
+	rb.nrune++
+}
+
+// assignRune sets a rune at position pos. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) assignRune(pos int, r rune) {
+	bn := rb.rune[pos].pos
+	sz := utf8.EncodeRune(rb.byte[bn:], rune(r))
+	rb.rune[pos] = Properties{pos: bn, size: uint8(sz)}
+}
+
+// runeAt returns the rune at position n. It is used for Hangul and recomposition.
+func (rb *reorderBuffer) runeAt(n int) rune {
+	inf := rb.rune[n]
+	r, _ := utf8.DecodeRune(rb.byte[inf.pos : inf.pos+inf.size])
+	return r
+}
+
+// bytesAt returns the UTF-8 encoding of the rune at position n.
+// It is used for Hangul and recomposition.
+func (rb *reorderBuffer) bytesAt(n int) []byte {
+	inf := rb.rune[n]
+	return rb.byte[inf.pos : int(inf.pos)+int(inf.size)]
+}
+
+// For Hangul we combine algorithmically, instead of using tables.
+const (
+	hangulBase  = 0xAC00 // UTF-8(hangulBase) -> EA B0 80
+	hangulBase0 = 0xEA
+	hangulBase1 = 0xB0
+	hangulBase2 = 0x80
+
+	hangulEnd  = hangulBase + jamoLVTCount // UTF-8(0xD7A4) -> ED 9E A4
+	hangulEnd0 = 0xED
+	hangulEnd1 = 0x9E
+	hangulEnd2 = 0xA4
+
+	jamoLBase  = 0x1100 // UTF-8(jamoLBase) -> E1 84 00
+	jamoLBase0 = 0xE1
+	jamoLBase1 = 0x84
+	jamoLEnd   = 0x1113
+	jamoVBase  = 0x1161
+	jamoVEnd   = 0x1176
+	jamoTBase  = 0x11A7
+	jamoTEnd   = 0x11C3
+
+	jamoTCount   = 28
+	jamoVCount   = 21
+	jamoVTCount  = 21 * 28
+	jamoLVTCount = 19 * 21 * 28
+)
+
+const hangulUTF8Size = 3
+
+func isHangul(b []byte) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+func isHangulString(b string) bool {
+	if len(b) < hangulUTF8Size {
+		return false
+	}
+	b0 := b[0]
+	if b0 < hangulBase0 {
+		return false
+	}
+	b1 := b[1]
+	switch {
+	case b0 == hangulBase0:
+		return b1 >= hangulBase1
+	case b0 < hangulEnd0:
+		return true
+	case b0 > hangulEnd0:
+		return false
+	case b1 < hangulEnd1:
+		return true
+	}
+	return b1 == hangulEnd1 && b[2] < hangulEnd2
+}
+
+// Caller must ensure len(b) >= 2.
+func isJamoVT(b []byte) bool {
+	// True if (rune & 0xff00) == jamoLBase
+	return b[0] == jamoLBase0 && (b[1]&0xFC) == jamoLBase1
+}
+
+func isHangulWithoutJamoT(b []byte) bool {
+	c, _ := utf8.DecodeRune(b)
+	c -= hangulBase
+	return c < jamoLVTCount && c%jamoTCount == 0
+}
+
+// decomposeHangul writes the decomposed Hangul to buf and returns the number
+// of bytes written.  len(buf) should be at least 9.
+func decomposeHangul(buf []byte, r rune) int {
+	const JamoUTF8Len = 3
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	utf8.EncodeRune(buf, jamoLBase+r/jamoVCount)
+	utf8.EncodeRune(buf[JamoUTF8Len:], jamoVBase+r%jamoVCount)
+	if x != 0 {
+		utf8.EncodeRune(buf[2*JamoUTF8Len:], jamoTBase+x)
+		return 3 * JamoUTF8Len
+	}
+	return 2 * JamoUTF8Len
+}
+
+// decomposeHangul algorithmically decomposes a Hangul rune into
+// its Jamo components.
+// See https://unicode.org/reports/tr15/#Hangul for details on decomposing Hangul.
+func (rb *reorderBuffer) decomposeHangul(r rune) {
+	r -= hangulBase
+	x := r % jamoTCount
+	r /= jamoTCount
+	rb.appendRune(jamoLBase + r/jamoVCount)
+	rb.appendRune(jamoVBase + r%jamoVCount)
+	if x != 0 {
+		rb.appendRune(jamoTBase + x)
+	}
+}
+
+// combineHangul algorithmically combines Jamo character components into Hangul.
+// See https://unicode.org/reports/tr15/#Hangul for details on combining Hangul.
+func (rb *reorderBuffer) combineHangul(s, i, k int) {
+	b := rb.rune[:]
+	bn := rb.nrune
+	for ; i < bn; i++ {
+		cccB := b[k-1].ccc
+		cccC := b[i].ccc
+		if cccB == 0 {
+			s = k - 1
+		}
+		if s != k-1 && cccB >= cccC {
+			// b[i] is blocked by greater-equal cccX below it
+			b[k] = b[i]
+			k++
+		} else {
+			l := rb.runeAt(s) // also used to compare to hangulBase
+			v := rb.runeAt(i) // also used to compare to jamoT
+			switch {
+			case jamoLBase <= l && l < jamoLEnd &&
+				jamoVBase <= v && v < jamoVEnd:
+				// 11xx plus 116x to LV
+				rb.assignRune(s, hangulBase+
+					(l-jamoLBase)*jamoVTCount+(v-jamoVBase)*jamoTCount)
+			case hangulBase <= l && l < hangulEnd &&
+				jamoTBase < v && v < jamoTEnd &&
+				((l-hangulBase)%jamoTCount) == 0:
+				// ACxx plus 11Ax to LVT
+				rb.assignRune(s, l+v-jamoTBase)
+			default:
+				b[k] = b[i]
+				k++
+			}
+		}
+	}
+	rb.nrune = k
+}
+
+// compose recombines the runes in the buffer.
+// It should only be used to recompose a single segment, as it will not
+// handle alternations between Hangul and non-Hangul characters correctly.
+func (rb *reorderBuffer) compose() {
+	// Lazily load the map used by the combine func below, but do
+	// it outside of the loop.
+	recompMapOnce.Do(buildRecompMap)
+
+	// UAX #15, section X5 , including Corrigendum #5
+	// "In any character sequence beginning with starter S, a character C is
+	//  blocked from S if and only if there is some character B between S
+	//  and C, and either B is a starter or it has the same or higher
+	//  combining class as C."
+	bn := rb.nrune
+	if bn == 0 {
+		return
+	}
+	k := 1
+	b := rb.rune[:]
+	for s, i := 0, 1; i < bn; i++ {
+		if isJamoVT(rb.bytesAt(i)) {
+			// Redo from start in Hangul mode. Necessary to support
+			// U+320E..U+321E in NFKC mode.
+			rb.combineHangul(s, i, k)
+			return
+		}
+		ii := b[i]
+		// We can only use combineForward as a filter if we later
+		// get the info for the combined character. This is more
+		// expensive than using the filter. Using combinesBackward()
+		// is safe.
+		if ii.combinesBackward() {
+			cccB := b[k-1].ccc
+			cccC := ii.ccc
+			blocked := false // b[i] blocked by starter or greater or equal CCC?
+			if cccB == 0 {
+				s = k - 1
+			} else {
+				blocked = s != k-1 && cccB >= cccC
+			}
+			if !blocked {
+				combined := combine(rb.runeAt(s), rb.runeAt(i))
+				if combined != 0 {
+					rb.assignRune(s, combined)
+					continue
+				}
+			}
+		}
+		b[k] = b[i]
+		k++
+	}
+	rb.nrune = k
+}

vendor/golang.org/x/text/unicode/norm/forminfo.go

@@ -0,0 +1,278 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "encoding/binary"
+
+// This file contains Form-specific logic and wrappers for data in tables.go.
+
+// Rune info is stored in a separate trie per composing form. A composing form
+// and its corresponding decomposing form share the same trie.  Each trie maps
+// a rune to a uint16. The values take two forms.  For v >= 0x8000:
+//   bits
+//   15:    1 (inverse of NFD_QC bit of qcInfo)
+//   13..7: qcInfo (see below). isYesD is always true (no decompostion).
+//    6..0: ccc (compressed CCC value).
+// For v < 0x8000, the respective rune has a decomposition and v is an index
+// into a byte array of UTF-8 decomposition sequences and additional info and
+// has the form:
+//    <header> <decomp_byte>* [<tccc> [<lccc>]]
+// The header contains the number of bytes in the decomposition (excluding this
+// length byte). The two most significant bits of this length byte correspond
+// to bit 5 and 4 of qcInfo (see below).  The byte sequence itself starts at v+1.
+// The byte sequence is followed by a trailing and leading CCC if the values
+// for these are not zero.  The value of v determines which ccc are appended
+// to the sequences.  For v < firstCCC, there are none, for v >= firstCCC,
+// the sequence is followed by a trailing ccc, and for v >= firstLeadingCC
+// there is an additional leading ccc. The value of tccc itself is the
+// trailing CCC shifted left 2 bits. The two least-significant bits of tccc
+// are the number of trailing non-starters.
+
+const (
+	qcInfoMask      = 0x3F // to clear all but the relevant bits in a qcInfo
+	headerLenMask   = 0x3F // extract the length value from the header byte
+	headerFlagsMask = 0xC0 // extract the qcInfo bits from the header byte
+)
+
+// Properties provides access to normalization properties of a rune.
+type Properties struct {
+	pos   uint8  // start position in reorderBuffer; used in composition.go
+	size  uint8  // length of UTF-8 encoding of this rune
+	ccc   uint8  // leading canonical combining class (ccc if not decomposition)
+	tccc  uint8  // trailing canonical combining class (ccc if not decomposition)
+	nLead uint8  // number of leading non-starters.
+	flags qcInfo // quick check flags
+	index uint16
+}
+
+// functions dispatchable per form
+type lookupFunc func(b input, i int) Properties
+
+// formInfo holds Form-specific functions and tables.
+type formInfo struct {
+	form                     Form
+	composing, compatibility bool // form type
+	info                     lookupFunc
+	nextMain                 iterFunc
+}
+
+var formTable = []*formInfo{{
+	form:          NFC,
+	composing:     true,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFD,
+	composing:     false,
+	compatibility: false,
+	info:          lookupInfoNFC,
+	nextMain:      nextDecomposed,
+}, {
+	form:          NFKC,
+	composing:     true,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextComposed,
+}, {
+	form:          NFKD,
+	composing:     false,
+	compatibility: true,
+	info:          lookupInfoNFKC,
+	nextMain:      nextDecomposed,
+}}
+
+// We do not distinguish between boundaries for NFC, NFD, etc. to avoid
+// unexpected behavior for the user.  For example, in NFD, there is a boundary
+// after 'a'.  However, 'a' might combine with modifiers, so from the application's
+// perspective it is not a good boundary. We will therefore always use the
+// boundaries for the combining variants.
+
+// BoundaryBefore returns true if this rune starts a new segment and
+// cannot combine with any rune on the left.
+func (p Properties) BoundaryBefore() bool {
+	if p.ccc == 0 && !p.combinesBackward() {
+		return true
+	}
+	// We assume that the CCC of the first character in a decomposition
+	// is always non-zero if different from info.ccc and that we can return
+	// false at this point. This is verified by maketables.
+	return false
+}
+
+// BoundaryAfter returns true if runes cannot combine with or otherwise
+// interact with this or previous runes.
+func (p Properties) BoundaryAfter() bool {
+	// TODO: loosen these conditions.
+	return p.isInert()
+}
+
+// We pack quick check data in 4 bits:
+//   5:    Combines forward  (0 == false, 1 == true)
+//   4..3: NFC_QC Yes(00), No (10), or Maybe (11)
+//   2:    NFD_QC Yes (0) or No (1). No also means there is a decomposition.
+//   1..0: Number of trailing non-starters.
+//
+// When all 4 bits are zero, the character is inert, meaning it is never
+// influenced by normalization.
+type qcInfo uint8
+
+func (p Properties) isYesC() bool { return p.flags&0x10 == 0 }
+func (p Properties) isYesD() bool { return p.flags&0x4 == 0 }
+
+func (p Properties) combinesForward() bool  { return p.flags&0x20 != 0 }
+func (p Properties) combinesBackward() bool { return p.flags&0x8 != 0 } // == isMaybe
+func (p Properties) hasDecomposition() bool { return p.flags&0x4 != 0 } // == isNoD
+
+func (p Properties) isInert() bool {
+	return p.flags&qcInfoMask == 0 && p.ccc == 0
+}
+
+func (p Properties) multiSegment() bool {
+	return p.index >= firstMulti && p.index < endMulti
+}
+
+func (p Properties) nLeadingNonStarters() uint8 {
+	return p.nLead
+}
+
+func (p Properties) nTrailingNonStarters() uint8 {
+	return uint8(p.flags & 0x03)
+}
+
+// Decomposition returns the decomposition for the underlying rune
+// or nil if there is none.
+func (p Properties) Decomposition() []byte {
+	// TODO: create the decomposition for Hangul?
+	if p.index == 0 {
+		return nil
+	}
+	i := p.index
+	n := decomps[i] & headerLenMask
+	i++
+	return decomps[i : i+uint16(n)]
+}
+
+// Size returns the length of UTF-8 encoding of the rune.
+func (p Properties) Size() int {
+	return int(p.size)
+}
+
+// CCC returns the canonical combining class of the underlying rune.
+func (p Properties) CCC() uint8 {
+	if p.index >= firstCCCZeroExcept {
+		return 0
+	}
+	return ccc[p.ccc]
+}
+
+// LeadCCC returns the CCC of the first rune in the decomposition.
+// If there is no decomposition, LeadCCC equals CCC.
+func (p Properties) LeadCCC() uint8 {
+	return ccc[p.ccc]
+}
+
+// TrailCCC returns the CCC of the last rune in the decomposition.
+// If there is no decomposition, TrailCCC equals CCC.
+func (p Properties) TrailCCC() uint8 {
+	return ccc[p.tccc]
+}
+
+func buildRecompMap() {
+	recompMap = make(map[uint32]rune, len(recompMapPacked)/8)
+	var buf [8]byte
+	for i := 0; i < len(recompMapPacked); i += 8 {
+		copy(buf[:], recompMapPacked[i:i+8])
+		key := binary.BigEndian.Uint32(buf[:4])
+		val := binary.BigEndian.Uint32(buf[4:])
+		recompMap[key] = rune(val)
+	}
+}
+
+// Recomposition
+// We use 32-bit keys instead of 64-bit for the two codepoint keys.
+// This clips off the bits of three entries, but we know this will not
+// result in a collision. In the unlikely event that changes to
+// UnicodeData.txt introduce collisions, the compiler will catch it.
+// Note that the recomposition map for NFC and NFKC are identical.
+
+// combine returns the combined rune or 0 if it doesn't exist.
+//
+// The caller is responsible for calling
+// recompMapOnce.Do(buildRecompMap) sometime before this is called.
+func combine(a, b rune) rune {
+	key := uint32(uint16(a))<<16 + uint32(uint16(b))
+	if recompMap == nil {
+		panic("caller error") // see func comment
+	}
+	return recompMap[key]
+}
+
+func lookupInfoNFC(b input, i int) Properties {
+	v, sz := b.charinfoNFC(i)
+	return compInfo(v, sz)
+}
+
+func lookupInfoNFKC(b input, i int) Properties {
+	v, sz := b.charinfoNFKC(i)
+	return compInfo(v, sz)
+}
+
+// Properties returns properties for the first rune in s.
+func (f Form) Properties(s []byte) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookup(s))
+	}
+	return compInfo(nfkcData.lookup(s))
+}
+
+// PropertiesString returns properties for the first rune in s.
+func (f Form) PropertiesString(s string) Properties {
+	if f == NFC || f == NFD {
+		return compInfo(nfcData.lookupString(s))
+	}
+	return compInfo(nfkcData.lookupString(s))
+}
+
+// compInfo converts the information contained in v and sz
+// to a Properties.  See the comment at the top of the file
+// for more information on the format.
+func compInfo(v uint16, sz int) Properties {
+	if v == 0 {
+		return Properties{size: uint8(sz)}
+	} else if v >= 0x8000 {
+		p := Properties{
+			size:  uint8(sz),
+			ccc:   uint8(v),
+			tccc:  uint8(v),
+			flags: qcInfo(v >> 8),
+		}
+		if p.ccc > 0 || p.combinesBackward() {
+			p.nLead = uint8(p.flags & 0x3)
+		}
+		return p
+	}
+	// has decomposition
+	h := decomps[v]
+	f := (qcInfo(h&headerFlagsMask) >> 2) | 0x4
+	p := Properties{size: uint8(sz), flags: f, index: v}
+	if v >= firstCCC {
+		v += uint16(h&headerLenMask) + 1
+		c := decomps[v]
+		p.tccc = c >> 2
+		p.flags |= qcInfo(c & 0x3)
+		if v >= firstLeadingCCC {
+			p.nLead = c & 0x3
+			if v >= firstStarterWithNLead {
+				// We were tricked. Remove the decomposition.
+				p.flags &= 0x03
+				p.index = 0
+				return p
+			}
+			p.ccc = decomps[v+1]
+		}
+	}
+	return p
+}

vendor/golang.org/x/text/unicode/norm/input.go

@@ -0,0 +1,109 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "unicode/utf8"
+
+type input struct {
+	str   string
+	bytes []byte
+}
+
+func inputBytes(str []byte) input {
+	return input{bytes: str}
+}
+
+func inputString(str string) input {
+	return input{str: str}
+}
+
+func (in *input) setBytes(str []byte) {
+	in.str = ""
+	in.bytes = str
+}
+
+func (in *input) setString(str string) {
+	in.str = str
+	in.bytes = nil
+}
+
+func (in *input) _byte(p int) byte {
+	if in.bytes == nil {
+		return in.str[p]
+	}
+	return in.bytes[p]
+}
+
+func (in *input) skipASCII(p, max int) int {
+	if in.bytes == nil {
+		for ; p < max && in.str[p] < utf8.RuneSelf; p++ {
+		}
+	} else {
+		for ; p < max && in.bytes[p] < utf8.RuneSelf; p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) skipContinuationBytes(p int) int {
+	if in.bytes == nil {
+		for ; p < len(in.str) && !utf8.RuneStart(in.str[p]); p++ {
+		}
+	} else {
+		for ; p < len(in.bytes) && !utf8.RuneStart(in.bytes[p]); p++ {
+		}
+	}
+	return p
+}
+
+func (in *input) appendSlice(buf []byte, b, e int) []byte {
+	if in.bytes != nil {
+		return append(buf, in.bytes[b:e]...)
+	}
+	for i := b; i < e; i++ {
+		buf = append(buf, in.str[i])
+	}
+	return buf
+}
+
+func (in *input) copySlice(buf []byte, b, e int) int {
+	if in.bytes == nil {
+		return copy(buf, in.str[b:e])
+	}
+	return copy(buf, in.bytes[b:e])
+}
+
+func (in *input) charinfoNFC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfcData.lookupString(in.str[p:])
+	}
+	return nfcData.lookup(in.bytes[p:])
+}
+
+func (in *input) charinfoNFKC(p int) (uint16, int) {
+	if in.bytes == nil {
+		return nfkcData.lookupString(in.str[p:])
+	}
+	return nfkcData.lookup(in.bytes[p:])
+}
+
+func (in *input) hangul(p int) (r rune) {
+	var size int
+	if in.bytes == nil {
+		if !isHangulString(in.str[p:]) {
+			return 0
+		}
+		r, size = utf8.DecodeRuneInString(in.str[p:])
+	} else {
+		if !isHangul(in.bytes[p:]) {
+			return 0
+		}
+		r, size = utf8.DecodeRune(in.bytes[p:])
+	}
+	if size != hangulUTF8Size {
+		return 0
+	}
+	return r
+}

vendor/golang.org/x/text/unicode/norm/iter.go

@@ -0,0 +1,458 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"fmt"
+	"unicode/utf8"
+)
+
+// MaxSegmentSize is the maximum size of a byte buffer needed to consider any
+// sequence of starter and non-starter runes for the purpose of normalization.
+const MaxSegmentSize = maxByteBufferSize
+
+// An Iter iterates over a string or byte slice, while normalizing it
+// to a given Form.
+type Iter struct {
+	rb     reorderBuffer
+	buf    [maxByteBufferSize]byte
+	info   Properties // first character saved from previous iteration
+	next   iterFunc   // implementation of next depends on form
+	asciiF iterFunc
+
+	p        int    // current position in input source
+	multiSeg []byte // remainder of multi-segment decomposition
+}
+
+type iterFunc func(*Iter) []byte
+
+// Init initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) Init(f Form, src []byte) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.init(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIBytes
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+}
+
+// InitString initializes i to iterate over src after normalizing it to Form f.
+func (i *Iter) InitString(f Form, src string) {
+	i.p = 0
+	if len(src) == 0 {
+		i.setDone()
+		i.rb.nsrc = 0
+		return
+	}
+	i.multiSeg = nil
+	i.rb.initString(f, src)
+	i.next = i.rb.f.nextMain
+	i.asciiF = nextASCIIString
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+}
+
+// Seek sets the segment to be returned by the next call to Next to start
+// at position p.  It is the responsibility of the caller to set p to the
+// start of a segment.
+func (i *Iter) Seek(offset int64, whence int) (int64, error) {
+	var abs int64
+	switch whence {
+	case 0:
+		abs = offset
+	case 1:
+		abs = int64(i.p) + offset
+	case 2:
+		abs = int64(i.rb.nsrc) + offset
+	default:
+		return 0, fmt.Errorf("norm: invalid whence")
+	}
+	if abs < 0 {
+		return 0, fmt.Errorf("norm: negative position")
+	}
+	if int(abs) >= i.rb.nsrc {
+		i.setDone()
+		return int64(i.p), nil
+	}
+	i.p = int(abs)
+	i.multiSeg = nil
+	i.next = i.rb.f.nextMain
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+	return abs, nil
+}
+
+// returnSlice returns a slice of the underlying input type as a byte slice.
+// If the underlying is of type []byte, it will simply return a slice.
+// If the underlying is of type string, it will copy the slice to the buffer
+// and return that.
+func (i *Iter) returnSlice(a, b int) []byte {
+	if i.rb.src.bytes == nil {
+		return i.buf[:copy(i.buf[:], i.rb.src.str[a:b])]
+	}
+	return i.rb.src.bytes[a:b]
+}
+
+// Pos returns the byte position at which the next call to Next will commence processing.
+func (i *Iter) Pos() int {
+	return i.p
+}
+
+func (i *Iter) setDone() {
+	i.next = nextDone
+	i.p = i.rb.nsrc
+}
+
+// Done returns true if there is no more input to process.
+func (i *Iter) Done() bool {
+	return i.p >= i.rb.nsrc
+}
+
+// Next returns f(i.input[i.Pos():n]), where n is a boundary of i.input.
+// For any input a and b for which f(a) == f(b), subsequent calls
+// to Next will return the same segments.
+// Modifying runes are grouped together with the preceding starter, if such a starter exists.
+// Although not guaranteed, n will typically be the smallest possible n.
+func (i *Iter) Next() []byte {
+	return i.next(i)
+}
+
+func nextASCIIBytes(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		p0 := i.p
+		i.setDone()
+		return i.rb.src.bytes[p0:p]
+	}
+	if i.rb.src.bytes[p] < utf8.RuneSelf {
+		p0 := i.p
+		i.p = p
+		return i.rb.src.bytes[p0:p]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextASCIIString(i *Iter) []byte {
+	p := i.p + 1
+	if p >= i.rb.nsrc {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.setDone()
+		return i.buf[:1]
+	}
+	if i.rb.src.str[p] < utf8.RuneSelf {
+		i.buf[0] = i.rb.src.str[i.p]
+		i.p = p
+		return i.buf[:1]
+	}
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+func nextHangul(i *Iter) []byte {
+	p := i.p
+	next := p + hangulUTF8Size
+	if next >= i.rb.nsrc {
+		i.setDone()
+	} else if i.rb.src.hangul(next) == 0 {
+		i.rb.ss.next(i.info)
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		i.next = i.rb.f.nextMain
+		return i.next(i)
+	}
+	i.p = next
+	return i.buf[:decomposeHangul(i.buf[:], i.rb.src.hangul(p))]
+}
+
+func nextDone(i *Iter) []byte {
+	return nil
+}
+
+// nextMulti is used for iterating over multi-segment decompositions
+// for decomposing normal forms.
+func nextMulti(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	// skip first rune
+	for j = 1; j < len(d) && !utf8.RuneStart(d[j]); j++ {
+	}
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.multiSeg = d[j:]
+			return d[:j]
+		}
+		j += int(info.size)
+	}
+	// treat last segment as normal decomposition
+	i.next = i.rb.f.nextMain
+	return i.next(i)
+}
+
+// nextMultiNorm is used for iterating over multi-segment decompositions
+// for composing normal forms.
+func nextMultiNorm(i *Iter) []byte {
+	j := 0
+	d := i.multiSeg
+	for j < len(d) {
+		info := i.rb.f.info(input{bytes: d}, j)
+		if info.BoundaryBefore() {
+			i.rb.compose()
+			seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+			i.rb.insertUnsafe(input{bytes: d}, j, info)
+			i.multiSeg = d[j+int(info.size):]
+			return seg
+		}
+		i.rb.insertUnsafe(input{bytes: d}, j, info)
+		j += int(info.size)
+	}
+	i.multiSeg = nil
+	i.next = nextComposed
+	return doNormComposed(i)
+}
+
+// nextDecomposed is the implementation of Next for forms NFD and NFKD.
+func nextDecomposed(i *Iter) (next []byte) {
+	outp := 0
+	inCopyStart, outCopyStart := i.p, 0
+	for {
+		if sz := int(i.info.size); sz <= 1 {
+			i.rb.ss = 0
+			p := i.p
+			i.p++ // ASCII or illegal byte.  Either way, advance by 1.
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				return i.returnSlice(p, i.p)
+			} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+				i.next = i.asciiF
+				return i.returnSlice(p, i.p)
+			}
+			outp++
+		} else if d := i.info.Decomposition(); d != nil {
+			// Note: If leading CCC != 0, then len(d) == 2 and last is also non-zero.
+			// Case 1: there is a leftover to copy.  In this case the decomposition
+			// must begin with a modifier and should always be appended.
+			// Case 2: no leftover. Simply return d if followed by a ccc == 0 value.
+			p := outp + len(d)
+			if outp > 0 {
+				i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+				// TODO: this condition should not be possible, but we leave it
+				// in for defensive purposes.
+				if p > len(i.buf) {
+					return i.buf[:outp]
+				}
+			} else if i.info.multiSegment() {
+				// outp must be 0 as multi-segment decompositions always
+				// start a new segment.
+				if i.multiSeg == nil {
+					i.multiSeg = d
+					i.next = nextMulti
+					return nextMulti(i)
+				}
+				// We are in the last segment.  Treat as normal decomposition.
+				d = i.multiSeg
+				i.multiSeg = nil
+				p = len(d)
+			}
+			prevCC := i.info.tccc
+			if i.p += sz; i.p >= i.rb.nsrc {
+				i.setDone()
+				i.info = Properties{} // Force BoundaryBefore to succeed.
+			} else {
+				i.info = i.rb.f.info(i.rb.src, i.p)
+			}
+			switch i.rb.ss.next(i.info) {
+			case ssOverflow:
+				i.next = nextCGJDecompose
+				fallthrough
+			case ssStarter:
+				if outp > 0 {
+					copy(i.buf[outp:], d)
+					return i.buf[:p]
+				}
+				return d
+			}
+			copy(i.buf[outp:], d)
+			outp = p
+			inCopyStart, outCopyStart = i.p, outp
+			if i.info.ccc < prevCC {
+				goto doNorm
+			}
+			continue
+		} else if r := i.rb.src.hangul(i.p); r != 0 {
+			outp = decomposeHangul(i.buf[:], r)
+			i.p += hangulUTF8Size
+			inCopyStart, outCopyStart = i.p, outp
+			if i.p >= i.rb.nsrc {
+				i.setDone()
+				break
+			} else if i.rb.src.hangul(i.p) != 0 {
+				i.next = nextHangul
+				return i.buf[:outp]
+			}
+		} else {
+			p := outp + sz
+			if p > len(i.buf) {
+				break
+			}
+			outp = p
+			i.p += sz
+		}
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		prevCC := i.info.tccc
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := i.rb.ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	if outCopyStart == 0 {
+		return i.returnSlice(inCopyStart, i.p)
+	} else if inCopyStart < i.p {
+		i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	}
+	return i.buf[:outp]
+doNorm:
+	// Insert what we have decomposed so far in the reorderBuffer.
+	// As we will only reorder, there will always be enough room.
+	i.rb.src.copySlice(i.buf[outCopyStart:], inCopyStart, i.p)
+	i.rb.insertDecomposed(i.buf[0:outp])
+	return doNormDecomposed(i)
+}
+
+func doNormDecomposed(i *Iter) []byte {
+	for {
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if i.info.ccc == 0 {
+			break
+		}
+		if s := i.rb.ss.next(i.info); s == ssOverflow {
+			i.next = nextCGJDecompose
+			break
+		}
+	}
+	// new segment or too many combining characters: exit normalization
+	return i.buf[:i.rb.flushCopy(i.buf[:])]
+}
+
+func nextCGJDecompose(i *Iter) []byte {
+	i.rb.ss = 0
+	i.rb.insertCGJ()
+	i.next = nextDecomposed
+	i.rb.ss.first(i.info)
+	buf := doNormDecomposed(i)
+	return buf
+}
+
+// nextComposed is the implementation of Next for forms NFC and NFKC.
+func nextComposed(i *Iter) []byte {
+	outp, startp := 0, i.p
+	var prevCC uint8
+	for {
+		if !i.info.isYesC() {
+			goto doNorm
+		}
+		prevCC = i.info.tccc
+		sz := int(i.info.size)
+		if sz == 0 {
+			sz = 1 // illegal rune: copy byte-by-byte
+		}
+		p := outp + sz
+		if p > len(i.buf) {
+			break
+		}
+		outp = p
+		i.p += sz
+		if i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		} else if i.rb.src._byte(i.p) < utf8.RuneSelf {
+			i.rb.ss = 0
+			i.next = i.asciiF
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if v := i.rb.ss.next(i.info); v == ssStarter {
+			break
+		} else if v == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		if i.info.ccc < prevCC {
+			goto doNorm
+		}
+	}
+	return i.returnSlice(startp, i.p)
+doNorm:
+	// reset to start position
+	i.p = startp
+	i.info = i.rb.f.info(i.rb.src, i.p)
+	i.rb.ss.first(i.info)
+	if i.info.multiSegment() {
+		d := i.info.Decomposition()
+		info := i.rb.f.info(input{bytes: d}, 0)
+		i.rb.insertUnsafe(input{bytes: d}, 0, info)
+		i.multiSeg = d[int(info.size):]
+		i.next = nextMultiNorm
+		return nextMultiNorm(i)
+	}
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}
+
+func doNormComposed(i *Iter) []byte {
+	// First rune should already be inserted.
+	for {
+		if i.p += int(i.info.size); i.p >= i.rb.nsrc {
+			i.setDone()
+			break
+		}
+		i.info = i.rb.f.info(i.rb.src, i.p)
+		if s := i.rb.ss.next(i.info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			i.next = nextCGJCompose
+			break
+		}
+		i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	}
+	i.rb.compose()
+	seg := i.buf[:i.rb.flushCopy(i.buf[:])]
+	return seg
+}
+
+func nextCGJCompose(i *Iter) []byte {
+	i.rb.ss = 0 // instead of first
+	i.rb.insertCGJ()
+	i.next = nextComposed
+	// Note that we treat any rune with nLeadingNonStarters > 0 as a non-starter,
+	// even if they are not. This is particularly dubious for U+FF9E and UFF9A.
+	// If we ever change that, insert a check here.
+	i.rb.ss.first(i.info)
+	i.rb.insertUnsafe(i.rb.src, i.p, i.info)
+	return doNormComposed(i)
+}

vendor/golang.org/x/text/unicode/norm/maketables.go

@@ -0,0 +1,986 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Normalization table generator.
+// Data read from the web.
+// See forminfo.go for a description of the trie values associated with each rune.
+
+package main
+
+import (
+	"bytes"
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"sort"
+	"strconv"
+	"strings"
+
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/triegen"
+	"golang.org/x/text/internal/ucd"
+)
+
+func main() {
+	gen.Init()
+	loadUnicodeData()
+	compactCCC()
+	loadCompositionExclusions()
+	completeCharFields(FCanonical)
+	completeCharFields(FCompatibility)
+	computeNonStarterCounts()
+	verifyComputed()
+	printChars()
+	testDerived()
+	printTestdata()
+	makeTables()
+}
+
+var (
+	tablelist = flag.String("tables",
+		"all",
+		"comma-separated list of which tables to generate; "+
+			"can be 'decomp', 'recomp', 'info' and 'all'")
+	test = flag.Bool("test",
+		false,
+		"test existing tables against DerivedNormalizationProps and generate test data for regression testing")
+	verbose = flag.Bool("verbose",
+		false,
+		"write data to stdout as it is parsed")
+)
+
+const MaxChar = 0x10FFFF // anything above this shouldn't exist
+
+// Quick Check properties of runes allow us to quickly
+// determine whether a rune may occur in a normal form.
+// For a given normal form, a rune may be guaranteed to occur
+// verbatim (QC=Yes), may or may not combine with another
+// rune (QC=Maybe), or may not occur (QC=No).
+type QCResult int
+
+const (
+	QCUnknown QCResult = iota
+	QCYes
+	QCNo
+	QCMaybe
+)
+
+func (r QCResult) String() string {
+	switch r {
+	case QCYes:
+		return "Yes"
+	case QCNo:
+		return "No"
+	case QCMaybe:
+		return "Maybe"
+	}
+	return "***UNKNOWN***"
+}
+
+const (
+	FCanonical     = iota // NFC or NFD
+	FCompatibility        // NFKC or NFKD
+	FNumberOfFormTypes
+)
+
+const (
+	MComposed   = iota // NFC or NFKC
+	MDecomposed        // NFD or NFKD
+	MNumberOfModes
+)
+
+// This contains only the properties we're interested in.
+type Char struct {
+	name          string
+	codePoint     rune  // if zero, this index is not a valid code point.
+	ccc           uint8 // canonical combining class
+	origCCC       uint8
+	excludeInComp bool // from CompositionExclusions.txt
+	compatDecomp  bool // it has a compatibility expansion
+
+	nTrailingNonStarters uint8
+	nLeadingNonStarters  uint8 // must be equal to trailing if non-zero
+
+	forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility
+
+	state State
+}
+
+var chars = make([]Char, MaxChar+1)
+var cccMap = make(map[uint8]uint8)
+
+func (c Char) String() string {
+	buf := new(bytes.Buffer)
+
+	fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name)
+	fmt.Fprintf(buf, "  ccc: %v\n", c.ccc)
+	fmt.Fprintf(buf, "  excludeInComp: %v\n", c.excludeInComp)
+	fmt.Fprintf(buf, "  compatDecomp: %v\n", c.compatDecomp)
+	fmt.Fprintf(buf, "  state: %v\n", c.state)
+	fmt.Fprintf(buf, "  NFC:\n")
+	fmt.Fprint(buf, c.forms[FCanonical])
+	fmt.Fprintf(buf, "  NFKC:\n")
+	fmt.Fprint(buf, c.forms[FCompatibility])
+
+	return buf.String()
+}
+
+// In UnicodeData.txt, some ranges are marked like this:
+//	3400;<CJK Ideograph Extension A, First>;Lo;0;L;;;;;N;;;;;
+//	4DB5;<CJK Ideograph Extension A, Last>;Lo;0;L;;;;;N;;;;;
+// parseCharacter keeps a state variable indicating the weirdness.
+type State int
+
+const (
+	SNormal State = iota // known to be zero for the type
+	SFirst
+	SLast
+	SMissing
+)
+
+var lastChar = rune('\u0000')
+
+func (c Char) isValid() bool {
+	return c.codePoint != 0 && c.state != SMissing
+}
+
+type FormInfo struct {
+	quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed
+	verified   [MNumberOfModes]bool     // index: MComposed or MDecomposed
+
+	combinesForward  bool // May combine with rune on the right
+	combinesBackward bool // May combine with rune on the left
+	isOneWay         bool // Never appears in result
+	inDecomp         bool // Some decompositions result in this char.
+	decomp           Decomposition
+	expandedDecomp   Decomposition
+}
+
+func (f FormInfo) String() string {
+	buf := bytes.NewBuffer(make([]byte, 0))
+
+	fmt.Fprintf(buf, "    quickCheck[C]: %v\n", f.quickCheck[MComposed])
+	fmt.Fprintf(buf, "    quickCheck[D]: %v\n", f.quickCheck[MDecomposed])
+	fmt.Fprintf(buf, "    cmbForward: %v\n", f.combinesForward)
+	fmt.Fprintf(buf, "    cmbBackward: %v\n", f.combinesBackward)
+	fmt.Fprintf(buf, "    isOneWay: %v\n", f.isOneWay)
+	fmt.Fprintf(buf, "    inDecomp: %v\n", f.inDecomp)
+	fmt.Fprintf(buf, "    decomposition: %X\n", f.decomp)
+	fmt.Fprintf(buf, "    expandedDecomp: %X\n", f.expandedDecomp)
+
+	return buf.String()
+}
+
+type Decomposition []rune
+
+func parseDecomposition(s string, skipfirst bool) (a []rune, err error) {
+	decomp := strings.Split(s, " ")
+	if len(decomp) > 0 && skipfirst {
+		decomp = decomp[1:]
+	}
+	for _, d := range decomp {
+		point, err := strconv.ParseUint(d, 16, 64)
+		if err != nil {
+			return a, err
+		}
+		a = append(a, rune(point))
+	}
+	return a, nil
+}
+
+func loadUnicodeData() {
+	f := gen.OpenUCDFile("UnicodeData.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(ucd.CodePoint)
+		char := &chars[r]
+
+		char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass))
+		decmap := p.String(ucd.DecompMapping)
+
+		exp, err := parseDecomposition(decmap, false)
+		isCompat := false
+		if err != nil {
+			if len(decmap) > 0 {
+				exp, err = parseDecomposition(decmap, true)
+				if err != nil {
+					log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err)
+				}
+				isCompat = true
+			}
+		}
+
+		char.name = p.String(ucd.Name)
+		char.codePoint = r
+		char.forms[FCompatibility].decomp = exp
+		if !isCompat {
+			char.forms[FCanonical].decomp = exp
+		} else {
+			char.compatDecomp = true
+		}
+		if len(decmap) > 0 {
+			char.forms[FCompatibility].decomp = exp
+		}
+	}
+	if err := p.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// compactCCC converts the sparse set of CCC values to a continguous one,
+// reducing the number of bits needed from 8 to 6.
+func compactCCC() {
+	m := make(map[uint8]uint8)
+	for i := range chars {
+		c := &chars[i]
+		m[c.ccc] = 0
+	}
+	cccs := []int{}
+	for v, _ := range m {
+		cccs = append(cccs, int(v))
+	}
+	sort.Ints(cccs)
+	for i, c := range cccs {
+		cccMap[uint8(i)] = uint8(c)
+		m[uint8(c)] = uint8(i)
+	}
+	for i := range chars {
+		c := &chars[i]
+		c.origCCC = c.ccc
+		c.ccc = m[c.ccc]
+	}
+	if len(m) >= 1<<6 {
+		log.Fatalf("too many difference CCC values: %d >= 64", len(m))
+	}
+}
+
+// CompositionExclusions.txt has form:
+// 0958    # ...
+// See https://unicode.org/reports/tr44/ for full explanation
+func loadCompositionExclusions() {
+	f := gen.OpenUCDFile("CompositionExclusions.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		c := &chars[p.Rune(0)]
+		if c.excludeInComp {
+			log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint)
+		}
+		c.excludeInComp = true
+	}
+	if e := p.Err(); e != nil {
+		log.Fatal(e)
+	}
+}
+
+// hasCompatDecomp returns true if any of the recursive
+// decompositions contains a compatibility expansion.
+// In this case, the character may not occur in NFK*.
+func hasCompatDecomp(r rune) bool {
+	c := &chars[r]
+	if c.compatDecomp {
+		return true
+	}
+	for _, d := range c.forms[FCompatibility].decomp {
+		if hasCompatDecomp(d) {
+			return true
+		}
+	}
+	return false
+}
+
+// Hangul related constants.
+const (
+	HangulBase = 0xAC00
+	HangulEnd  = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28)
+
+	JamoLBase = 0x1100
+	JamoLEnd  = 0x1113
+	JamoVBase = 0x1161
+	JamoVEnd  = 0x1176
+	JamoTBase = 0x11A8
+	JamoTEnd  = 0x11C3
+
+	JamoLVTCount = 19 * 21 * 28
+	JamoTCount   = 28
+)
+
+func isHangul(r rune) bool {
+	return HangulBase <= r && r < HangulEnd
+}
+
+func isHangulWithoutJamoT(r rune) bool {
+	if !isHangul(r) {
+		return false
+	}
+	r -= HangulBase
+	return r < JamoLVTCount && r%JamoTCount == 0
+}
+
+func ccc(r rune) uint8 {
+	return chars[r].ccc
+}
+
+// Insert a rune in a buffer, ordered by Canonical Combining Class.
+func insertOrdered(b Decomposition, r rune) Decomposition {
+	n := len(b)
+	b = append(b, 0)
+	cc := ccc(r)
+	if cc > 0 {
+		// Use bubble sort.
+		for ; n > 0; n-- {
+			if ccc(b[n-1]) <= cc {
+				break
+			}
+			b[n] = b[n-1]
+		}
+	}
+	b[n] = r
+	return b
+}
+
+// Recursively decompose.
+func decomposeRecursive(form int, r rune, d Decomposition) Decomposition {
+	dcomp := chars[r].forms[form].decomp
+	if len(dcomp) == 0 {
+		return insertOrdered(d, r)
+	}
+	for _, c := range dcomp {
+		d = decomposeRecursive(form, c, d)
+	}
+	return d
+}
+
+func completeCharFields(form int) {
+	// Phase 0: pre-expand decomposition.
+	for i := range chars {
+		f := &chars[i].forms[form]
+		if len(f.decomp) == 0 {
+			continue
+		}
+		exp := make(Decomposition, 0)
+		for _, c := range f.decomp {
+			exp = decomposeRecursive(form, c, exp)
+		}
+		f.expandedDecomp = exp
+	}
+
+	// Phase 1: composition exclusion, mark decomposition.
+	for i := range chars {
+		c := &chars[i]
+		f := &c.forms[form]
+
+		// Marks script-specific exclusions and version restricted.
+		f.isOneWay = c.excludeInComp
+
+		// Singletons
+		f.isOneWay = f.isOneWay || len(f.decomp) == 1
+
+		// Non-starter decompositions
+		if len(f.decomp) > 1 {
+			chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0
+			f.isOneWay = f.isOneWay || chk
+		}
+
+		// Runes that decompose into more than two runes.
+		f.isOneWay = f.isOneWay || len(f.decomp) > 2
+
+		if form == FCompatibility {
+			f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint)
+		}
+
+		for _, r := range f.decomp {
+			chars[r].forms[form].inDecomp = true
+		}
+	}
+
+	// Phase 2: forward and backward combining.
+	for i := range chars {
+		c := &chars[i]
+		f := &c.forms[form]
+
+		if !f.isOneWay && len(f.decomp) == 2 {
+			f0 := &chars[f.decomp[0]].forms[form]
+			f1 := &chars[f.decomp[1]].forms[form]
+			if !f0.isOneWay {
+				f0.combinesForward = true
+			}
+			if !f1.isOneWay {
+				f1.combinesBackward = true
+			}
+		}
+		if isHangulWithoutJamoT(rune(i)) {
+			f.combinesForward = true
+		}
+	}
+
+	// Phase 3: quick check values.
+	for i := range chars {
+		c := &chars[i]
+		f := &c.forms[form]
+
+		switch {
+		case len(f.decomp) > 0:
+			f.quickCheck[MDecomposed] = QCNo
+		case isHangul(rune(i)):
+			f.quickCheck[MDecomposed] = QCNo
+		default:
+			f.quickCheck[MDecomposed] = QCYes
+		}
+		switch {
+		case f.isOneWay:
+			f.quickCheck[MComposed] = QCNo
+		case (i & 0xffff00) == JamoLBase:
+			f.quickCheck[MComposed] = QCYes
+			if JamoLBase <= i && i < JamoLEnd {
+				f.combinesForward = true
+			}
+			if JamoVBase <= i && i < JamoVEnd {
+				f.quickCheck[MComposed] = QCMaybe
+				f.combinesBackward = true
+				f.combinesForward = true
+			}
+			if JamoTBase <= i && i < JamoTEnd {
+				f.quickCheck[MComposed] = QCMaybe
+				f.combinesBackward = true
+			}
+		case !f.combinesBackward:
+			f.quickCheck[MComposed] = QCYes
+		default:
+			f.quickCheck[MComposed] = QCMaybe
+		}
+	}
+}
+
+func computeNonStarterCounts() {
+	// Phase 4: leading and trailing non-starter count
+	for i := range chars {
+		c := &chars[i]
+
+		runes := []rune{rune(i)}
+		// We always use FCompatibility so that the CGJ insertion points do not
+		// change for repeated normalizations with different forms.
+		if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 {
+			runes = exp
+		}
+		// We consider runes that combine backwards to be non-starters for the
+		// purpose of Stream-Safe Text Processing.
+		for _, r := range runes {
+			if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
+				break
+			}
+			c.nLeadingNonStarters++
+		}
+		for i := len(runes) - 1; i >= 0; i-- {
+			if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward {
+				break
+			}
+			c.nTrailingNonStarters++
+		}
+		if c.nTrailingNonStarters > 3 {
+			log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes)
+		}
+
+		if isHangul(rune(i)) {
+			c.nTrailingNonStarters = 2
+			if isHangulWithoutJamoT(rune(i)) {
+				c.nTrailingNonStarters = 1
+			}
+		}
+
+		if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t {
+			log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t)
+		}
+		if t := c.nTrailingNonStarters; t > 3 {
+			log.Fatalf("%U: number of trailing non-starters is %d > 3", t)
+		}
+	}
+}
+
+func printBytes(w io.Writer, b []byte, name string) {
+	fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b))
+	fmt.Fprintf(w, "var %s = [...]byte {", name)
+	for i, c := range b {
+		switch {
+		case i%64 == 0:
+			fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63)
+		case i%8 == 0:
+			fmt.Fprintf(w, "\n")
+		}
+		fmt.Fprintf(w, "0x%.2X, ", c)
+	}
+	fmt.Fprint(w, "\n}\n\n")
+}
+
+// See forminfo.go for format.
+func makeEntry(f *FormInfo, c *Char) uint16 {
+	e := uint16(0)
+	if r := c.codePoint; HangulBase <= r && r < HangulEnd {
+		e |= 0x40
+	}
+	if f.combinesForward {
+		e |= 0x20
+	}
+	if f.quickCheck[MDecomposed] == QCNo {
+		e |= 0x4
+	}
+	switch f.quickCheck[MComposed] {
+	case QCYes:
+	case QCNo:
+		e |= 0x10
+	case QCMaybe:
+		e |= 0x18
+	default:
+		log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed])
+	}
+	e |= uint16(c.nTrailingNonStarters)
+	return e
+}
+
+// decompSet keeps track of unique decompositions, grouped by whether
+// the decomposition is followed by a trailing and/or leading CCC.
+type decompSet [7]map[string]bool
+
+const (
+	normalDecomp = iota
+	firstMulti
+	firstCCC
+	endMulti
+	firstLeadingCCC
+	firstCCCZeroExcept
+	firstStarterWithNLead
+	lastDecomp
+)
+
+var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"}
+
+func makeDecompSet() decompSet {
+	m := decompSet{}
+	for i := range m {
+		m[i] = make(map[string]bool)
+	}
+	return m
+}
+func (m *decompSet) insert(key int, s string) {
+	m[key][s] = true
+}
+
+func printCharInfoTables(w io.Writer) int {
+	mkstr := func(r rune, f *FormInfo) (int, string) {
+		d := f.expandedDecomp
+		s := string([]rune(d))
+		if max := 1 << 6; len(s) >= max {
+			const msg = "%U: too many bytes in decomposition: %d >= %d"
+			log.Fatalf(msg, r, len(s), max)
+		}
+		head := uint8(len(s))
+		if f.quickCheck[MComposed] != QCYes {
+			head |= 0x40
+		}
+		if f.combinesForward {
+			head |= 0x80
+		}
+		s = string([]byte{head}) + s
+
+		lccc := ccc(d[0])
+		tccc := ccc(d[len(d)-1])
+		cc := ccc(r)
+		if cc != 0 && lccc == 0 && tccc == 0 {
+			log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc)
+		}
+		if tccc < lccc && lccc != 0 {
+			const msg = "%U: lccc (%d) must be <= tcc (%d)"
+			log.Fatalf(msg, r, lccc, tccc)
+		}
+		index := normalDecomp
+		nTrail := chars[r].nTrailingNonStarters
+		nLead := chars[r].nLeadingNonStarters
+		if tccc > 0 || lccc > 0 || nTrail > 0 {
+			tccc <<= 2
+			tccc |= nTrail
+			s += string([]byte{tccc})
+			index = endMulti
+			for _, r := range d[1:] {
+				if ccc(r) == 0 {
+					index = firstCCC
+				}
+			}
+			if lccc > 0 || nLead > 0 {
+				s += string([]byte{lccc})
+				if index == firstCCC {
+					log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r)
+				}
+				index = firstLeadingCCC
+			}
+			if cc != lccc {
+				if cc != 0 {
+					log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc)
+				}
+				index = firstCCCZeroExcept
+			}
+		} else if len(d) > 1 {
+			index = firstMulti
+		}
+		return index, s
+	}
+
+	decompSet := makeDecompSet()
+	const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail.
+	decompSet.insert(firstStarterWithNLead, nLeadStr)
+
+	// Store the uniqued decompositions in a byte buffer,
+	// preceded by their byte length.
+	for _, c := range chars {
+		for _, f := range c.forms {
+			if len(f.expandedDecomp) == 0 {
+				continue
+			}
+			if f.combinesBackward {
+				log.Fatalf("%U: combinesBackward and decompose", c.codePoint)
+			}
+			index, s := mkstr(c.codePoint, &f)
+			decompSet.insert(index, s)
+		}
+	}
+
+	decompositions := bytes.NewBuffer(make([]byte, 0, 10000))
+	size := 0
+	positionMap := make(map[string]uint16)
+	decompositions.WriteString("\000")
+	fmt.Fprintln(w, "const (")
+	for i, m := range decompSet {
+		sa := []string{}
+		for s := range m {
+			sa = append(sa, s)
+		}
+		sort.Strings(sa)
+		for _, s := range sa {
+			p := decompositions.Len()
+			decompositions.WriteString(s)
+			positionMap[s] = uint16(p)
+		}
+		if cname[i] != "" {
+			fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len())
+		}
+	}
+	fmt.Fprintln(w, "maxDecomp = 0x8000")
+	fmt.Fprintln(w, ")")
+	b := decompositions.Bytes()
+	printBytes(w, b, "decomps")
+	size += len(b)
+
+	varnames := []string{"nfc", "nfkc"}
+	for i := 0; i < FNumberOfFormTypes; i++ {
+		trie := triegen.NewTrie(varnames[i])
+
+		for r, c := range chars {
+			f := c.forms[i]
+			d := f.expandedDecomp
+			if len(d) != 0 {
+				_, key := mkstr(c.codePoint, &f)
+				trie.Insert(rune(r), uint64(positionMap[key]))
+				if c.ccc != ccc(d[0]) {
+					// We assume the lead ccc of a decomposition !=0 in this case.
+					if ccc(d[0]) == 0 {
+						log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc)
+					}
+				}
+			} else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward {
+				// Handle cases where it can't be detected that the nLead should be equal
+				// to nTrail.
+				trie.Insert(c.codePoint, uint64(positionMap[nLeadStr]))
+			} else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 {
+				trie.Insert(c.codePoint, uint64(0x8000|v))
+			}
+		}
+		sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]}))
+		if err != nil {
+			log.Fatal(err)
+		}
+		size += sz
+	}
+	return size
+}
+
+func contains(sa []string, s string) bool {
+	for _, a := range sa {
+		if a == s {
+			return true
+		}
+	}
+	return false
+}
+
+func makeTables() {
+	w := &bytes.Buffer{}
+
+	size := 0
+	if *tablelist == "" {
+		return
+	}
+	list := strings.Split(*tablelist, ",")
+	if *tablelist == "all" {
+		list = []string{"recomp", "info"}
+	}
+
+	// Compute maximum decomposition size.
+	max := 0
+	for _, c := range chars {
+		if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max {
+			max = n
+		}
+	}
+	fmt.Fprintln(w, `import "sync"`)
+	fmt.Fprintln(w)
+
+	fmt.Fprintln(w, "const (")
+	fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.")
+	fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion())
+	fmt.Fprintln(w)
+	fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform")
+	fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at")
+	fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that")
+	fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.")
+	fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max)
+	fmt.Fprintln(w, ")\n")
+
+	// Print the CCC remap table.
+	size += len(cccMap)
+	fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap))
+	for i := 0; i < len(cccMap); i++ {
+		if i%8 == 0 {
+			fmt.Fprintln(w)
+		}
+		fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)])
+	}
+	fmt.Fprintln(w, "\n}\n")
+
+	if contains(list, "info") {
+		size += printCharInfoTables(w)
+	}
+
+	if contains(list, "recomp") {
+		// Note that we use 32 bit keys, instead of 64 bit.
+		// This clips the bits of three entries, but we know
+		// this won't cause a collision. The compiler will catch
+		// any changes made to UnicodeData.txt that introduces
+		// a collision.
+		// Note that the recomposition map for NFC and NFKC
+		// are identical.
+
+		// Recomposition map
+		nrentries := 0
+		for _, c := range chars {
+			f := c.forms[FCanonical]
+			if !f.isOneWay && len(f.decomp) > 0 {
+				nrentries++
+			}
+		}
+		sz := nrentries * 8
+		size += sz
+		fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz)
+		fmt.Fprintln(w, "var recompMap map[uint32]rune")
+		fmt.Fprintln(w, "var recompMapOnce sync.Once\n")
+		fmt.Fprintln(w, `const recompMapPacked = "" +`)
+		var buf [8]byte
+		for i, c := range chars {
+			f := c.forms[FCanonical]
+			d := f.decomp
+			if !f.isOneWay && len(d) > 0 {
+				key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1]))
+				binary.BigEndian.PutUint32(buf[:4], key)
+				binary.BigEndian.PutUint32(buf[4:], uint32(i))
+				fmt.Fprintf(w, "\t\t%q + // 0x%.8X: 0x%.8X\n", string(buf[:]), key, uint32(i))
+			}
+		}
+		// hack so we don't have to special case the trailing plus sign
+		fmt.Fprintf(w, `	""`)
+		fmt.Fprintln(w)
+	}
+
+	fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size)
+	gen.WriteVersionedGoFile("tables.go", "norm", w.Bytes())
+}
+
+func printChars() {
+	if *verbose {
+		for _, c := range chars {
+			if !c.isValid() || c.state == SMissing {
+				continue
+			}
+			fmt.Println(c)
+		}
+	}
+}
+
+// verifyComputed does various consistency tests.
+func verifyComputed() {
+	for i, c := range chars {
+		for _, f := range c.forms {
+			isNo := (f.quickCheck[MDecomposed] == QCNo)
+			if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) {
+				log.Fatalf("%U: NF*D QC must be No if rune decomposes", i)
+			}
+
+			isMaybe := f.quickCheck[MComposed] == QCMaybe
+			if f.combinesBackward != isMaybe {
+				log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i)
+			}
+			if len(f.decomp) > 0 && f.combinesForward && isMaybe {
+				log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i)
+			}
+
+			if len(f.expandedDecomp) != 0 {
+				continue
+			}
+			if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b {
+				// We accept these runes to be treated differently (it only affects
+				// segment breaking in iteration, most likely on improper use), but
+				// reconsider if more characters are added.
+				// U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L;<narrow> 3099;;;;N;;;;;
+				// U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L;<narrow> 309A;;;;N;;;;;
+				// U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<compat> 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;;
+				// U+318E HANGUL LETTER ARAEAE;Lo;0;L;<compat> 11A1;;;;N;HANGUL LETTER ALAE AE;;;;
+				// U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L;<narrow> 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;;
+				// U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L;<narrow> 3163;;;;N;;;;;
+				if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) {
+					log.Fatalf("%U: nLead was %v; want %v", i, a, b)
+				}
+			}
+		}
+		nfc := c.forms[FCanonical]
+		nfkc := c.forms[FCompatibility]
+		if nfc.combinesBackward != nfkc.combinesBackward {
+			log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint)
+		}
+	}
+}
+
+// Use values in DerivedNormalizationProps.txt to compare against the
+// values we computed.
+// DerivedNormalizationProps.txt has form:
+// 00C0..00C5    ; NFD_QC; N # ...
+// 0374          ; NFD_QC; N # ...
+// See https://unicode.org/reports/tr44/ for full explanation
+func testDerived() {
+	f := gen.OpenUCDFile("DerivedNormalizationProps.txt")
+	defer f.Close()
+	p := ucd.New(f)
+	for p.Next() {
+		r := p.Rune(0)
+		c := &chars[r]
+
+		var ftype, mode int
+		qt := p.String(1)
+		switch qt {
+		case "NFC_QC":
+			ftype, mode = FCanonical, MComposed
+		case "NFD_QC":
+			ftype, mode = FCanonical, MDecomposed
+		case "NFKC_QC":
+			ftype, mode = FCompatibility, MComposed
+		case "NFKD_QC":
+			ftype, mode = FCompatibility, MDecomposed
+		default:
+			continue
+		}
+		var qr QCResult
+		switch p.String(2) {
+		case "Y":
+			qr = QCYes
+		case "N":
+			qr = QCNo
+		case "M":
+			qr = QCMaybe
+		default:
+			log.Fatalf(`Unexpected quick check value "%s"`, p.String(2))
+		}
+		if got := c.forms[ftype].quickCheck[mode]; got != qr {
+			log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr)
+		}
+		c.forms[ftype].verified[mode] = true
+	}
+	if err := p.Err(); err != nil {
+		log.Fatal(err)
+	}
+	// Any unspecified value must be QCYes. Verify this.
+	for i, c := range chars {
+		for j, fd := range c.forms {
+			for k, qr := range fd.quickCheck {
+				if !fd.verified[k] && qr != QCYes {
+					m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n"
+					log.Printf(m, i, j, k, qr, c.name)
+				}
+			}
+		}
+	}
+}
+
+var testHeader = `const (
+	Yes = iota
+	No
+	Maybe
+)
+
+type formData struct {
+	qc              uint8
+	combinesForward bool
+	decomposition   string
+}
+
+type runeData struct {
+	r      rune
+	ccc    uint8
+	nLead  uint8
+	nTrail uint8
+	f      [2]formData // 0: canonical; 1: compatibility
+}
+
+func f(qc uint8, cf bool, dec string) [2]formData {
+	return [2]formData{{qc, cf, dec}, {qc, cf, dec}}
+}
+
+func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData {
+	return [2]formData{{qc, cf, d}, {qck, cfk, dk}}
+}
+
+var testData = []runeData{
+`
+
+func printTestdata() {
+	type lastInfo struct {
+		ccc    uint8
+		nLead  uint8
+		nTrail uint8
+		f      string
+	}
+
+	last := lastInfo{}
+	w := &bytes.Buffer{}
+	fmt.Fprintf(w, testHeader)
+	for r, c := range chars {
+		f := c.forms[FCanonical]
+		qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
+		f = c.forms[FCompatibility]
+		qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp)
+		s := ""
+		if d == dk && qc == qck && cf == cfk {
+			s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d)
+		} else {
+			s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk)
+		}
+		current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s}
+		if last != current {
+			fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s)
+			last = current
+		}
+	}
+	fmt.Fprintln(w, "}")
+	gen.WriteVersionedGoFile("data_test.go", "norm", w.Bytes())
+}

vendor/golang.org/x/text/unicode/norm/normalize.go

@@ -0,0 +1,609 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Note: the file data_test.go that is generated should not be checked in.
+//go:generate go run maketables.go triegen.go
+//go:generate go test -tags test
+
+// Package norm contains types and functions for normalizing Unicode strings.
+package norm // import "golang.org/x/text/unicode/norm"
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// A Form denotes a canonical representation of Unicode code points.
+// The Unicode-defined normalization and equivalence forms are:
+//
+//   NFC   Unicode Normalization Form C
+//   NFD   Unicode Normalization Form D
+//   NFKC  Unicode Normalization Form KC
+//   NFKD  Unicode Normalization Form KD
+//
+// For a Form f, this documentation uses the notation f(x) to mean
+// the bytes or string x converted to the given form.
+// A position n in x is called a boundary if conversion to the form can
+// proceed independently on both sides:
+//   f(x) == append(f(x[0:n]), f(x[n:])...)
+//
+// References: https://unicode.org/reports/tr15/ and
+// https://unicode.org/notes/tn5/.
+type Form int
+
+const (
+	NFC Form = iota
+	NFD
+	NFKC
+	NFKD
+)
+
+// Bytes returns f(b). May return b if f(b) = b.
+func (f Form) Bytes(b []byte) []byte {
+	src := inputBytes(b)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return b
+	}
+	out := make([]byte, n, len(b))
+	copy(out, b[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b), out: out, flushF: appendFlush}
+	return doAppendInner(&rb, n)
+}
+
+// String returns f(s).
+func (f Form) String(s string) string {
+	src := inputString(s)
+	ft := formTable[f]
+	n, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return s
+	}
+	out := make([]byte, n, len(s))
+	copy(out, s[0:n])
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s), out: out, flushF: appendFlush}
+	return string(doAppendInner(&rb, n))
+}
+
+// IsNormal returns true if b == f(b).
+func (f Form) IsNormal(b []byte) bool {
+	src := inputBytes(b)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(b), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(b)}
+	rb.setFlusher(nil, cmpNormalBytes)
+	for bp < len(b) {
+		rb.out = b[bp:]
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(b), true)
+	}
+	return true
+}
+
+func cmpNormalBytes(rb *reorderBuffer) bool {
+	b := rb.out
+	for i := 0; i < rb.nrune; i++ {
+		info := rb.rune[i]
+		if int(info.size) > len(b) {
+			return false
+		}
+		p := info.pos
+		pe := p + info.size
+		for ; p < pe; p++ {
+			if b[0] != rb.byte[p] {
+				return false
+			}
+			b = b[1:]
+		}
+	}
+	return true
+}
+
+// IsNormalString returns true if s == f(s).
+func (f Form) IsNormalString(s string) bool {
+	src := inputString(s)
+	ft := formTable[f]
+	bp, ok := ft.quickSpan(src, 0, len(s), true)
+	if ok {
+		return true
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: len(s)}
+	rb.setFlusher(nil, func(rb *reorderBuffer) bool {
+		for i := 0; i < rb.nrune; i++ {
+			info := rb.rune[i]
+			if bp+int(info.size) > len(s) {
+				return false
+			}
+			p := info.pos
+			pe := p + info.size
+			for ; p < pe; p++ {
+				if s[bp] != rb.byte[p] {
+					return false
+				}
+				bp++
+			}
+		}
+		return true
+	})
+	for bp < len(s) {
+		if bp = decomposeSegment(&rb, bp, true); bp < 0 {
+			return false
+		}
+		bp, _ = rb.f.quickSpan(rb.src, bp, len(s), true)
+	}
+	return true
+}
+
+// patchTail fixes a case where a rune may be incorrectly normalized
+// if it is followed by illegal continuation bytes. It returns the
+// patched buffer and whether the decomposition is still in progress.
+func patchTail(rb *reorderBuffer) bool {
+	info, p := lastRuneStart(&rb.f, rb.out)
+	if p == -1 || info.size == 0 {
+		return true
+	}
+	end := p + int(info.size)
+	extra := len(rb.out) - end
+	if extra > 0 {
+		// Potentially allocating memory. However, this only
+		// happens with ill-formed UTF-8.
+		x := make([]byte, 0)
+		x = append(x, rb.out[len(rb.out)-extra:]...)
+		rb.out = rb.out[:end]
+		decomposeToLastBoundary(rb)
+		rb.doFlush()
+		rb.out = append(rb.out, x...)
+		return false
+	}
+	buf := rb.out[p:]
+	rb.out = rb.out[:p]
+	decomposeToLastBoundary(rb)
+	if s := rb.ss.next(info); s == ssStarter {
+		rb.doFlush()
+		rb.ss.first(info)
+	} else if s == ssOverflow {
+		rb.doFlush()
+		rb.insertCGJ()
+		rb.ss = 0
+	}
+	rb.insertUnsafe(inputBytes(buf), 0, info)
+	return true
+}
+
+func appendQuick(rb *reorderBuffer, i int) int {
+	if rb.nsrc == i {
+		return i
+	}
+	end, _ := rb.f.quickSpan(rb.src, i, rb.nsrc, true)
+	rb.out = rb.src.appendSlice(rb.out, i, end)
+	return end
+}
+
+// Append returns f(append(out, b...)).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) Append(out []byte, src ...byte) []byte {
+	return f.doAppend(out, inputBytes(src), len(src))
+}
+
+func (f Form) doAppend(out []byte, src input, n int) []byte {
+	if n == 0 {
+		return out
+	}
+	ft := formTable[f]
+	// Attempt to do a quickSpan first so we can avoid initializing the reorderBuffer.
+	if len(out) == 0 {
+		p, _ := ft.quickSpan(src, 0, n, true)
+		out = src.appendSlice(out, 0, p)
+		if p == n {
+			return out
+		}
+		rb := reorderBuffer{f: *ft, src: src, nsrc: n, out: out, flushF: appendFlush}
+		return doAppendInner(&rb, p)
+	}
+	rb := reorderBuffer{f: *ft, src: src, nsrc: n}
+	return doAppend(&rb, out, 0)
+}
+
+func doAppend(rb *reorderBuffer, out []byte, p int) []byte {
+	rb.setFlusher(out, appendFlush)
+	src, n := rb.src, rb.nsrc
+	doMerge := len(out) > 0
+	if q := src.skipContinuationBytes(p); q > p {
+		// Move leading non-starters to destination.
+		rb.out = src.appendSlice(rb.out, p, q)
+		p = q
+		doMerge = patchTail(rb)
+	}
+	fd := &rb.f
+	if doMerge {
+		var info Properties
+		if p < n {
+			info = fd.info(src, p)
+			if !info.BoundaryBefore() || info.nLeadingNonStarters() > 0 {
+				if p == 0 {
+					decomposeToLastBoundary(rb)
+				}
+				p = decomposeSegment(rb, p, true)
+			}
+		}
+		if info.size == 0 {
+			rb.doFlush()
+			// Append incomplete UTF-8 encoding.
+			return src.appendSlice(rb.out, p, n)
+		}
+		if rb.nrune > 0 {
+			return doAppendInner(rb, p)
+		}
+	}
+	p = appendQuick(rb, p)
+	return doAppendInner(rb, p)
+}
+
+func doAppendInner(rb *reorderBuffer, p int) []byte {
+	for n := rb.nsrc; p < n; {
+		p = decomposeSegment(rb, p, true)
+		p = appendQuick(rb, p)
+	}
+	return rb.out
+}
+
+// AppendString returns f(append(out, []byte(s))).
+// The buffer out must be nil, empty, or equal to f(out).
+func (f Form) AppendString(out []byte, src string) []byte {
+	return f.doAppend(out, inputString(src), len(src))
+}
+
+// QuickSpan returns a boundary n such that b[0:n] == f(b[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpan(b []byte) int {
+	n, _ := formTable[f].quickSpan(inputBytes(b), 0, len(b), true)
+	return n
+}
+
+// Span implements transform.SpanningTransformer. It returns a boundary n such
+// that b[0:n] == f(b[0:n]). It is not guaranteed to return the largest such n.
+func (f Form) Span(b []byte, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), atEOF)
+	if n < len(b) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// SpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) SpanString(s string, atEOF bool) (n int, err error) {
+	n, ok := formTable[f].quickSpan(inputString(s), 0, len(s), atEOF)
+	if n < len(s) {
+		if !ok {
+			err = transform.ErrEndOfSpan
+		} else {
+			err = transform.ErrShortSrc
+		}
+	}
+	return n, err
+}
+
+// quickSpan returns a boundary n such that src[0:n] == f(src[0:n]) and
+// whether any non-normalized parts were found. If atEOF is false, n will
+// not point past the last segment if this segment might be become
+// non-normalized by appending other runes.
+func (f *formInfo) quickSpan(src input, i, end int, atEOF bool) (n int, ok bool) {
+	var lastCC uint8
+	ss := streamSafe(0)
+	lastSegStart := i
+	for n = end; i < n; {
+		if j := src.skipASCII(i, n); i != j {
+			i = j
+			lastSegStart = i - 1
+			lastCC = 0
+			ss = 0
+			continue
+		}
+		info := f.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				// include incomplete runes
+				return n, true
+			}
+			return lastSegStart, true
+		}
+		// This block needs to be before the next, because it is possible to
+		// have an overflow for runes that are starters (e.g. with U+FF9E).
+		switch ss.next(info) {
+		case ssStarter:
+			lastSegStart = i
+		case ssOverflow:
+			return lastSegStart, false
+		case ssSuccess:
+			if lastCC > info.ccc {
+				return lastSegStart, false
+			}
+		}
+		if f.composing {
+			if !info.isYesC() {
+				break
+			}
+		} else {
+			if !info.isYesD() {
+				break
+			}
+		}
+		lastCC = info.ccc
+		i += int(info.size)
+	}
+	if i == n {
+		if !atEOF {
+			n = lastSegStart
+		}
+		return n, true
+	}
+	return lastSegStart, false
+}
+
+// QuickSpanString returns a boundary n such that s[0:n] == f(s[0:n]).
+// It is not guaranteed to return the largest such n.
+func (f Form) QuickSpanString(s string) int {
+	n, _ := formTable[f].quickSpan(inputString(s), 0, len(s), true)
+	return n
+}
+
+// FirstBoundary returns the position i of the first boundary in b
+// or -1 if b contains no boundary.
+func (f Form) FirstBoundary(b []byte) int {
+	return f.firstBoundary(inputBytes(b), len(b))
+}
+
+func (f Form) firstBoundary(src input, nsrc int) int {
+	i := src.skipContinuationBytes(0)
+	if i >= nsrc {
+		return -1
+	}
+	fd := formTable[f]
+	ss := streamSafe(0)
+	// We should call ss.first here, but we can't as the first rune is
+	// skipped already. This means FirstBoundary can't really determine
+	// CGJ insertion points correctly. Luckily it doesn't have to.
+	for {
+		info := fd.info(src, i)
+		if info.size == 0 {
+			return -1
+		}
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+		i += int(info.size)
+		if i >= nsrc {
+			if !info.BoundaryAfter() && !ss.isMax() {
+				return -1
+			}
+			return nsrc
+		}
+	}
+}
+
+// FirstBoundaryInString returns the position i of the first boundary in s
+// or -1 if s contains no boundary.
+func (f Form) FirstBoundaryInString(s string) int {
+	return f.firstBoundary(inputString(s), len(s))
+}
+
+// NextBoundary reports the index of the boundary between the first and next
+// segment in b or -1 if atEOF is false and there are not enough bytes to
+// determine this boundary.
+func (f Form) NextBoundary(b []byte, atEOF bool) int {
+	return f.nextBoundary(inputBytes(b), len(b), atEOF)
+}
+
+// NextBoundaryInString reports the index of the boundary between the first and
+// next segment in b or -1 if atEOF is false and there are not enough bytes to
+// determine this boundary.
+func (f Form) NextBoundaryInString(s string, atEOF bool) int {
+	return f.nextBoundary(inputString(s), len(s), atEOF)
+}
+
+func (f Form) nextBoundary(src input, nsrc int, atEOF bool) int {
+	if nsrc == 0 {
+		if atEOF {
+			return 0
+		}
+		return -1
+	}
+	fd := formTable[f]
+	info := fd.info(src, 0)
+	if info.size == 0 {
+		if atEOF {
+			return 1
+		}
+		return -1
+	}
+	ss := streamSafe(0)
+	ss.first(info)
+
+	for i := int(info.size); i < nsrc; i += int(info.size) {
+		info = fd.info(src, i)
+		if info.size == 0 {
+			if atEOF {
+				return i
+			}
+			return -1
+		}
+		// TODO: Using streamSafe to determine the boundary isn't the same as
+		// using BoundaryBefore. Determine which should be used.
+		if s := ss.next(info); s != ssSuccess {
+			return i
+		}
+	}
+	if !atEOF && !info.BoundaryAfter() && !ss.isMax() {
+		return -1
+	}
+	return nsrc
+}
+
+// LastBoundary returns the position i of the last boundary in b
+// or -1 if b contains no boundary.
+func (f Form) LastBoundary(b []byte) int {
+	return lastBoundary(formTable[f], b)
+}
+
+func lastBoundary(fd *formInfo, b []byte) int {
+	i := len(b)
+	info, p := lastRuneStart(fd, b)
+	if p == -1 {
+		return -1
+	}
+	if info.size == 0 { // ends with incomplete rune
+		if p == 0 { // starts with incomplete rune
+			return -1
+		}
+		i = p
+		info, p = lastRuneStart(fd, b[:i])
+		if p == -1 { // incomplete UTF-8 encoding or non-starter bytes without a starter
+			return i
+		}
+	}
+	if p+int(info.size) != i { // trailing non-starter bytes: illegal UTF-8
+		return i
+	}
+	if info.BoundaryAfter() {
+		return i
+	}
+	ss := streamSafe(0)
+	v := ss.backwards(info)
+	for i = p; i >= 0 && v != ssStarter; i = p {
+		info, p = lastRuneStart(fd, b[:i])
+		if v = ss.backwards(info); v == ssOverflow {
+			break
+		}
+		if p+int(info.size) != i {
+			if p == -1 { // no boundary found
+				return -1
+			}
+			return i // boundary after an illegal UTF-8 encoding
+		}
+	}
+	return i
+}
+
+// decomposeSegment scans the first segment in src into rb. It inserts 0x034f
+// (Grapheme Joiner) when it encounters a sequence of more than 30 non-starters
+// and returns the number of bytes consumed from src or iShortDst or iShortSrc.
+func decomposeSegment(rb *reorderBuffer, sp int, atEOF bool) int {
+	// Force one character to be consumed.
+	info := rb.f.info(rb.src, sp)
+	if info.size == 0 {
+		return 0
+	}
+	if s := rb.ss.next(info); s == ssStarter {
+		// TODO: this could be removed if we don't support merging.
+		if rb.nrune > 0 {
+			goto end
+		}
+	} else if s == ssOverflow {
+		rb.insertCGJ()
+		goto end
+	}
+	if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+		return int(err)
+	}
+	for {
+		sp += int(info.size)
+		if sp >= rb.nsrc {
+			if !atEOF && !info.BoundaryAfter() {
+				return int(iShortSrc)
+			}
+			break
+		}
+		info = rb.f.info(rb.src, sp)
+		if info.size == 0 {
+			if !atEOF {
+				return int(iShortSrc)
+			}
+			break
+		}
+		if s := rb.ss.next(info); s == ssStarter {
+			break
+		} else if s == ssOverflow {
+			rb.insertCGJ()
+			break
+		}
+		if err := rb.insertFlush(rb.src, sp, info); err != iSuccess {
+			return int(err)
+		}
+	}
+end:
+	if !rb.doFlush() {
+		return int(iShortDst)
+	}
+	return sp
+}
+
+// lastRuneStart returns the runeInfo and position of the last
+// rune in buf or the zero runeInfo and -1 if no rune was found.
+func lastRuneStart(fd *formInfo, buf []byte) (Properties, int) {
+	p := len(buf) - 1
+	for ; p >= 0 && !utf8.RuneStart(buf[p]); p-- {
+	}
+	if p < 0 {
+		return Properties{}, -1
+	}
+	return fd.info(inputBytes(buf), p), p
+}
+
+// decomposeToLastBoundary finds an open segment at the end of the buffer
+// and scans it into rb. Returns the buffer minus the last segment.
+func decomposeToLastBoundary(rb *reorderBuffer) {
+	fd := &rb.f
+	info, i := lastRuneStart(fd, rb.out)
+	if int(info.size) != len(rb.out)-i {
+		// illegal trailing continuation bytes
+		return
+	}
+	if info.BoundaryAfter() {
+		return
+	}
+	var add [maxNonStarters + 1]Properties // stores runeInfo in reverse order
+	padd := 0
+	ss := streamSafe(0)
+	p := len(rb.out)
+	for {
+		add[padd] = info
+		v := ss.backwards(info)
+		if v == ssOverflow {
+			// Note that if we have an overflow, it the string we are appending to
+			// is not correctly normalized. In this case the behavior is undefined.
+			break
+		}
+		padd++
+		p -= int(info.size)
+		if v == ssStarter || p < 0 {
+			break
+		}
+		info, i = lastRuneStart(fd, rb.out[:p])
+		if int(info.size) != p-i {
+			break
+		}
+	}
+	rb.ss = ss
+	// Copy bytes for insertion as we may need to overwrite rb.out.
+	var buf [maxBufferSize * utf8.UTFMax]byte
+	cp := buf[:copy(buf[:], rb.out[p:])]
+	rb.out = rb.out[:p]
+	for padd--; padd >= 0; padd-- {
+		info = add[padd]
+		rb.insertUnsafe(inputBytes(cp), 0, info)
+		cp = cp[info.size:]
+	}
+}

vendor/golang.org/x/text/unicode/norm/readwriter.go

@@ -0,0 +1,125 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import "io"
+
+type normWriter struct {
+	rb  reorderBuffer
+	w   io.Writer
+	buf []byte
+}
+
+// Write implements the standard write interface.  If the last characters are
+// not at a normalization boundary, the bytes will be buffered for the next
+// write. The remaining bytes will be written on close.
+func (w *normWriter) Write(data []byte) (n int, err error) {
+	// Process data in pieces to keep w.buf size bounded.
+	const chunk = 4000
+
+	for len(data) > 0 {
+		// Normalize into w.buf.
+		m := len(data)
+		if m > chunk {
+			m = chunk
+		}
+		w.rb.src = inputBytes(data[:m])
+		w.rb.nsrc = m
+		w.buf = doAppend(&w.rb, w.buf, 0)
+		data = data[m:]
+		n += m
+
+		// Write out complete prefix, save remainder.
+		// Note that lastBoundary looks back at most 31 runes.
+		i := lastBoundary(&w.rb.f, w.buf)
+		if i == -1 {
+			i = 0
+		}
+		if i > 0 {
+			if _, err = w.w.Write(w.buf[:i]); err != nil {
+				break
+			}
+			bn := copy(w.buf, w.buf[i:])
+			w.buf = w.buf[:bn]
+		}
+	}
+	return n, err
+}
+
+// Close forces data that remains in the buffer to be written.
+func (w *normWriter) Close() error {
+	if len(w.buf) > 0 {
+		_, err := w.w.Write(w.buf)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Writer returns a new writer that implements Write(b)
+// by writing f(b) to w. The returned writer may use an
+// internal buffer to maintain state across Write calls.
+// Calling its Close method writes any buffered data to w.
+func (f Form) Writer(w io.Writer) io.WriteCloser {
+	wr := &normWriter{rb: reorderBuffer{}, w: w}
+	wr.rb.init(f, nil)
+	return wr
+}
+
+type normReader struct {
+	rb           reorderBuffer
+	r            io.Reader
+	inbuf        []byte
+	outbuf       []byte
+	bufStart     int
+	lastBoundary int
+	err          error
+}
+
+// Read implements the standard read interface.
+func (r *normReader) Read(p []byte) (int, error) {
+	for {
+		if r.lastBoundary-r.bufStart > 0 {
+			n := copy(p, r.outbuf[r.bufStart:r.lastBoundary])
+			r.bufStart += n
+			if r.lastBoundary-r.bufStart > 0 {
+				return n, nil
+			}
+			return n, r.err
+		}
+		if r.err != nil {
+			return 0, r.err
+		}
+		outn := copy(r.outbuf, r.outbuf[r.lastBoundary:])
+		r.outbuf = r.outbuf[0:outn]
+		r.bufStart = 0
+
+		n, err := r.r.Read(r.inbuf)
+		r.rb.src = inputBytes(r.inbuf[0:n])
+		r.rb.nsrc, r.err = n, err
+		if n > 0 {
+			r.outbuf = doAppend(&r.rb, r.outbuf, 0)
+		}
+		if err == io.EOF {
+			r.lastBoundary = len(r.outbuf)
+		} else {
+			r.lastBoundary = lastBoundary(&r.rb.f, r.outbuf)
+			if r.lastBoundary == -1 {
+				r.lastBoundary = 0
+			}
+		}
+	}
+}
+
+// Reader returns a new reader that implements Read
+// by reading data from r and returning f(data).
+func (f Form) Reader(r io.Reader) io.Reader {
+	const chunk = 4000
+	buf := make([]byte, chunk)
+	rr := &normReader{rb: reorderBuffer{}, r: r, inbuf: buf}
+	rr.rb.init(f, buf)
+	return rr
+}

vendor/golang.org/x/text/unicode/norm/transform.go

@@ -0,0 +1,88 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+import (
+	"unicode/utf8"
+
+	"golang.org/x/text/transform"
+)
+
+// Reset implements the Reset method of the transform.Transformer interface.
+func (Form) Reset() {}
+
+// Transform implements the Transform method of the transform.Transformer
+// interface. It may need to write segments of up to MaxSegmentSize at once.
+// Users should either catch ErrShortDst and allow dst to grow or have dst be at
+// least of size MaxTransformChunkSize to be guaranteed of progress.
+func (f Form) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// Cap the maximum number of src bytes to check.
+	b := src
+	eof := atEOF
+	if ns := len(dst); ns < len(b) {
+		err = transform.ErrShortDst
+		eof = false
+		b = b[:ns]
+	}
+	i, ok := formTable[f].quickSpan(inputBytes(b), 0, len(b), eof)
+	n := copy(dst, b[:i])
+	if !ok {
+		nDst, nSrc, err = f.transform(dst[n:], src[n:], atEOF)
+		return nDst + n, nSrc + n, err
+	}
+
+	if err == nil && n < len(src) && !atEOF {
+		err = transform.ErrShortSrc
+	}
+	return n, n, err
+}
+
+func flushTransform(rb *reorderBuffer) bool {
+	// Write out (must fully fit in dst, or else it is an ErrShortDst).
+	if len(rb.out) < rb.nrune*utf8.UTFMax {
+		return false
+	}
+	rb.out = rb.out[rb.flushCopy(rb.out):]
+	return true
+}
+
+var errs = []error{nil, transform.ErrShortDst, transform.ErrShortSrc}
+
+// transform implements the transform.Transformer interface. It is only called
+// when quickSpan does not pass for a given string.
+func (f Form) transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
+	// TODO: get rid of reorderBuffer. See CL 23460044.
+	rb := reorderBuffer{}
+	rb.init(f, src)
+	for {
+		// Load segment into reorder buffer.
+		rb.setFlusher(dst[nDst:], flushTransform)
+		end := decomposeSegment(&rb, nSrc, atEOF)
+		if end < 0 {
+			return nDst, nSrc, errs[-end]
+		}
+		nDst = len(dst) - len(rb.out)
+		nSrc = end
+
+		// Next quickSpan.
+		end = rb.nsrc
+		eof := atEOF
+		if n := nSrc + len(dst) - nDst; n < end {
+			err = transform.ErrShortDst
+			end = n
+			eof = false
+		}
+		end, ok := rb.f.quickSpan(rb.src, nSrc, end, eof)
+		n := copy(dst[nDst:], rb.src.bytes[nSrc:end])
+		nSrc += n
+		nDst += n
+		if ok {
+			if err == nil && n < rb.nsrc && !atEOF {
+				err = transform.ErrShortSrc
+			}
+			return nDst, nSrc, err
+		}
+	}
+}

vendor/golang.org/x/text/unicode/norm/trie.go

@@ -0,0 +1,54 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package norm
+
+type valueRange struct {
+	value  uint16 // header: value:stride
+	lo, hi byte   // header: lo:n
+}
+
+type sparseBlocks struct {
+	values []valueRange
+	offset []uint16
+}
+
+var nfcSparse = sparseBlocks{
+	values: nfcSparseValues[:],
+	offset: nfcSparseOffset[:],
+}
+
+var nfkcSparse = sparseBlocks{
+	values: nfkcSparseValues[:],
+	offset: nfkcSparseOffset[:],
+}
+
+var (
+	nfcData  = newNfcTrie(0)
+	nfkcData = newNfkcTrie(0)
+)
+
+// lookupValue determines the type of block n and looks up the value for b.
+// For n < t.cutoff, the block is a simple lookup table. Otherwise, the block
+// is a list of ranges with an accompanying value. Given a matching range r,
+// the value for b is by r.value + (b - r.lo) * stride.
+func (t *sparseBlocks) lookup(n uint32, b byte) uint16 {
+	offset := t.offset[n]
+	header := t.values[offset]
+	lo := offset + 1
+	hi := lo + uint16(header.lo)
+	for lo < hi {
+		m := lo + (hi-lo)/2
+		r := t.values[m]
+		if r.lo <= b && b <= r.hi {
+			return r.value + uint16(b-r.lo)*header.value
+		}
+		if b < r.lo {
+			hi = m
+		} else {
+			lo = m + 1
+		}
+	}
+	return 0
+}

vendor/golang.org/x/text/unicode/norm/triegen.go

@@ -0,0 +1,117 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Trie table generator.
+// Used by make*tables tools to generate a go file with trie data structures
+// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte
+// sequence are used to lookup offsets in the index table to be used for the
+// next byte. The last byte is used to index into a table with 16-bit values.
+
+package main
+
+import (
+	"fmt"
+	"io"
+)
+
+const maxSparseEntries = 16
+
+type normCompacter struct {
+	sparseBlocks [][]uint64
+	sparseOffset []uint16
+	sparseCount  int
+	name         string
+}
+
+func mostFrequentStride(a []uint64) int {
+	counts := make(map[int]int)
+	var v int
+	for _, x := range a {
+		if stride := int(x) - v; v != 0 && stride >= 0 {
+			counts[stride]++
+		}
+		v = int(x)
+	}
+	var maxs, maxc int
+	for stride, cnt := range counts {
+		if cnt > maxc || (cnt == maxc && stride < maxs) {
+			maxs, maxc = stride, cnt
+		}
+	}
+	return maxs
+}
+
+func countSparseEntries(a []uint64) int {
+	stride := mostFrequentStride(a)
+	var v, count int
+	for _, tv := range a {
+		if int(tv)-v != stride {
+			if tv != 0 {
+				count++
+			}
+		}
+		v = int(tv)
+	}
+	return count
+}
+
+func (c *normCompacter) Size(v []uint64) (sz int, ok bool) {
+	if n := countSparseEntries(v); n <= maxSparseEntries {
+		return (n+1)*4 + 2, true
+	}
+	return 0, false
+}
+
+func (c *normCompacter) Store(v []uint64) uint32 {
+	h := uint32(len(c.sparseOffset))
+	c.sparseBlocks = append(c.sparseBlocks, v)
+	c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount))
+	c.sparseCount += countSparseEntries(v) + 1
+	return h
+}
+
+func (c *normCompacter) Handler() string {
+	return c.name + "Sparse.lookup"
+}
+
+func (c *normCompacter) Print(w io.Writer) (retErr error) {
+	p := func(f string, x ...interface{}) {
+		if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil {
+			retErr = err
+		}
+	}
+
+	ls := len(c.sparseBlocks)
+	p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2)
+	p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset)
+
+	ns := c.sparseCount
+	p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4)
+	p("var %sSparseValues = [%d]valueRange {", c.name, ns)
+	for i, b := range c.sparseBlocks {
+		p("\n// Block %#x, offset %#x", i, c.sparseOffset[i])
+		var v int
+		stride := mostFrequentStride(b)
+		n := countSparseEntries(b)
+		p("\n{value:%#04x,lo:%#02x},", stride, uint8(n))
+		for i, nv := range b {
+			if int(nv)-v != stride {
+				if v != 0 {
+					p(",hi:%#02x},", 0x80+i-1)
+				}
+				if nv != 0 {
+					p("\n{value:%#04x,lo:%#02x", nv, 0x80+i)
+				}
+			}
+			v = int(nv)
+		}
+		if v != 0 {
+			p(",hi:%#02x},", 0x80+len(b)-1)
+		}
+	}
+	p("\n}\n\n")
+	return
+}

vendor/golang.org/x/text/unicode/rangetable/gen.go

@@ -0,0 +1,115 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"reflect"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/internal/ucd"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/rangetable"
+)
+
+var versionList = flag.String("versions", "",
+	"list of versions for which to generate RangeTables")
+
+const bootstrapMessage = `No versions specified.
+To bootstrap the code generation, run:
+	go run gen.go --versions=4.1.0,5.0.0,6.0.0,6.1.0,6.2.0,6.3.0,7.0.0
+
+and ensure that the latest versions are included by checking:
+	https://www.unicode.org/Public/`
+
+func getVersions() []string {
+	if *versionList == "" {
+		log.Fatal(bootstrapMessage)
+	}
+
+	c := collate.New(language.Und, collate.Numeric)
+	versions := strings.Split(*versionList, ",")
+	c.SortStrings(versions)
+
+	// Ensure that at least the current version is included.
+	for _, v := range versions {
+		if v == gen.UnicodeVersion() {
+			return versions
+		}
+	}
+
+	versions = append(versions, gen.UnicodeVersion())
+	c.SortStrings(versions)
+	return versions
+}
+
+func main() {
+	gen.Init()
+
+	versions := getVersions()
+
+	w := &bytes.Buffer{}
+
+	fmt.Fprintf(w, "//go:generate go run gen.go --versions=%s\n\n", strings.Join(versions, ","))
+	fmt.Fprintf(w, "import \"unicode\"\n\n")
+
+	vstr := func(s string) string { return strings.Replace(s, ".", "_", -1) }
+
+	fmt.Fprintf(w, "var assigned = map[string]*unicode.RangeTable{\n")
+	for _, v := range versions {
+		fmt.Fprintf(w, "\t%q: assigned%s,\n", v, vstr(v))
+	}
+	fmt.Fprintf(w, "}\n\n")
+
+	var size int
+	for _, v := range versions {
+		assigned := []rune{}
+
+		r := gen.Open("https://www.unicode.org/Public/", "", v+"/ucd/UnicodeData.txt")
+		ucd.Parse(r, func(p *ucd.Parser) {
+			assigned = append(assigned, p.Rune(0))
+		})
+
+		rt := rangetable.New(assigned...)
+		sz := int(reflect.TypeOf(unicode.RangeTable{}).Size())
+		sz += int(reflect.TypeOf(unicode.Range16{}).Size()) * len(rt.R16)
+		sz += int(reflect.TypeOf(unicode.Range32{}).Size()) * len(rt.R32)
+
+		fmt.Fprintf(w, "// size %d bytes (%d KiB)\n", sz, sz/1024)
+		fmt.Fprintf(w, "var assigned%s = ", vstr(v))
+		print(w, rt)
+
+		size += sz
+	}
+
+	fmt.Fprintf(w, "// Total size %d bytes (%d KiB)\n", size, size/1024)
+
+	gen.WriteVersionedGoFile("tables.go", "rangetable", w.Bytes())
+}
+
+func print(w io.Writer, rt *unicode.RangeTable) {
+	fmt.Fprintln(w, "&unicode.RangeTable{")
+	fmt.Fprintln(w, "\tR16: []unicode.Range16{")
+	for _, r := range rt.R16 {
+		fmt.Fprintf(w, "\t\t{%#04x, %#04x, %d},\n", r.Lo, r.Hi, r.Stride)
+	}
+	fmt.Fprintln(w, "\t},")
+	fmt.Fprintln(w, "\tR32: []unicode.Range32{")
+	for _, r := range rt.R32 {
+		fmt.Fprintf(w, "\t\t{%#08x, %#08x, %d},\n", r.Lo, r.Hi, r.Stride)
+	}
+	fmt.Fprintln(w, "\t},")
+	fmt.Fprintf(w, "\tLatinOffset: %d,\n", rt.LatinOffset)
+	fmt.Fprintf(w, "}\n\n")
+}

vendor/golang.org/x/text/unicode/rangetable/merge.go

@@ -0,0 +1,260 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package rangetable
+
+import (
+	"unicode"
+)
+
+// atEnd is used to mark a completed iteration.
+const atEnd = unicode.MaxRune + 1
+
+// Merge returns a new RangeTable that is the union of the given tables.
+// It can also be used to compact user-created RangeTables. The entries in
+// R16 and R32 for any given RangeTable should be sorted and non-overlapping.
+//
+// A lookup in the resulting table can be several times faster than using In
+// directly on the ranges. Merge is an expensive operation, however, and only
+// makes sense if one intends to use the result for more than a couple of
+// hundred lookups.
+func Merge(ranges ...*unicode.RangeTable) *unicode.RangeTable {
+	rt := &unicode.RangeTable{}
+	if len(ranges) == 0 {
+		return rt
+	}
+
+	iter := tablesIter(make([]tableIndex, len(ranges)))
+
+	for i, t := range ranges {
+		iter[i] = tableIndex{t, 0, atEnd}
+		if len(t.R16) > 0 {
+			iter[i].next = rune(t.R16[0].Lo)
+		}
+	}
+
+	if r0 := iter.next16(); r0.Stride != 0 {
+		for {
+			r1 := iter.next16()
+			if r1.Stride == 0 {
+				rt.R16 = append(rt.R16, r0)
+				break
+			}
+			stride := r1.Lo - r0.Hi
+			if (r1.Lo == r1.Hi || stride == r1.Stride) && (r0.Lo == r0.Hi || stride == r0.Stride) {
+				// Fully merge the next range into the previous one.
+				r0.Hi, r0.Stride = r1.Hi, stride
+				continue
+			} else if stride == r0.Stride {
+				// Move the first element of r1 to r0. This may eliminate an
+				// entry.
+				r0.Hi = r1.Lo
+				r0.Stride = stride
+				r1.Lo = r1.Lo + r1.Stride
+				if r1.Lo > r1.Hi {
+					continue
+				}
+			}
+			rt.R16 = append(rt.R16, r0)
+			r0 = r1
+		}
+	}
+
+	for i, t := range ranges {
+		iter[i] = tableIndex{t, 0, atEnd}
+		if len(t.R32) > 0 {
+			iter[i].next = rune(t.R32[0].Lo)
+		}
+	}
+
+	if r0 := iter.next32(); r0.Stride != 0 {
+		for {
+			r1 := iter.next32()
+			if r1.Stride == 0 {
+				rt.R32 = append(rt.R32, r0)
+				break
+			}
+			stride := r1.Lo - r0.Hi
+			if (r1.Lo == r1.Hi || stride == r1.Stride) && (r0.Lo == r0.Hi || stride == r0.Stride) {
+				// Fully merge the next range into the previous one.
+				r0.Hi, r0.Stride = r1.Hi, stride
+				continue
+			} else if stride == r0.Stride {
+				// Move the first element of r1 to r0. This may eliminate an
+				// entry.
+				r0.Hi = r1.Lo
+				r1.Lo = r1.Lo + r1.Stride
+				if r1.Lo > r1.Hi {
+					continue
+				}
+			}
+			rt.R32 = append(rt.R32, r0)
+			r0 = r1
+		}
+	}
+
+	for i := 0; i < len(rt.R16) && rt.R16[i].Hi <= unicode.MaxLatin1; i++ {
+		rt.LatinOffset = i + 1
+	}
+
+	return rt
+}
+
+type tableIndex struct {
+	t    *unicode.RangeTable
+	p    uint32
+	next rune
+}
+
+type tablesIter []tableIndex
+
+// sortIter does an insertion sort using the next field of tableIndex. Insertion
+// sort is a good sorting algorithm for this case.
+func sortIter(t []tableIndex) {
+	for i := range t {
+		for j := i; j > 0 && t[j-1].next > t[j].next; j-- {
+			t[j], t[j-1] = t[j-1], t[j]
+		}
+	}
+}
+
+// next16 finds the ranged to be added to the table. If ranges overlap between
+// multiple tables it clips the result to a non-overlapping range if the
+// elements are not fully subsumed. It returns a zero range if there are no more
+// ranges.
+func (ti tablesIter) next16() unicode.Range16 {
+	sortIter(ti)
+
+	t0 := ti[0]
+	if t0.next == atEnd {
+		return unicode.Range16{}
+	}
+	r0 := t0.t.R16[t0.p]
+	r0.Lo = uint16(t0.next)
+
+	// We restrict the Hi of the current range if it overlaps with another range.
+	for i := range ti {
+		tn := ti[i]
+		// Since our tableIndices are sorted by next, we can break if the there
+		// is no overlap. The first value of a next range can always be merged
+		// into the current one, so we can break in case of equality as well.
+		if rune(r0.Hi) <= tn.next {
+			break
+		}
+		rn := tn.t.R16[tn.p]
+		rn.Lo = uint16(tn.next)
+
+		// Limit r0.Hi based on next ranges in list, but allow it to overlap
+		// with ranges as long as it subsumes it.
+		m := (rn.Lo - r0.Lo) % r0.Stride
+		if m == 0 && (rn.Stride == r0.Stride || rn.Lo == rn.Hi) {
+			// Overlap, take the min of the two Hi values: for simplicity's sake
+			// we only process one range at a time.
+			if r0.Hi > rn.Hi {
+				r0.Hi = rn.Hi
+			}
+		} else {
+			// Not a compatible stride. Set to the last possible value before
+			// rn.Lo, but ensure there is at least one value.
+			if x := rn.Lo - m; r0.Lo <= x {
+				r0.Hi = x
+			}
+			break
+		}
+	}
+
+	// Update the next values for each table.
+	for i := range ti {
+		tn := &ti[i]
+		if rune(r0.Hi) < tn.next {
+			break
+		}
+		rn := tn.t.R16[tn.p]
+		stride := rune(rn.Stride)
+		tn.next += stride * (1 + ((rune(r0.Hi) - tn.next) / stride))
+		if rune(rn.Hi) < tn.next {
+			if tn.p++; int(tn.p) == len(tn.t.R16) {
+				tn.next = atEnd
+			} else {
+				tn.next = rune(tn.t.R16[tn.p].Lo)
+			}
+		}
+	}
+
+	if r0.Lo == r0.Hi {
+		r0.Stride = 1
+	}
+
+	return r0
+}
+
+// next32 finds the ranged to be added to the table. If ranges overlap between
+// multiple tables it clips the result to a non-overlapping range if the
+// elements are not fully subsumed. It returns a zero range if there are no more
+// ranges.
+func (ti tablesIter) next32() unicode.Range32 {
+	sortIter(ti)
+
+	t0 := ti[0]
+	if t0.next == atEnd {
+		return unicode.Range32{}
+	}
+	r0 := t0.t.R32[t0.p]
+	r0.Lo = uint32(t0.next)
+
+	// We restrict the Hi of the current range if it overlaps with another range.
+	for i := range ti {
+		tn := ti[i]
+		// Since our tableIndices are sorted by next, we can break if the there
+		// is no overlap. The first value of a next range can always be merged
+		// into the current one, so we can break in case of equality as well.
+		if rune(r0.Hi) <= tn.next {
+			break
+		}
+		rn := tn.t.R32[tn.p]
+		rn.Lo = uint32(tn.next)
+
+		// Limit r0.Hi based on next ranges in list, but allow it to overlap
+		// with ranges as long as it subsumes it.
+		m := (rn.Lo - r0.Lo) % r0.Stride
+		if m == 0 && (rn.Stride == r0.Stride || rn.Lo == rn.Hi) {
+			// Overlap, take the min of the two Hi values: for simplicity's sake
+			// we only process one range at a time.
+			if r0.Hi > rn.Hi {
+				r0.Hi = rn.Hi
+			}
+		} else {
+			// Not a compatible stride. Set to the last possible value before
+			// rn.Lo, but ensure there is at least one value.
+			if x := rn.Lo - m; r0.Lo <= x {
+				r0.Hi = x
+			}
+			break
+		}
+	}
+
+	// Update the next values for each table.
+	for i := range ti {
+		tn := &ti[i]
+		if rune(r0.Hi) < tn.next {
+			break
+		}
+		rn := tn.t.R32[tn.p]
+		stride := rune(rn.Stride)
+		tn.next += stride * (1 + ((rune(r0.Hi) - tn.next) / stride))
+		if rune(rn.Hi) < tn.next {
+			if tn.p++; int(tn.p) == len(tn.t.R32) {
+				tn.next = atEnd
+			} else {
+				tn.next = rune(tn.t.R32[tn.p].Lo)
+			}
+		}
+	}
+
+	if r0.Lo == r0.Hi {
+		r0.Stride = 1
+	}
+
+	return r0
+}

vendor/golang.org/x/text/unicode/rangetable/rangetable.go

@@ -0,0 +1,70 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package rangetable provides utilities for creating and inspecting
+// unicode.RangeTables.
+package rangetable
+
+import (
+	"sort"
+	"unicode"
+)
+
+// New creates a RangeTable from the given runes, which may contain duplicates.
+func New(r ...rune) *unicode.RangeTable {
+	if len(r) == 0 {
+		return &unicode.RangeTable{}
+	}
+
+	sort.Sort(byRune(r))
+
+	// Remove duplicates.
+	k := 1
+	for i := 1; i < len(r); i++ {
+		if r[k-1] != r[i] {
+			r[k] = r[i]
+			k++
+		}
+	}
+
+	var rt unicode.RangeTable
+	for _, r := range r[:k] {
+		if r <= 0xFFFF {
+			rt.R16 = append(rt.R16, unicode.Range16{Lo: uint16(r), Hi: uint16(r), Stride: 1})
+		} else {
+			rt.R32 = append(rt.R32, unicode.Range32{Lo: uint32(r), Hi: uint32(r), Stride: 1})
+		}
+	}
+
+	// Optimize RangeTable.
+	return Merge(&rt)
+}
+
+type byRune []rune
+
+func (r byRune) Len() int           { return len(r) }
+func (r byRune) Swap(i, j int)      { r[i], r[j] = r[j], r[i] }
+func (r byRune) Less(i, j int) bool { return r[i] < r[j] }
+
+// Visit visits all runes in the given RangeTable in order, calling fn for each.
+func Visit(rt *unicode.RangeTable, fn func(rune)) {
+	for _, r16 := range rt.R16 {
+		for r := rune(r16.Lo); r <= rune(r16.Hi); r += rune(r16.Stride) {
+			fn(r)
+		}
+	}
+	for _, r32 := range rt.R32 {
+		for r := rune(r32.Lo); r <= rune(r32.Hi); r += rune(r32.Stride) {
+			fn(r)
+		}
+	}
+}
+
+// Assigned returns a RangeTable with all assigned code points for a given
+// Unicode version. This includes graphic, format, control, and private-use
+// characters. It returns nil if the data for the given version is not
+// available.
+func Assigned(version string) *unicode.RangeTable {
+	return assigned[version]
+}