Initial

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

vendor/golang.org/x/text/collate/build/builder.go

@@ -0,0 +1,702 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build // import "golang.org/x/text/collate/build"
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"sort"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/norm"
+)
+
+// TODO: optimizations:
+// - expandElem is currently 20K. By putting unique colElems in a separate
+//   table and having a byte array of indexes into this table, we can reduce
+//   the total size to about 7K. By also factoring out the length bytes, we
+//   can reduce this to about 6K.
+// - trie valueBlocks are currently 100K. There are a lot of sparse blocks
+//   and many consecutive values with the same stride. This can be further
+//   compacted.
+// - Compress secondary weights into 8 bits.
+// - Some LDML specs specify a context element. Currently we simply concatenate
+//   those.  Context can be implemented using the contraction trie. If Builder
+//   could analyze and detect when using a context makes sense, there is no
+//   need to expose this construct in the API.
+
+// A Builder builds a root collation table.  The user must specify the
+// collation elements for each entry.  A common use will be to base the weights
+// on those specified in the allkeys* file as provided by the UCA or CLDR.
+type Builder struct {
+	index  *trieBuilder
+	root   ordering
+	locale []*Tailoring
+	t      *table
+	err    error
+	built  bool
+
+	minNonVar int // lowest primary recorded for a variable
+	varTop    int // highest primary recorded for a non-variable
+
+	// indexes used for reusing expansions and contractions
+	expIndex map[string]int      // positions of expansions keyed by their string representation
+	ctHandle map[string]ctHandle // contraction handles keyed by a concatenation of the suffixes
+	ctElem   map[string]int      // contraction elements keyed by their string representation
+}
+
+// A Tailoring builds a collation table based on another collation table.
+// The table is defined by specifying tailorings to the underlying table.
+// See https://unicode.org/reports/tr35/ for an overview of tailoring
+// collation tables.  The CLDR contains pre-defined tailorings for a variety
+// of languages (See https://www.unicode.org/Public/cldr/<version>/core.zip.)
+type Tailoring struct {
+	id      string
+	builder *Builder
+	index   *ordering
+
+	anchor *entry
+	before bool
+}
+
+// NewBuilder returns a new Builder.
+func NewBuilder() *Builder {
+	return &Builder{
+		index:    newTrieBuilder(),
+		root:     makeRootOrdering(),
+		expIndex: make(map[string]int),
+		ctHandle: make(map[string]ctHandle),
+		ctElem:   make(map[string]int),
+	}
+}
+
+// Tailoring returns a Tailoring for the given locale.  One should
+// have completed all calls to Add before calling Tailoring.
+func (b *Builder) Tailoring(loc language.Tag) *Tailoring {
+	t := &Tailoring{
+		id:      loc.String(),
+		builder: b,
+		index:   b.root.clone(),
+	}
+	t.index.id = t.id
+	b.locale = append(b.locale, t)
+	return t
+}
+
+// Add adds an entry to the collation element table, mapping
+// a slice of runes to a sequence of collation elements.
+// A collation element is specified as list of weights: []int{primary, secondary, ...}.
+// The entries are typically obtained from a collation element table
+// as defined in https://www.unicode.org/reports/tr10/#Data_Table_Format.
+// Note that the collation elements specified by colelems are only used
+// as a guide.  The actual weights generated by Builder may differ.
+// The argument variables is a list of indices into colelems that should contain
+// a value for each colelem that is a variable. (See the reference above.)
+func (b *Builder) Add(runes []rune, colelems [][]int, variables []int) error {
+	str := string(runes)
+	elems := make([]rawCE, len(colelems))
+	for i, ce := range colelems {
+		if len(ce) == 0 {
+			break
+		}
+		elems[i] = makeRawCE(ce, 0)
+		if len(ce) == 1 {
+			elems[i].w[1] = defaultSecondary
+		}
+		if len(ce) <= 2 {
+			elems[i].w[2] = defaultTertiary
+		}
+		if len(ce) <= 3 {
+			elems[i].w[3] = ce[0]
+		}
+	}
+	for i, ce := range elems {
+		p := ce.w[0]
+		isvar := false
+		for _, j := range variables {
+			if i == j {
+				isvar = true
+			}
+		}
+		if isvar {
+			if p >= b.minNonVar && b.minNonVar > 0 {
+				return fmt.Errorf("primary value %X of variable is larger than the smallest non-variable %X", p, b.minNonVar)
+			}
+			if p > b.varTop {
+				b.varTop = p
+			}
+		} else if p > 1 { // 1 is a special primary value reserved for FFFE
+			if p <= b.varTop {
+				return fmt.Errorf("primary value %X of non-variable is smaller than the highest variable %X", p, b.varTop)
+			}
+			if b.minNonVar == 0 || p < b.minNonVar {
+				b.minNonVar = p
+			}
+		}
+	}
+	elems, err := convertLargeWeights(elems)
+	if err != nil {
+		return err
+	}
+	cccs := []uint8{}
+	nfd := norm.NFD.String(str)
+	for i := range nfd {
+		cccs = append(cccs, norm.NFD.PropertiesString(nfd[i:]).CCC())
+	}
+	if len(cccs) < len(elems) {
+		if len(cccs) > 2 {
+			return fmt.Errorf("number of decomposed characters should be greater or equal to the number of collation elements for len(colelems) > 3 (%d < %d)", len(cccs), len(elems))
+		}
+		p := len(elems) - 1
+		for ; p > 0 && elems[p].w[0] == 0; p-- {
+			elems[p].ccc = cccs[len(cccs)-1]
+		}
+		for ; p >= 0; p-- {
+			elems[p].ccc = cccs[0]
+		}
+	} else {
+		for i := range elems {
+			elems[i].ccc = cccs[i]
+		}
+	}
+	// doNorm in collate.go assumes that the following conditions hold.
+	if len(elems) > 1 && len(cccs) > 1 && cccs[0] != 0 && cccs[0] != cccs[len(cccs)-1] {
+		return fmt.Errorf("incompatible CCC values for expansion %X (%d)", runes, cccs)
+	}
+	b.root.newEntry(str, elems)
+	return nil
+}
+
+func (t *Tailoring) setAnchor(anchor string) error {
+	anchor = norm.NFC.String(anchor)
+	a := t.index.find(anchor)
+	if a == nil {
+		a = t.index.newEntry(anchor, nil)
+		a.implicit = true
+		a.modified = true
+		for _, r := range []rune(anchor) {
+			e := t.index.find(string(r))
+			e.lock = true
+		}
+	}
+	t.anchor = a
+	return nil
+}
+
+// SetAnchor sets the point after which elements passed in subsequent calls to
+// Insert will be inserted.  It is equivalent to the reset directive in an LDML
+// specification.  See Insert for an example.
+// SetAnchor supports the following logical reset positions:
+// <first_tertiary_ignorable/>, <last_teriary_ignorable/>, <first_primary_ignorable/>,
+// and <last_non_ignorable/>.
+func (t *Tailoring) SetAnchor(anchor string) error {
+	if err := t.setAnchor(anchor); err != nil {
+		return err
+	}
+	t.before = false
+	return nil
+}
+
+// SetAnchorBefore is similar to SetAnchor, except that subsequent calls to
+// Insert will insert entries before the anchor.
+func (t *Tailoring) SetAnchorBefore(anchor string) error {
+	if err := t.setAnchor(anchor); err != nil {
+		return err
+	}
+	t.before = true
+	return nil
+}
+
+// Insert sets the ordering of str relative to the entry set by the previous
+// call to SetAnchor or Insert.  The argument extend corresponds
+// to the extend elements as defined in LDML.  A non-empty value for extend
+// will cause the collation elements corresponding to extend to be appended
+// to the collation elements generated for the entry added by Insert.
+// This has the same net effect as sorting str after the string anchor+extend.
+// See https://www.unicode.org/reports/tr10/#Tailoring_Example for details
+// on parametric tailoring and https://unicode.org/reports/tr35/#Collation_Elements
+// for full details on LDML.
+//
+// Examples: create a tailoring for Swedish, where "ä" is ordered after "z"
+// at the primary sorting level:
+//      t := b.Tailoring("se")
+// 		t.SetAnchor("z")
+// 		t.Insert(colltab.Primary, "ä", "")
+// Order "ü" after "ue" at the secondary sorting level:
+//		t.SetAnchor("ue")
+//		t.Insert(colltab.Secondary, "ü","")
+// or
+//		t.SetAnchor("u")
+//		t.Insert(colltab.Secondary, "ü", "e")
+// Order "q" afer "ab" at the secondary level and "Q" after "q"
+// at the tertiary level:
+// 		t.SetAnchor("ab")
+// 		t.Insert(colltab.Secondary, "q", "")
+// 		t.Insert(colltab.Tertiary, "Q", "")
+// Order "b" before "a":
+//      t.SetAnchorBefore("a")
+//      t.Insert(colltab.Primary, "b", "")
+// Order "0" after the last primary ignorable:
+//      t.SetAnchor("<last_primary_ignorable/>")
+//      t.Insert(colltab.Primary, "0", "")
+func (t *Tailoring) Insert(level colltab.Level, str, extend string) error {
+	if t.anchor == nil {
+		return fmt.Errorf("%s:Insert: no anchor point set for tailoring of %s", t.id, str)
+	}
+	str = norm.NFC.String(str)
+	e := t.index.find(str)
+	if e == nil {
+		e = t.index.newEntry(str, nil)
+	} else if e.logical != noAnchor {
+		return fmt.Errorf("%s:Insert: cannot reinsert logical reset position %q", t.id, e.str)
+	}
+	if e.lock {
+		return fmt.Errorf("%s:Insert: cannot reinsert element %q", t.id, e.str)
+	}
+	a := t.anchor
+	// Find the first element after the anchor which differs at a level smaller or
+	// equal to the given level.  Then insert at this position.
+	// See https://unicode.org/reports/tr35/#Collation_Elements, Section 5.14.5 for details.
+	e.before = t.before
+	if t.before {
+		t.before = false
+		if a.prev == nil {
+			a.insertBefore(e)
+		} else {
+			for a = a.prev; a.level > level; a = a.prev {
+			}
+			a.insertAfter(e)
+		}
+		e.level = level
+	} else {
+		for ; a.level > level; a = a.next {
+		}
+		e.level = a.level
+		if a != e {
+			a.insertAfter(e)
+			a.level = level
+		} else {
+			// We don't set a to prev itself. This has the effect of the entry
+			// getting new collation elements that are an increment of itself.
+			// This is intentional.
+			a.prev.level = level
+		}
+	}
+	e.extend = norm.NFD.String(extend)
+	e.exclude = false
+	e.modified = true
+	e.elems = nil
+	t.anchor = e
+	return nil
+}
+
+func (o *ordering) getWeight(e *entry) []rawCE {
+	if len(e.elems) == 0 && e.logical == noAnchor {
+		if e.implicit {
+			for _, r := range e.runes {
+				e.elems = append(e.elems, o.getWeight(o.find(string(r)))...)
+			}
+		} else if e.before {
+			count := [colltab.Identity + 1]int{}
+			a := e
+			for ; a.elems == nil && !a.implicit; a = a.next {
+				count[a.level]++
+			}
+			e.elems = []rawCE{makeRawCE(a.elems[0].w, a.elems[0].ccc)}
+			for i := colltab.Primary; i < colltab.Quaternary; i++ {
+				if count[i] != 0 {
+					e.elems[0].w[i] -= count[i]
+					break
+				}
+			}
+			if e.prev != nil {
+				o.verifyWeights(e.prev, e, e.prev.level)
+			}
+		} else {
+			prev := e.prev
+			e.elems = nextWeight(prev.level, o.getWeight(prev))
+			o.verifyWeights(e, e.next, e.level)
+		}
+	}
+	return e.elems
+}
+
+func (o *ordering) addExtension(e *entry) {
+	if ex := o.find(e.extend); ex != nil {
+		e.elems = append(e.elems, ex.elems...)
+	} else {
+		for _, r := range []rune(e.extend) {
+			e.elems = append(e.elems, o.find(string(r)).elems...)
+		}
+	}
+	e.extend = ""
+}
+
+func (o *ordering) verifyWeights(a, b *entry, level colltab.Level) error {
+	if level == colltab.Identity || b == nil || b.elems == nil || a.elems == nil {
+		return nil
+	}
+	for i := colltab.Primary; i < level; i++ {
+		if a.elems[0].w[i] < b.elems[0].w[i] {
+			return nil
+		}
+	}
+	if a.elems[0].w[level] >= b.elems[0].w[level] {
+		err := fmt.Errorf("%s:overflow: collation elements of %q (%X) overflows those of %q (%X) at level %d (%X >= %X)", o.id, a.str, a.runes, b.str, b.runes, level, a.elems, b.elems)
+		log.Println(err)
+		// TODO: return the error instead, or better, fix the conflicting entry by making room.
+	}
+	return nil
+}
+
+func (b *Builder) error(e error) {
+	if e != nil {
+		b.err = e
+	}
+}
+
+func (b *Builder) errorID(locale string, e error) {
+	if e != nil {
+		b.err = fmt.Errorf("%s:%v", locale, e)
+	}
+}
+
+// patchNorm ensures that NFC and NFD counterparts are consistent.
+func (o *ordering) patchNorm() {
+	// Insert the NFD counterparts, if necessary.
+	for _, e := range o.ordered {
+		nfd := norm.NFD.String(e.str)
+		if nfd != e.str {
+			if e0 := o.find(nfd); e0 != nil && !e0.modified {
+				e0.elems = e.elems
+			} else if e.modified && !equalCEArrays(o.genColElems(nfd), e.elems) {
+				e := o.newEntry(nfd, e.elems)
+				e.modified = true
+			}
+		}
+	}
+	// Update unchanged composed forms if one of their parts changed.
+	for _, e := range o.ordered {
+		nfd := norm.NFD.String(e.str)
+		if e.modified || nfd == e.str {
+			continue
+		}
+		if e0 := o.find(nfd); e0 != nil {
+			e.elems = e0.elems
+		} else {
+			e.elems = o.genColElems(nfd)
+			if norm.NFD.LastBoundary([]byte(nfd)) == 0 {
+				r := []rune(nfd)
+				head := string(r[0])
+				tail := ""
+				for i := 1; i < len(r); i++ {
+					s := norm.NFC.String(head + string(r[i]))
+					if e0 := o.find(s); e0 != nil && e0.modified {
+						head = s
+					} else {
+						tail += string(r[i])
+					}
+				}
+				e.elems = append(o.genColElems(head), o.genColElems(tail)...)
+			}
+		}
+	}
+	// Exclude entries for which the individual runes generate the same collation elements.
+	for _, e := range o.ordered {
+		if len(e.runes) > 1 && equalCEArrays(o.genColElems(e.str), e.elems) {
+			e.exclude = true
+		}
+	}
+}
+
+func (b *Builder) buildOrdering(o *ordering) {
+	for _, e := range o.ordered {
+		o.getWeight(e)
+	}
+	for _, e := range o.ordered {
+		o.addExtension(e)
+	}
+	o.patchNorm()
+	o.sort()
+	simplify(o)
+	b.processExpansions(o)   // requires simplify
+	b.processContractions(o) // requires simplify
+
+	t := newNode()
+	for e := o.front(); e != nil; e, _ = e.nextIndexed() {
+		if !e.skip() {
+			ce, err := e.encode()
+			b.errorID(o.id, err)
+			t.insert(e.runes[0], ce)
+		}
+	}
+	o.handle = b.index.addTrie(t)
+}
+
+func (b *Builder) build() (*table, error) {
+	if b.built {
+		return b.t, b.err
+	}
+	b.built = true
+	b.t = &table{
+		Table: colltab.Table{
+			MaxContractLen: utf8.UTFMax,
+			VariableTop:    uint32(b.varTop),
+		},
+	}
+
+	b.buildOrdering(&b.root)
+	b.t.root = b.root.handle
+	for _, t := range b.locale {
+		b.buildOrdering(t.index)
+		if b.err != nil {
+			break
+		}
+	}
+	i, err := b.index.generate()
+	b.t.trie = *i
+	b.t.Index = colltab.Trie{
+		Index:   i.index,
+		Values:  i.values,
+		Index0:  i.index[blockSize*b.t.root.lookupStart:],
+		Values0: i.values[blockSize*b.t.root.valueStart:],
+	}
+	b.error(err)
+	return b.t, b.err
+}
+
+// Build builds the root Collator.
+func (b *Builder) Build() (colltab.Weighter, error) {
+	table, err := b.build()
+	if err != nil {
+		return nil, err
+	}
+	return table, nil
+}
+
+// Build builds a Collator for Tailoring t.
+func (t *Tailoring) Build() (colltab.Weighter, error) {
+	// TODO: implement.
+	return nil, nil
+}
+
+// Print prints the tables for b and all its Tailorings as a Go file
+// that can be included in the Collate package.
+func (b *Builder) Print(w io.Writer) (n int, err error) {
+	p := func(nn int, e error) {
+		n += nn
+		if err == nil {
+			err = e
+		}
+	}
+	t, err := b.build()
+	if err != nil {
+		return 0, err
+	}
+	p(fmt.Fprintf(w, `var availableLocales = "und`))
+	for _, loc := range b.locale {
+		if loc.id != "und" {
+			p(fmt.Fprintf(w, ",%s", loc.id))
+		}
+	}
+	p(fmt.Fprint(w, "\"\n\n"))
+	p(fmt.Fprintf(w, "const varTop = 0x%x\n\n", b.varTop))
+	p(fmt.Fprintln(w, "var locales = [...]tableIndex{"))
+	for _, loc := range b.locale {
+		if loc.id == "und" {
+			p(t.fprintIndex(w, loc.index.handle, loc.id))
+		}
+	}
+	for _, loc := range b.locale {
+		if loc.id != "und" {
+			p(t.fprintIndex(w, loc.index.handle, loc.id))
+		}
+	}
+	p(fmt.Fprint(w, "}\n\n"))
+	n, _, err = t.fprint(w, "main")
+	return
+}
+
+// reproducibleFromNFKD checks whether the given expansion could be generated
+// from an NFKD expansion.
+func reproducibleFromNFKD(e *entry, exp, nfkd []rawCE) bool {
+	// Length must be equal.
+	if len(exp) != len(nfkd) {
+		return false
+	}
+	for i, ce := range exp {
+		// Primary and secondary values should be equal.
+		if ce.w[0] != nfkd[i].w[0] || ce.w[1] != nfkd[i].w[1] {
+			return false
+		}
+		// Tertiary values should be equal to maxTertiary for third element onwards.
+		// TODO: there seem to be a lot of cases in CLDR (e.g. ㏭ in zh.xml) that can
+		// simply be dropped.  Try this out by dropping the following code.
+		if i >= 2 && ce.w[2] != maxTertiary {
+			return false
+		}
+		if _, err := makeCE(ce); err != nil {
+			// Simply return false. The error will be caught elsewhere.
+			return false
+		}
+	}
+	return true
+}
+
+func simplify(o *ordering) {
+	// Runes that are a starter of a contraction should not be removed.
+	// (To date, there is only Kannada character 0CCA.)
+	keep := make(map[rune]bool)
+	for e := o.front(); e != nil; e, _ = e.nextIndexed() {
+		if len(e.runes) > 1 {
+			keep[e.runes[0]] = true
+		}
+	}
+	// Tag entries for which the runes NFKD decompose to identical values.
+	for e := o.front(); e != nil; e, _ = e.nextIndexed() {
+		s := e.str
+		nfkd := norm.NFKD.String(s)
+		nfd := norm.NFD.String(s)
+		if e.decompose || len(e.runes) > 1 || len(e.elems) == 1 || keep[e.runes[0]] || nfkd == nfd {
+			continue
+		}
+		if reproducibleFromNFKD(e, e.elems, o.genColElems(nfkd)) {
+			e.decompose = true
+		}
+	}
+}
+
+// appendExpansion converts the given collation sequence to
+// collation elements and adds them to the expansion table.
+// It returns an index to the expansion table.
+func (b *Builder) appendExpansion(e *entry) int {
+	t := b.t
+	i := len(t.ExpandElem)
+	ce := uint32(len(e.elems))
+	t.ExpandElem = append(t.ExpandElem, ce)
+	for _, w := range e.elems {
+		ce, err := makeCE(w)
+		if err != nil {
+			b.error(err)
+			return -1
+		}
+		t.ExpandElem = append(t.ExpandElem, ce)
+	}
+	return i
+}
+
+// processExpansions extracts data necessary to generate
+// the extraction tables.
+func (b *Builder) processExpansions(o *ordering) {
+	for e := o.front(); e != nil; e, _ = e.nextIndexed() {
+		if !e.expansion() {
+			continue
+		}
+		key := fmt.Sprintf("%v", e.elems)
+		i, ok := b.expIndex[key]
+		if !ok {
+			i = b.appendExpansion(e)
+			b.expIndex[key] = i
+		}
+		e.expansionIndex = i
+	}
+}
+
+func (b *Builder) processContractions(o *ordering) {
+	// Collate contractions per starter rune.
+	starters := []rune{}
+	cm := make(map[rune][]*entry)
+	for e := o.front(); e != nil; e, _ = e.nextIndexed() {
+		if e.contraction() {
+			if len(e.str) > b.t.MaxContractLen {
+				b.t.MaxContractLen = len(e.str)
+			}
+			r := e.runes[0]
+			if _, ok := cm[r]; !ok {
+				starters = append(starters, r)
+			}
+			cm[r] = append(cm[r], e)
+		}
+	}
+	// Add entries of single runes that are at a start of a contraction.
+	for e := o.front(); e != nil; e, _ = e.nextIndexed() {
+		if !e.contraction() {
+			r := e.runes[0]
+			if _, ok := cm[r]; ok {
+				cm[r] = append(cm[r], e)
+			}
+		}
+	}
+	// Build the tries for the contractions.
+	t := b.t
+	for _, r := range starters {
+		l := cm[r]
+		// Compute suffix strings. There are 31 different contraction suffix
+		// sets for 715 contractions and 82 contraction starter runes as of
+		// version 6.0.0.
+		sufx := []string{}
+		hasSingle := false
+		for _, e := range l {
+			if len(e.runes) > 1 {
+				sufx = append(sufx, string(e.runes[1:]))
+			} else {
+				hasSingle = true
+			}
+		}
+		if !hasSingle {
+			b.error(fmt.Errorf("no single entry for starter rune %U found", r))
+			continue
+		}
+		// Unique the suffix set.
+		sort.Strings(sufx)
+		key := strings.Join(sufx, "\n")
+		handle, ok := b.ctHandle[key]
+		if !ok {
+			var err error
+			handle, err = appendTrie(&t.ContractTries, sufx)
+			if err != nil {
+				b.error(err)
+			}
+			b.ctHandle[key] = handle
+		}
+		// Bucket sort entries in index order.
+		es := make([]*entry, len(l))
+		for _, e := range l {
+			var p, sn int
+			if len(e.runes) > 1 {
+				str := []byte(string(e.runes[1:]))
+				p, sn = lookup(&t.ContractTries, handle, str)
+				if sn != len(str) {
+					log.Fatalf("%s: processContractions: unexpected length for '%X'; len=%d; want %d", o.id, e.runes, sn, len(str))
+				}
+			}
+			if es[p] != nil {
+				log.Fatalf("%s: multiple contractions for position %d for rune %U", o.id, p, e.runes[0])
+			}
+			es[p] = e
+		}
+		// Create collation elements for contractions.
+		elems := []uint32{}
+		for _, e := range es {
+			ce, err := e.encodeBase()
+			b.errorID(o.id, err)
+			elems = append(elems, ce)
+		}
+		key = fmt.Sprintf("%v", elems)
+		i, ok := b.ctElem[key]
+		if !ok {
+			i = len(t.ContractElem)
+			b.ctElem[key] = i
+			t.ContractElem = append(t.ContractElem, elems...)
+		}
+		// Store info in entry for starter rune.
+		es[0].contractionIndex = i
+		es[0].contractionHandle = handle
+	}
+}

vendor/golang.org/x/text/collate/build/colelem.go

@@ -0,0 +1,294 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"fmt"
+	"unicode"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+const (
+	defaultSecondary = 0x20
+	defaultTertiary  = 0x2
+	maxTertiary      = 0x1F
+)
+
+type rawCE struct {
+	w   []int
+	ccc uint8
+}
+
+func makeRawCE(w []int, ccc uint8) rawCE {
+	ce := rawCE{w: make([]int, 4), ccc: ccc}
+	copy(ce.w, w)
+	return ce
+}
+
+// A collation element is represented as an uint32.
+// In the typical case, a rune maps to a single collation element. If a rune
+// can be the start of a contraction or expands into multiple collation elements,
+// then the collation element that is associated with a rune will have a special
+// form to represent such m to n mappings.  Such special collation elements
+// have a value >= 0x80000000.
+
+const (
+	maxPrimaryBits   = 21
+	maxSecondaryBits = 12
+	maxTertiaryBits  = 8
+)
+
+func makeCE(ce rawCE) (uint32, error) {
+	v, e := colltab.MakeElem(ce.w[0], ce.w[1], ce.w[2], ce.ccc)
+	return uint32(v), e
+}
+
+// For contractions, collation elements are of the form
+// 110bbbbb bbbbbbbb iiiiiiii iiiinnnn, where
+//   - n* is the size of the first node in the contraction trie.
+//   - i* is the index of the first node in the contraction trie.
+//   - b* is the offset into the contraction collation element table.
+// See contract.go for details on the contraction trie.
+const (
+	contractID            = 0xC0000000
+	maxNBits              = 4
+	maxTrieIndexBits      = 12
+	maxContractOffsetBits = 13
+)
+
+func makeContractIndex(h ctHandle, offset int) (uint32, error) {
+	if h.n >= 1<<maxNBits {
+		return 0, fmt.Errorf("size of contraction trie node too large: %d >= %d", h.n, 1<<maxNBits)
+	}
+	if h.index >= 1<<maxTrieIndexBits {
+		return 0, fmt.Errorf("size of contraction trie offset too large: %d >= %d", h.index, 1<<maxTrieIndexBits)
+	}
+	if offset >= 1<<maxContractOffsetBits {
+		return 0, fmt.Errorf("contraction offset out of bounds: %x >= %x", offset, 1<<maxContractOffsetBits)
+	}
+	ce := uint32(contractID)
+	ce += uint32(offset << (maxNBits + maxTrieIndexBits))
+	ce += uint32(h.index << maxNBits)
+	ce += uint32(h.n)
+	return ce, nil
+}
+
+// For expansions, collation elements are of the form
+// 11100000 00000000 bbbbbbbb bbbbbbbb,
+// where b* is the index into the expansion sequence table.
+const (
+	expandID           = 0xE0000000
+	maxExpandIndexBits = 16
+)
+
+func makeExpandIndex(index int) (uint32, error) {
+	if index >= 1<<maxExpandIndexBits {
+		return 0, fmt.Errorf("expansion index out of bounds: %x >= %x", index, 1<<maxExpandIndexBits)
+	}
+	return expandID + uint32(index), nil
+}
+
+// Each list of collation elements corresponding to an expansion starts with
+// a header indicating the length of the sequence.
+func makeExpansionHeader(n int) (uint32, error) {
+	return uint32(n), nil
+}
+
+// Some runes can be expanded using NFKD decomposition. Instead of storing the full
+// sequence of collation elements, we decompose the rune and lookup the collation
+// elements for each rune in the decomposition and modify the tertiary weights.
+// The collation element, in this case, is of the form
+// 11110000 00000000 wwwwwwww vvvvvvvv, where
+//   - v* is the replacement tertiary weight for the first rune,
+//   - w* is the replacement tertiary weight for the second rune,
+// Tertiary weights of subsequent runes should be replaced with maxTertiary.
+// See https://www.unicode.org/reports/tr10/#Compatibility_Decompositions for more details.
+const (
+	decompID = 0xF0000000
+)
+
+func makeDecompose(t1, t2 int) (uint32, error) {
+	if t1 >= 256 || t1 < 0 {
+		return 0, fmt.Errorf("first tertiary weight out of bounds: %d >= 256", t1)
+	}
+	if t2 >= 256 || t2 < 0 {
+		return 0, fmt.Errorf("second tertiary weight out of bounds: %d >= 256", t2)
+	}
+	return uint32(t2<<8+t1) + decompID, nil
+}
+
+const (
+	// These constants were taken from https://www.unicode.org/versions/Unicode6.0.0/ch12.pdf.
+	minUnified       rune = 0x4E00
+	maxUnified            = 0x9FFF
+	minCompatibility      = 0xF900
+	maxCompatibility      = 0xFAFF
+	minRare               = 0x3400
+	maxRare               = 0x4DBF
+)
+const (
+	commonUnifiedOffset = 0x10000
+	rareUnifiedOffset   = 0x20000 // largest rune in common is U+FAFF
+	otherOffset         = 0x50000 // largest rune in rare is U+2FA1D
+	illegalOffset       = otherOffset + int(unicode.MaxRune)
+	maxPrimary          = illegalOffset + 1
+)
+
+// implicitPrimary returns the primary weight for the a rune
+// for which there is no entry for the rune in the collation table.
+// We take a different approach from the one specified in
+// https://unicode.org/reports/tr10/#Implicit_Weights,
+// but preserve the resulting relative ordering of the runes.
+func implicitPrimary(r rune) int {
+	if unicode.Is(unicode.Ideographic, r) {
+		if r >= minUnified && r <= maxUnified {
+			// The most common case for CJK.
+			return int(r) + commonUnifiedOffset
+		}
+		if r >= minCompatibility && r <= maxCompatibility {
+			// This will typically not hit. The DUCET explicitly specifies mappings
+			// for all characters that do not decompose.
+			return int(r) + commonUnifiedOffset
+		}
+		return int(r) + rareUnifiedOffset
+	}
+	return int(r) + otherOffset
+}
+
+// convertLargeWeights converts collation elements with large
+// primaries (either double primaries or for illegal runes)
+// to our own representation.
+// A CJK character C is represented in the DUCET as
+//   [.FBxx.0020.0002.C][.BBBB.0000.0000.C]
+// We will rewrite these characters to a single CE.
+// We assume the CJK values start at 0x8000.
+// See https://unicode.org/reports/tr10/#Implicit_Weights
+func convertLargeWeights(elems []rawCE) (res []rawCE, err error) {
+	const (
+		cjkPrimaryStart   = 0xFB40
+		rarePrimaryStart  = 0xFB80
+		otherPrimaryStart = 0xFBC0
+		illegalPrimary    = 0xFFFE
+		highBitsMask      = 0x3F
+		lowBitsMask       = 0x7FFF
+		lowBitsFlag       = 0x8000
+		shiftBits         = 15
+	)
+	for i := 0; i < len(elems); i++ {
+		ce := elems[i].w
+		p := ce[0]
+		if p < cjkPrimaryStart {
+			continue
+		}
+		if p > 0xFFFF {
+			return elems, fmt.Errorf("found primary weight %X; should be <= 0xFFFF", p)
+		}
+		if p >= illegalPrimary {
+			ce[0] = illegalOffset + p - illegalPrimary
+		} else {
+			if i+1 >= len(elems) {
+				return elems, fmt.Errorf("second part of double primary weight missing: %v", elems)
+			}
+			if elems[i+1].w[0]&lowBitsFlag == 0 {
+				return elems, fmt.Errorf("malformed second part of double primary weight: %v", elems)
+			}
+			np := ((p & highBitsMask) << shiftBits) + elems[i+1].w[0]&lowBitsMask
+			switch {
+			case p < rarePrimaryStart:
+				np += commonUnifiedOffset
+			case p < otherPrimaryStart:
+				np += rareUnifiedOffset
+			default:
+				p += otherOffset
+			}
+			ce[0] = np
+			for j := i + 1; j+1 < len(elems); j++ {
+				elems[j] = elems[j+1]
+			}
+			elems = elems[:len(elems)-1]
+		}
+	}
+	return elems, nil
+}
+
+// nextWeight computes the first possible collation weights following elems
+// for the given level.
+func nextWeight(level colltab.Level, elems []rawCE) []rawCE {
+	if level == colltab.Identity {
+		next := make([]rawCE, len(elems))
+		copy(next, elems)
+		return next
+	}
+	next := []rawCE{makeRawCE(elems[0].w, elems[0].ccc)}
+	next[0].w[level]++
+	if level < colltab.Secondary {
+		next[0].w[colltab.Secondary] = defaultSecondary
+	}
+	if level < colltab.Tertiary {
+		next[0].w[colltab.Tertiary] = defaultTertiary
+	}
+	// Filter entries that cannot influence ordering.
+	for _, ce := range elems[1:] {
+		skip := true
+		for i := colltab.Primary; i < level; i++ {
+			skip = skip && ce.w[i] == 0
+		}
+		if !skip {
+			next = append(next, ce)
+		}
+	}
+	return next
+}
+
+func nextVal(elems []rawCE, i int, level colltab.Level) (index, value int) {
+	for ; i < len(elems) && elems[i].w[level] == 0; i++ {
+	}
+	if i < len(elems) {
+		return i, elems[i].w[level]
+	}
+	return i, 0
+}
+
+// compareWeights returns -1 if a < b, 1 if a > b, or 0 otherwise.
+// It also returns the collation level at which the difference is found.
+func compareWeights(a, b []rawCE) (result int, level colltab.Level) {
+	for level := colltab.Primary; level < colltab.Identity; level++ {
+		var va, vb int
+		for ia, ib := 0, 0; ia < len(a) || ib < len(b); ia, ib = ia+1, ib+1 {
+			ia, va = nextVal(a, ia, level)
+			ib, vb = nextVal(b, ib, level)
+			if va != vb {
+				if va < vb {
+					return -1, level
+				} else {
+					return 1, level
+				}
+			}
+		}
+	}
+	return 0, colltab.Identity
+}
+
+func equalCE(a, b rawCE) bool {
+	for i := 0; i < 3; i++ {
+		if b.w[i] != a.w[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func equalCEArrays(a, b []rawCE) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i := range a {
+		if !equalCE(a[i], b[i]) {
+			return false
+		}
+	}
+	return true
+}

vendor/golang.org/x/text/collate/build/contract.go

@@ -0,0 +1,309 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+	"sort"
+	"strings"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+// This file contains code for detecting contractions and generating
+// the necessary tables.
+// Any Unicode Collation Algorithm (UCA) table entry that has more than
+// one rune one the left-hand side is called a contraction.
+// See https://www.unicode.org/reports/tr10/#Contractions for more details.
+//
+// We define the following terms:
+//   initial:     a rune that appears as the first rune in a contraction.
+//   suffix:      a sequence of runes succeeding the initial rune
+//                in a given contraction.
+//   non-initial: a rune that appears in a suffix.
+//
+// A rune may be both an initial and a non-initial and may be so in
+// many contractions.  An initial may typically also appear by itself.
+// In case of ambiguities, the UCA requires we match the longest
+// contraction.
+//
+// Many contraction rules share the same set of possible suffixes.
+// We store sets of suffixes in a trie that associates an index with
+// each suffix in the set.  This index can be used to look up a
+// collation element associated with the (starter rune, suffix) pair.
+//
+// The trie is defined on a UTF-8 byte sequence.
+// The overall trie is represented as an array of ctEntries.  Each node of the trie
+// is represented as a subsequence of ctEntries, where each entry corresponds to
+// a possible match of a next character in the search string.  An entry
+// also includes the length and offset to the next sequence of entries
+// to check in case of a match.
+
+const (
+	final   = 0
+	noIndex = 0xFF
+)
+
+// ctEntry associates to a matching byte an offset and/or next sequence of
+// bytes to check. A ctEntry c is called final if a match means that the
+// longest suffix has been found.  An entry c is final if c.N == 0.
+// A single final entry can match a range of characters to an offset.
+// A non-final entry always matches a single byte. Note that a non-final
+// entry might still resemble a completed suffix.
+// Examples:
+// The suffix strings "ab" and "ac" can be represented as:
+// []ctEntry{
+//     {'a', 1, 1, noIndex},  // 'a' by itself does not match, so i is 0xFF.
+//     {'b', 'c', 0, 1},   // "ab" -> 1, "ac" -> 2
+// }
+//
+// The suffix strings "ab", "abc", "abd", and "abcd" can be represented as:
+// []ctEntry{
+//     {'a', 1, 1, noIndex}, // 'a' must be followed by 'b'.
+//     {'b', 1, 2, 1},    // "ab" -> 1, may be followed by 'c' or 'd'.
+//     {'d', 'd', final, 3},  // "abd" -> 3
+//     {'c', 4, 1, 2},    // "abc" -> 2, may be followed by 'd'.
+//     {'d', 'd', final, 4},  // "abcd" -> 4
+// }
+// See genStateTests in contract_test.go for more examples.
+type ctEntry struct {
+	L uint8 // non-final: byte value to match; final: lowest match in range.
+	H uint8 // non-final: relative index to next block; final: highest match in range.
+	N uint8 // non-final: length of next block; final: final
+	I uint8 // result offset. Will be noIndex if more bytes are needed to complete.
+}
+
+// contractTrieSet holds a set of contraction tries. The tries are stored
+// consecutively in the entry field.
+type contractTrieSet []struct{ l, h, n, i uint8 }
+
+// ctHandle is used to identify a trie in the trie set, consisting in an offset
+// in the array and the size of the first node.
+type ctHandle struct {
+	index, n int
+}
+
+// appendTrie adds a new trie for the given suffixes to the trie set and returns
+// a handle to it.  The handle will be invalid on error.
+func appendTrie(ct *colltab.ContractTrieSet, suffixes []string) (ctHandle, error) {
+	es := make([]stridx, len(suffixes))
+	for i, s := range suffixes {
+		es[i].str = s
+	}
+	sort.Sort(offsetSort(es))
+	for i := range es {
+		es[i].index = i + 1
+	}
+	sort.Sort(genidxSort(es))
+	i := len(*ct)
+	n, err := genStates(ct, es)
+	if err != nil {
+		*ct = (*ct)[:i]
+		return ctHandle{}, err
+	}
+	return ctHandle{i, n}, nil
+}
+
+// genStates generates ctEntries for a given suffix set and returns
+// the number of entries for the first node.
+func genStates(ct *colltab.ContractTrieSet, sis []stridx) (int, error) {
+	if len(sis) == 0 {
+		return 0, fmt.Errorf("genStates: list of suffices must be non-empty")
+	}
+	start := len(*ct)
+	// create entries for differing first bytes.
+	for _, si := range sis {
+		s := si.str
+		if len(s) == 0 {
+			continue
+		}
+		added := false
+		c := s[0]
+		if len(s) > 1 {
+			for j := len(*ct) - 1; j >= start; j-- {
+				if (*ct)[j].L == c {
+					added = true
+					break
+				}
+			}
+			if !added {
+				*ct = append(*ct, ctEntry{L: c, I: noIndex})
+			}
+		} else {
+			for j := len(*ct) - 1; j >= start; j-- {
+				// Update the offset for longer suffixes with the same byte.
+				if (*ct)[j].L == c {
+					(*ct)[j].I = uint8(si.index)
+					added = true
+				}
+				// Extend range of final ctEntry, if possible.
+				if (*ct)[j].H+1 == c {
+					(*ct)[j].H = c
+					added = true
+				}
+			}
+			if !added {
+				*ct = append(*ct, ctEntry{L: c, H: c, N: final, I: uint8(si.index)})
+			}
+		}
+	}
+	n := len(*ct) - start
+	// Append nodes for the remainder of the suffixes for each ctEntry.
+	sp := 0
+	for i, end := start, len(*ct); i < end; i++ {
+		fe := (*ct)[i]
+		if fe.H == 0 { // uninitialized non-final
+			ln := len(*ct) - start - n
+			if ln > 0xFF {
+				return 0, fmt.Errorf("genStates: relative block offset too large: %d > 255", ln)
+			}
+			fe.H = uint8(ln)
+			// Find first non-final strings with same byte as current entry.
+			for ; sis[sp].str[0] != fe.L; sp++ {
+			}
+			se := sp + 1
+			for ; se < len(sis) && len(sis[se].str) > 1 && sis[se].str[0] == fe.L; se++ {
+			}
+			sl := sis[sp:se]
+			sp = se
+			for i, si := range sl {
+				sl[i].str = si.str[1:]
+			}
+			nn, err := genStates(ct, sl)
+			if err != nil {
+				return 0, err
+			}
+			fe.N = uint8(nn)
+			(*ct)[i] = fe
+		}
+	}
+	sort.Sort(entrySort((*ct)[start : start+n]))
+	return n, nil
+}
+
+// There may be both a final and non-final entry for a byte if the byte
+// is implied in a range of matches in the final entry.
+// We need to ensure that the non-final entry comes first in that case.
+type entrySort colltab.ContractTrieSet
+
+func (fe entrySort) Len() int      { return len(fe) }
+func (fe entrySort) Swap(i, j int) { fe[i], fe[j] = fe[j], fe[i] }
+func (fe entrySort) Less(i, j int) bool {
+	return fe[i].L > fe[j].L
+}
+
+// stridx is used for sorting suffixes and their associated offsets.
+type stridx struct {
+	str   string
+	index int
+}
+
+// For computing the offsets, we first sort by size, and then by string.
+// This ensures that strings that only differ in the last byte by 1
+// are sorted consecutively in increasing order such that they can
+// be packed as a range in a final ctEntry.
+type offsetSort []stridx
+
+func (si offsetSort) Len() int      { return len(si) }
+func (si offsetSort) Swap(i, j int) { si[i], si[j] = si[j], si[i] }
+func (si offsetSort) Less(i, j int) bool {
+	if len(si[i].str) != len(si[j].str) {
+		return len(si[i].str) > len(si[j].str)
+	}
+	return si[i].str < si[j].str
+}
+
+// For indexing, we want to ensure that strings are sorted in string order, where
+// for strings with the same prefix, we put longer strings before shorter ones.
+type genidxSort []stridx
+
+func (si genidxSort) Len() int      { return len(si) }
+func (si genidxSort) Swap(i, j int) { si[i], si[j] = si[j], si[i] }
+func (si genidxSort) Less(i, j int) bool {
+	if strings.HasPrefix(si[j].str, si[i].str) {
+		return false
+	}
+	if strings.HasPrefix(si[i].str, si[j].str) {
+		return true
+	}
+	return si[i].str < si[j].str
+}
+
+// lookup matches the longest suffix in str and returns the associated offset
+// and the number of bytes consumed.
+func lookup(ct *colltab.ContractTrieSet, h ctHandle, str []byte) (index, ns int) {
+	states := (*ct)[h.index:]
+	p := 0
+	n := h.n
+	for i := 0; i < n && p < len(str); {
+		e := states[i]
+		c := str[p]
+		if c >= e.L {
+			if e.L == c {
+				p++
+				if e.I != noIndex {
+					index, ns = int(e.I), p
+				}
+				if e.N != final {
+					// set to new state
+					i, states, n = 0, states[int(e.H)+n:], int(e.N)
+				} else {
+					return
+				}
+				continue
+			} else if e.N == final && c <= e.H {
+				p++
+				return int(c-e.L) + int(e.I), p
+			}
+		}
+		i++
+	}
+	return
+}
+
+// print writes the contractTrieSet t as compilable Go code to w. It returns
+// the total number of bytes written and the size of the resulting data structure in bytes.
+func print(t *colltab.ContractTrieSet, w io.Writer, name string) (n, size int, err error) {
+	update3 := func(nn, sz int, e error) {
+		n += nn
+		if err == nil {
+			err = e
+		}
+		size += sz
+	}
+	update2 := func(nn int, e error) { update3(nn, 0, e) }
+
+	update3(printArray(*t, w, name))
+	update2(fmt.Fprintf(w, "var %sContractTrieSet = ", name))
+	update3(printStruct(*t, w, name))
+	update2(fmt.Fprintln(w))
+	return
+}
+
+func printArray(ct colltab.ContractTrieSet, w io.Writer, name string) (n, size int, err error) {
+	p := func(f string, a ...interface{}) {
+		nn, e := fmt.Fprintf(w, f, a...)
+		n += nn
+		if err == nil {
+			err = e
+		}
+	}
+	size = len(ct) * 4
+	p("// %sCTEntries: %d entries, %d bytes\n", name, len(ct), size)
+	p("var %sCTEntries = [%d]struct{L,H,N,I uint8}{\n", name, len(ct))
+	for _, fe := range ct {
+		p("\t{0x%X, 0x%X, %d, %d},\n", fe.L, fe.H, fe.N, fe.I)
+	}
+	p("}\n")
+	return
+}
+
+func printStruct(ct colltab.ContractTrieSet, w io.Writer, name string) (n, size int, err error) {
+	n, err = fmt.Fprintf(w, "colltab.ContractTrieSet( %sCTEntries[:] )", name)
+	size = int(reflect.TypeOf(ct).Size())
+	return
+}

vendor/golang.org/x/text/collate/build/order.go

@@ -0,0 +1,393 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"fmt"
+	"log"
+	"sort"
+	"strings"
+	"unicode"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/unicode/norm"
+)
+
+type logicalAnchor int
+
+const (
+	firstAnchor logicalAnchor = -1
+	noAnchor                  = 0
+	lastAnchor                = 1
+)
+
+// entry is used to keep track of a single entry in the collation element table
+// during building. Examples of entries can be found in the Default Unicode
+// Collation Element Table.
+// See https://www.unicode.org/Public/UCA/6.0.0/allkeys.txt.
+type entry struct {
+	str    string // same as string(runes)
+	runes  []rune
+	elems  []rawCE // the collation elements
+	extend string  // weights of extend to be appended to elems
+	before bool    // weights relative to next instead of previous.
+	lock   bool    // entry is used in extension and can no longer be moved.
+
+	// prev, next, and level are used to keep track of tailorings.
+	prev, next *entry
+	level      colltab.Level // next differs at this level
+	skipRemove bool          // do not unlink when removed
+
+	decompose bool // can use NFKD decomposition to generate elems
+	exclude   bool // do not include in table
+	implicit  bool // derived, is not included in the list
+	modified  bool // entry was modified in tailoring
+	logical   logicalAnchor
+
+	expansionIndex    int // used to store index into expansion table
+	contractionHandle ctHandle
+	contractionIndex  int // index into contraction elements
+}
+
+func (e *entry) String() string {
+	return fmt.Sprintf("%X (%q) -> %X (ch:%x; ci:%d, ei:%d)",
+		e.runes, e.str, e.elems, e.contractionHandle, e.contractionIndex, e.expansionIndex)
+}
+
+func (e *entry) skip() bool {
+	return e.contraction()
+}
+
+func (e *entry) expansion() bool {
+	return !e.decompose && len(e.elems) > 1
+}
+
+func (e *entry) contraction() bool {
+	return len(e.runes) > 1
+}
+
+func (e *entry) contractionStarter() bool {
+	return e.contractionHandle.n != 0
+}
+
+// nextIndexed gets the next entry that needs to be stored in the table.
+// It returns the entry and the collation level at which the next entry differs
+// from the current entry.
+// Entries that can be explicitly derived and logical reset positions are
+// examples of entries that will not be indexed.
+func (e *entry) nextIndexed() (*entry, colltab.Level) {
+	level := e.level
+	for e = e.next; e != nil && (e.exclude || len(e.elems) == 0); e = e.next {
+		if e.level < level {
+			level = e.level
+		}
+	}
+	return e, level
+}
+
+// remove unlinks entry e from the sorted chain and clears the collation
+// elements. e may not be at the front or end of the list. This should always
+// be the case, as the front and end of the list are always logical anchors,
+// which may not be removed.
+func (e *entry) remove() {
+	if e.logical != noAnchor {
+		log.Fatalf("may not remove anchor %q", e.str)
+	}
+	// TODO: need to set e.prev.level to e.level if e.level is smaller?
+	e.elems = nil
+	if !e.skipRemove {
+		if e.prev != nil {
+			e.prev.next = e.next
+		}
+		if e.next != nil {
+			e.next.prev = e.prev
+		}
+	}
+	e.skipRemove = false
+}
+
+// insertAfter inserts n after e.
+func (e *entry) insertAfter(n *entry) {
+	if e == n {
+		panic("e == anchor")
+	}
+	if e == nil {
+		panic("unexpected nil anchor")
+	}
+	n.remove()
+	n.decompose = false // redo decomposition test
+
+	n.next = e.next
+	n.prev = e
+	if e.next != nil {
+		e.next.prev = n
+	}
+	e.next = n
+}
+
+// insertBefore inserts n before e.
+func (e *entry) insertBefore(n *entry) {
+	if e == n {
+		panic("e == anchor")
+	}
+	if e == nil {
+		panic("unexpected nil anchor")
+	}
+	n.remove()
+	n.decompose = false // redo decomposition test
+
+	n.prev = e.prev
+	n.next = e
+	if e.prev != nil {
+		e.prev.next = n
+	}
+	e.prev = n
+}
+
+func (e *entry) encodeBase() (ce uint32, err error) {
+	switch {
+	case e.expansion():
+		ce, err = makeExpandIndex(e.expansionIndex)
+	default:
+		if e.decompose {
+			log.Fatal("decompose should be handled elsewhere")
+		}
+		ce, err = makeCE(e.elems[0])
+	}
+	return
+}
+
+func (e *entry) encode() (ce uint32, err error) {
+	if e.skip() {
+		log.Fatal("cannot build colElem for entry that should be skipped")
+	}
+	switch {
+	case e.decompose:
+		t1 := e.elems[0].w[2]
+		t2 := 0
+		if len(e.elems) > 1 {
+			t2 = e.elems[1].w[2]
+		}
+		ce, err = makeDecompose(t1, t2)
+	case e.contractionStarter():
+		ce, err = makeContractIndex(e.contractionHandle, e.contractionIndex)
+	default:
+		if len(e.runes) > 1 {
+			log.Fatal("colElem: contractions are handled in contraction trie")
+		}
+		ce, err = e.encodeBase()
+	}
+	return
+}
+
+// entryLess returns true if a sorts before b and false otherwise.
+func entryLess(a, b *entry) bool {
+	if res, _ := compareWeights(a.elems, b.elems); res != 0 {
+		return res == -1
+	}
+	if a.logical != noAnchor {
+		return a.logical == firstAnchor
+	}
+	if b.logical != noAnchor {
+		return b.logical == lastAnchor
+	}
+	return a.str < b.str
+}
+
+type sortedEntries []*entry
+
+func (s sortedEntries) Len() int {
+	return len(s)
+}
+
+func (s sortedEntries) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s sortedEntries) Less(i, j int) bool {
+	return entryLess(s[i], s[j])
+}
+
+type ordering struct {
+	id       string
+	entryMap map[string]*entry
+	ordered  []*entry
+	handle   *trieHandle
+}
+
+// insert inserts e into both entryMap and ordered.
+// Note that insert simply appends e to ordered.  To reattain a sorted
+// order, o.sort() should be called.
+func (o *ordering) insert(e *entry) {
+	if e.logical == noAnchor {
+		o.entryMap[e.str] = e
+	} else {
+		// Use key format as used in UCA rules.
+		o.entryMap[fmt.Sprintf("[%s]", e.str)] = e
+		// Also add index entry for XML format.
+		o.entryMap[fmt.Sprintf("<%s/>", strings.Replace(e.str, " ", "_", -1))] = e
+	}
+	o.ordered = append(o.ordered, e)
+}
+
+// newEntry creates a new entry for the given info and inserts it into
+// the index.
+func (o *ordering) newEntry(s string, ces []rawCE) *entry {
+	e := &entry{
+		runes: []rune(s),
+		elems: ces,
+		str:   s,
+	}
+	o.insert(e)
+	return e
+}
+
+// find looks up and returns the entry for the given string.
+// It returns nil if str is not in the index and if an implicit value
+// cannot be derived, that is, if str represents more than one rune.
+func (o *ordering) find(str string) *entry {
+	e := o.entryMap[str]
+	if e == nil {
+		r := []rune(str)
+		if len(r) == 1 {
+			const (
+				firstHangul = 0xAC00
+				lastHangul  = 0xD7A3
+			)
+			if r[0] >= firstHangul && r[0] <= lastHangul {
+				ce := []rawCE{}
+				nfd := norm.NFD.String(str)
+				for _, r := range nfd {
+					ce = append(ce, o.find(string(r)).elems...)
+				}
+				e = o.newEntry(nfd, ce)
+			} else {
+				e = o.newEntry(string(r[0]), []rawCE{
+					{w: []int{
+						implicitPrimary(r[0]),
+						defaultSecondary,
+						defaultTertiary,
+						int(r[0]),
+					},
+					},
+				})
+				e.modified = true
+			}
+			e.exclude = true // do not index implicits
+		}
+	}
+	return e
+}
+
+// makeRootOrdering returns a newly initialized ordering value and populates
+// it with a set of logical reset points that can be used as anchors.
+// The anchors first_tertiary_ignorable and __END__ will always sort at
+// the beginning and end, respectively. This means that prev and next are non-nil
+// for any indexed entry.
+func makeRootOrdering() ordering {
+	const max = unicode.MaxRune
+	o := ordering{
+		entryMap: make(map[string]*entry),
+	}
+	insert := func(typ logicalAnchor, s string, ce []int) {
+		e := &entry{
+			elems:   []rawCE{{w: ce}},
+			str:     s,
+			exclude: true,
+			logical: typ,
+		}
+		o.insert(e)
+	}
+	insert(firstAnchor, "first tertiary ignorable", []int{0, 0, 0, 0})
+	insert(lastAnchor, "last tertiary ignorable", []int{0, 0, 0, max})
+	insert(lastAnchor, "last primary ignorable", []int{0, defaultSecondary, defaultTertiary, max})
+	insert(lastAnchor, "last non ignorable", []int{maxPrimary, defaultSecondary, defaultTertiary, max})
+	insert(lastAnchor, "__END__", []int{1 << maxPrimaryBits, defaultSecondary, defaultTertiary, max})
+	return o
+}
+
+// patchForInsert eleminates entries from the list with more than one collation element.
+// The next and prev fields of the eliminated entries still point to appropriate
+// values in the newly created list.
+// It requires that sort has been called.
+func (o *ordering) patchForInsert() {
+	for i := 0; i < len(o.ordered)-1; {
+		e := o.ordered[i]
+		lev := e.level
+		n := e.next
+		for ; n != nil && len(n.elems) > 1; n = n.next {
+			if n.level < lev {
+				lev = n.level
+			}
+			n.skipRemove = true
+		}
+		for ; o.ordered[i] != n; i++ {
+			o.ordered[i].level = lev
+			o.ordered[i].next = n
+			o.ordered[i+1].prev = e
+		}
+	}
+}
+
+// clone copies all ordering of es into a new ordering value.
+func (o *ordering) clone() *ordering {
+	o.sort()
+	oo := ordering{
+		entryMap: make(map[string]*entry),
+	}
+	for _, e := range o.ordered {
+		ne := &entry{
+			runes:     e.runes,
+			elems:     e.elems,
+			str:       e.str,
+			decompose: e.decompose,
+			exclude:   e.exclude,
+			logical:   e.logical,
+		}
+		oo.insert(ne)
+	}
+	oo.sort() // link all ordering.
+	oo.patchForInsert()
+	return &oo
+}
+
+// front returns the first entry to be indexed.
+// It assumes that sort() has been called.
+func (o *ordering) front() *entry {
+	e := o.ordered[0]
+	if e.prev != nil {
+		log.Panicf("unexpected first entry: %v", e)
+	}
+	// The first entry is always a logical position, which should not be indexed.
+	e, _ = e.nextIndexed()
+	return e
+}
+
+// sort sorts all ordering based on their collation elements and initializes
+// the prev, next, and level fields accordingly.
+func (o *ordering) sort() {
+	sort.Sort(sortedEntries(o.ordered))
+	l := o.ordered
+	for i := 1; i < len(l); i++ {
+		k := i - 1
+		l[k].next = l[i]
+		_, l[k].level = compareWeights(l[k].elems, l[i].elems)
+		l[i].prev = l[k]
+	}
+}
+
+// genColElems generates a collation element array from the runes in str. This
+// assumes that all collation elements have already been added to the Builder.
+func (o *ordering) genColElems(str string) []rawCE {
+	elems := []rawCE{}
+	for _, r := range []rune(str) {
+		for _, ce := range o.find(string(r)).elems {
+			if ce.w[0] != 0 || ce.w[1] != 0 || ce.w[2] != 0 {
+				elems = append(elems, ce)
+			}
+		}
+	}
+	return elems
+}

vendor/golang.org/x/text/collate/build/table.go

@@ -0,0 +1,81 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package build
+
+import (
+	"fmt"
+	"io"
+	"reflect"
+
+	"golang.org/x/text/internal/colltab"
+)
+
+// table is an intermediate structure that roughly resembles the table in collate.
+type table struct {
+	colltab.Table
+	trie trie
+	root *trieHandle
+}
+
+// print writes the table as Go compilable code to w. It prefixes the
+// variable names with name. It returns the number of bytes written
+// and the size of the resulting table.
+func (t *table) fprint(w io.Writer, name string) (n, size int, err error) {
+	update := func(nn, sz int, e error) {
+		n += nn
+		if err == nil {
+			err = e
+		}
+		size += sz
+	}
+	// Write arrays needed for the structure.
+	update(printColElems(w, t.ExpandElem, name+"ExpandElem"))
+	update(printColElems(w, t.ContractElem, name+"ContractElem"))
+	update(t.trie.printArrays(w, name))
+	update(printArray(t.ContractTries, w, name))
+
+	nn, e := fmt.Fprintf(w, "// Total size of %sTable is %d bytes\n", name, size)
+	update(nn, 0, e)
+	return
+}
+
+func (t *table) fprintIndex(w io.Writer, h *trieHandle, id string) (n int, err error) {
+	p := func(f string, a ...interface{}) {
+		nn, e := fmt.Fprintf(w, f, a...)
+		n += nn
+		if err == nil {
+			err = e
+		}
+	}
+	p("\t{ // %s\n", id)
+	p("\t\tlookupOffset: 0x%x,\n", h.lookupStart)
+	p("\t\tvaluesOffset: 0x%x,\n", h.valueStart)
+	p("\t},\n")
+	return
+}
+
+func printColElems(w io.Writer, a []uint32, name string) (n, sz int, err error) {
+	p := func(f string, a ...interface{}) {
+		nn, e := fmt.Fprintf(w, f, a...)
+		n += nn
+		if err == nil {
+			err = e
+		}
+	}
+	sz = len(a) * int(reflect.TypeOf(uint32(0)).Size())
+	p("// %s: %d entries, %d bytes\n", name, len(a), sz)
+	p("var %s = [%d]uint32 {", name, len(a))
+	for i, c := range a {
+		switch {
+		case i%64 == 0:
+			p("\n\t// Block %d, offset 0x%x\n", i/64, i)
+		case (i%64)%6 == 0:
+			p("\n\t")
+		}
+		p("0x%.8X, ", c)
+	}
+	p("\n}\n\n")
+	return
+}

vendor/golang.org/x/text/collate/build/trie.go

@@ -0,0 +1,290 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// The trie in this file is used to associate the first full character
+// in a UTF-8 string to a collation element.
+// All but the last byte in a UTF-8 byte sequence are
+// used to look up offsets in the index table to be used for the next byte.
+// The last byte is used to index into a table of collation elements.
+// This file contains the code for the generation of the trie.
+
+package build
+
+import (
+	"fmt"
+	"hash/fnv"
+	"io"
+	"reflect"
+)
+
+const (
+	blockSize   = 64
+	blockOffset = 2 // Subtract 2 blocks to compensate for the 0x80 added to continuation bytes.
+)
+
+type trieHandle struct {
+	lookupStart uint16 // offset in table for first byte
+	valueStart  uint16 // offset in table for first byte
+}
+
+type trie struct {
+	index  []uint16
+	values []uint32
+}
+
+// trieNode is the intermediate trie structure used for generating a trie.
+type trieNode struct {
+	index    []*trieNode
+	value    []uint32
+	b        byte
+	refValue uint16
+	refIndex uint16
+}
+
+func newNode() *trieNode {
+	return &trieNode{
+		index: make([]*trieNode, 64),
+		value: make([]uint32, 128), // root node size is 128 instead of 64
+	}
+}
+
+func (n *trieNode) isInternal() bool {
+	return n.value != nil
+}
+
+func (n *trieNode) insert(r rune, value uint32) {
+	const maskx = 0x3F // mask out two most-significant bits
+	str := string(r)
+	if len(str) == 1 {
+		n.value[str[0]] = value
+		return
+	}
+	for i := 0; i < len(str)-1; i++ {
+		b := str[i] & maskx
+		if n.index == nil {
+			n.index = make([]*trieNode, blockSize)
+		}
+		nn := n.index[b]
+		if nn == nil {
+			nn = &trieNode{}
+			nn.b = b
+			n.index[b] = nn
+		}
+		n = nn
+	}
+	if n.value == nil {
+		n.value = make([]uint32, blockSize)
+	}
+	b := str[len(str)-1] & maskx
+	n.value[b] = value
+}
+
+type trieBuilder struct {
+	t *trie
+
+	roots []*trieHandle
+
+	lookupBlocks []*trieNode
+	valueBlocks  []*trieNode
+
+	lookupBlockIdx map[uint32]*trieNode
+	valueBlockIdx  map[uint32]*trieNode
+}
+
+func newTrieBuilder() *trieBuilder {
+	index := &trieBuilder{}
+	index.lookupBlocks = make([]*trieNode, 0)
+	index.valueBlocks = make([]*trieNode, 0)
+	index.lookupBlockIdx = make(map[uint32]*trieNode)
+	index.valueBlockIdx = make(map[uint32]*trieNode)
+	// The third nil is the default null block.  The other two blocks
+	// are used to guarantee an offset of at least 3 for each block.
+	index.lookupBlocks = append(index.lookupBlocks, nil, nil, nil)
+	index.t = &trie{}
+	return index
+}
+
+func (b *trieBuilder) computeOffsets(n *trieNode) *trieNode {
+	hasher := fnv.New32()
+	if n.index != nil {
+		for i, nn := range n.index {
+			var vi, vv uint16
+			if nn != nil {
+				nn = b.computeOffsets(nn)
+				n.index[i] = nn
+				vi = nn.refIndex
+				vv = nn.refValue
+			}
+			hasher.Write([]byte{byte(vi >> 8), byte(vi)})
+			hasher.Write([]byte{byte(vv >> 8), byte(vv)})
+		}
+		h := hasher.Sum32()
+		nn, ok := b.lookupBlockIdx[h]
+		if !ok {
+			n.refIndex = uint16(len(b.lookupBlocks)) - blockOffset
+			b.lookupBlocks = append(b.lookupBlocks, n)
+			b.lookupBlockIdx[h] = n
+		} else {
+			n = nn
+		}
+	} else {
+		for _, v := range n.value {
+			hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
+		}
+		h := hasher.Sum32()
+		nn, ok := b.valueBlockIdx[h]
+		if !ok {
+			n.refValue = uint16(len(b.valueBlocks)) - blockOffset
+			n.refIndex = n.refValue
+			b.valueBlocks = append(b.valueBlocks, n)
+			b.valueBlockIdx[h] = n
+		} else {
+			n = nn
+		}
+	}
+	return n
+}
+
+func (b *trieBuilder) addStartValueBlock(n *trieNode) uint16 {
+	hasher := fnv.New32()
+	for _, v := range n.value[:2*blockSize] {
+		hasher.Write([]byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)})
+	}
+	h := hasher.Sum32()
+	nn, ok := b.valueBlockIdx[h]
+	if !ok {
+		n.refValue = uint16(len(b.valueBlocks))
+		n.refIndex = n.refValue
+		b.valueBlocks = append(b.valueBlocks, n)
+		// Add a dummy block to accommodate the double block size.
+		b.valueBlocks = append(b.valueBlocks, nil)
+		b.valueBlockIdx[h] = n
+	} else {
+		n = nn
+	}
+	return n.refValue
+}
+
+func genValueBlock(t *trie, n *trieNode) {
+	if n != nil {
+		for _, v := range n.value {
+			t.values = append(t.values, v)
+		}
+	}
+}
+
+func genLookupBlock(t *trie, n *trieNode) {
+	for _, nn := range n.index {
+		v := uint16(0)
+		if nn != nil {
+			if n.index != nil {
+				v = nn.refIndex
+			} else {
+				v = nn.refValue
+			}
+		}
+		t.index = append(t.index, v)
+	}
+}
+
+func (b *trieBuilder) addTrie(n *trieNode) *trieHandle {
+	h := &trieHandle{}
+	b.roots = append(b.roots, h)
+	h.valueStart = b.addStartValueBlock(n)
+	if len(b.roots) == 1 {
+		// We insert a null block after the first start value block.
+		// This ensures that continuation bytes UTF-8 sequences of length
+		// greater than 2 will automatically hit a null block if there
+		// was an undefined entry.
+		b.valueBlocks = append(b.valueBlocks, nil)
+	}
+	n = b.computeOffsets(n)
+	// Offset by one extra block as the first byte starts at 0xC0 instead of 0x80.
+	h.lookupStart = n.refIndex - 1
+	return h
+}
+
+// generate generates and returns the trie for n.
+func (b *trieBuilder) generate() (t *trie, err error) {
+	t = b.t
+	if len(b.valueBlocks) >= 1<<16 {
+		return nil, fmt.Errorf("maximum number of value blocks exceeded (%d > %d)", len(b.valueBlocks), 1<<16)
+	}
+	if len(b.lookupBlocks) >= 1<<16 {
+		return nil, fmt.Errorf("maximum number of lookup blocks exceeded (%d > %d)", len(b.lookupBlocks), 1<<16)
+	}
+	genValueBlock(t, b.valueBlocks[0])
+	genValueBlock(t, &trieNode{value: make([]uint32, 64)})
+	for i := 2; i < len(b.valueBlocks); i++ {
+		genValueBlock(t, b.valueBlocks[i])
+	}
+	n := &trieNode{index: make([]*trieNode, 64)}
+	genLookupBlock(t, n)
+	genLookupBlock(t, n)
+	genLookupBlock(t, n)
+	for i := 3; i < len(b.lookupBlocks); i++ {
+		genLookupBlock(t, b.lookupBlocks[i])
+	}
+	return b.t, nil
+}
+
+func (t *trie) printArrays(w io.Writer, name string) (n, size int, err error) {
+	p := func(f string, a ...interface{}) {
+		nn, e := fmt.Fprintf(w, f, a...)
+		n += nn
+		if err == nil {
+			err = e
+		}
+	}
+	nv := len(t.values)
+	p("// %sValues: %d entries, %d bytes\n", name, nv, nv*4)
+	p("// Block 2 is the null block.\n")
+	p("var %sValues = [%d]uint32 {", name, nv)
+	var printnewline bool
+	for i, v := range t.values {
+		if i%blockSize == 0 {
+			p("\n\t// Block %#x, offset %#x", i/blockSize, i)
+		}
+		if i%4 == 0 {
+			printnewline = true
+		}
+		if v != 0 {
+			if printnewline {
+				p("\n\t")
+				printnewline = false
+			}
+			p("%#04x:%#08x, ", i, v)
+		}
+	}
+	p("\n}\n\n")
+	ni := len(t.index)
+	p("// %sLookup: %d entries, %d bytes\n", name, ni, ni*2)
+	p("// Block 0 is the null block.\n")
+	p("var %sLookup = [%d]uint16 {", name, ni)
+	printnewline = false
+	for i, v := range t.index {
+		if i%blockSize == 0 {
+			p("\n\t// Block %#x, offset %#x", i/blockSize, i)
+		}
+		if i%8 == 0 {
+			printnewline = true
+		}
+		if v != 0 {
+			if printnewline {
+				p("\n\t")
+				printnewline = false
+			}
+			p("%#03x:%#02x, ", i, v)
+		}
+	}
+	p("\n}\n\n")
+	return n, nv*4 + ni*2, err
+}
+
+func (t *trie) printStruct(w io.Writer, handle *trieHandle, name string) (n, sz int, err error) {
+	const msg = "trie{ %sLookup[%d:], %sValues[%d:], %sLookup[:], %sValues[:]}"
+	n, err = fmt.Fprintf(w, msg, name, handle.lookupStart*blockSize, name, handle.valueStart*blockSize, name, name)
+	sz += int(reflect.TypeOf(trie{}).Size())
+	return
+}

vendor/golang.org/x/text/collate/collate.go

@@ -0,0 +1,403 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// TODO: remove hard-coded versions when we have implemented fractional weights.
+// The current implementation is incompatible with later CLDR versions.
+//go:generate go run maketables.go -cldr=23 -unicode=6.2.0
+
+// Package collate contains types for comparing and sorting Unicode strings
+// according to a given collation order.
+package collate // import "golang.org/x/text/collate"
+
+import (
+	"bytes"
+	"strings"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+)
+
+// Collator provides functionality for comparing strings for a given
+// collation order.
+type Collator struct {
+	options
+
+	sorter sorter
+
+	_iter [2]iter
+}
+
+func (c *Collator) iter(i int) *iter {
+	// TODO: evaluate performance for making the second iterator optional.
+	return &c._iter[i]
+}
+
+// Supported returns the list of languages for which collating differs from its parent.
+func Supported() []language.Tag {
+	// TODO: use language.Coverage instead.
+
+	t := make([]language.Tag, len(tags))
+	copy(t, tags)
+	return t
+}
+
+func init() {
+	ids := strings.Split(availableLocales, ",")
+	tags = make([]language.Tag, len(ids))
+	for i, s := range ids {
+		tags[i] = language.Raw.MustParse(s)
+	}
+}
+
+var tags []language.Tag
+
+// New returns a new Collator initialized for the given locale.
+func New(t language.Tag, o ...Option) *Collator {
+	index := colltab.MatchLang(t, tags)
+	c := newCollator(getTable(locales[index]))
+
+	// Set options from the user-supplied tag.
+	c.setFromTag(t)
+
+	// Set the user-supplied options.
+	c.setOptions(o)
+
+	c.init()
+	return c
+}
+
+// NewFromTable returns a new Collator for the given Weighter.
+func NewFromTable(w colltab.Weighter, o ...Option) *Collator {
+	c := newCollator(w)
+	c.setOptions(o)
+	c.init()
+	return c
+}
+
+func (c *Collator) init() {
+	if c.numeric {
+		c.t = colltab.NewNumericWeighter(c.t)
+	}
+	c._iter[0].init(c)
+	c._iter[1].init(c)
+}
+
+// Buffer holds keys generated by Key and KeyString.
+type Buffer struct {
+	buf [4096]byte
+	key []byte
+}
+
+func (b *Buffer) init() {
+	if b.key == nil {
+		b.key = b.buf[:0]
+	}
+}
+
+// Reset clears the buffer from previous results generated by Key and KeyString.
+func (b *Buffer) Reset() {
+	b.key = b.key[:0]
+}
+
+// Compare returns an integer comparing the two byte slices.
+// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
+func (c *Collator) Compare(a, b []byte) int {
+	// TODO: skip identical prefixes once we have a fast way to detect if a rune is
+	// part of a contraction. This would lead to roughly a 10% speedup for the colcmp regtest.
+	c.iter(0).SetInput(a)
+	c.iter(1).SetInput(b)
+	if res := c.compare(); res != 0 {
+		return res
+	}
+	if !c.ignore[colltab.Identity] {
+		return bytes.Compare(a, b)
+	}
+	return 0
+}
+
+// CompareString returns an integer comparing the two strings.
+// The result will be 0 if a==b, -1 if a < b, and +1 if a > b.
+func (c *Collator) CompareString(a, b string) int {
+	// TODO: skip identical prefixes once we have a fast way to detect if a rune is
+	// part of a contraction. This would lead to roughly a 10% speedup for the colcmp regtest.
+	c.iter(0).SetInputString(a)
+	c.iter(1).SetInputString(b)
+	if res := c.compare(); res != 0 {
+		return res
+	}
+	if !c.ignore[colltab.Identity] {
+		if a < b {
+			return -1
+		} else if a > b {
+			return 1
+		}
+	}
+	return 0
+}
+
+func compareLevel(f func(i *iter) int, a, b *iter) int {
+	a.pce = 0
+	b.pce = 0
+	for {
+		va := f(a)
+		vb := f(b)
+		if va != vb {
+			if va < vb {
+				return -1
+			}
+			return 1
+		} else if va == 0 {
+			break
+		}
+	}
+	return 0
+}
+
+func (c *Collator) compare() int {
+	ia, ib := c.iter(0), c.iter(1)
+	// Process primary level
+	if c.alternate != altShifted {
+		// TODO: implement script reordering
+		if res := compareLevel((*iter).nextPrimary, ia, ib); res != 0 {
+			return res
+		}
+	} else {
+		// TODO: handle shifted
+	}
+	if !c.ignore[colltab.Secondary] {
+		f := (*iter).nextSecondary
+		if c.backwards {
+			f = (*iter).prevSecondary
+		}
+		if res := compareLevel(f, ia, ib); res != 0 {
+			return res
+		}
+	}
+	// TODO: special case handling (Danish?)
+	if !c.ignore[colltab.Tertiary] || c.caseLevel {
+		if res := compareLevel((*iter).nextTertiary, ia, ib); res != 0 {
+			return res
+		}
+		if !c.ignore[colltab.Quaternary] {
+			if res := compareLevel((*iter).nextQuaternary, ia, ib); res != 0 {
+				return res
+			}
+		}
+	}
+	return 0
+}
+
+// Key returns the collation key for str.
+// Passing the buffer buf may avoid memory allocations.
+// The returned slice will point to an allocation in Buffer and will remain
+// valid until the next call to buf.Reset().
+func (c *Collator) Key(buf *Buffer, str []byte) []byte {
+	// See https://www.unicode.org/reports/tr10/#Main_Algorithm for more details.
+	buf.init()
+	return c.key(buf, c.getColElems(str))
+}
+
+// KeyFromString returns the collation key for str.
+// Passing the buffer buf may avoid memory allocations.
+// The returned slice will point to an allocation in Buffer and will retain
+// valid until the next call to buf.ResetKeys().
+func (c *Collator) KeyFromString(buf *Buffer, str string) []byte {
+	// See https://www.unicode.org/reports/tr10/#Main_Algorithm for more details.
+	buf.init()
+	return c.key(buf, c.getColElemsString(str))
+}
+
+func (c *Collator) key(buf *Buffer, w []colltab.Elem) []byte {
+	processWeights(c.alternate, c.t.Top(), w)
+	kn := len(buf.key)
+	c.keyFromElems(buf, w)
+	return buf.key[kn:]
+}
+
+func (c *Collator) getColElems(str []byte) []colltab.Elem {
+	i := c.iter(0)
+	i.SetInput(str)
+	for i.Next() {
+	}
+	return i.Elems
+}
+
+func (c *Collator) getColElemsString(str string) []colltab.Elem {
+	i := c.iter(0)
+	i.SetInputString(str)
+	for i.Next() {
+	}
+	return i.Elems
+}
+
+type iter struct {
+	wa [512]colltab.Elem
+
+	colltab.Iter
+	pce int
+}
+
+func (i *iter) init(c *Collator) {
+	i.Weighter = c.t
+	i.Elems = i.wa[:0]
+}
+
+func (i *iter) nextPrimary() int {
+	for {
+		for ; i.pce < i.N; i.pce++ {
+			if v := i.Elems[i.pce].Primary(); v != 0 {
+				i.pce++
+				return v
+			}
+		}
+		if !i.Next() {
+			return 0
+		}
+	}
+	panic("should not reach here")
+}
+
+func (i *iter) nextSecondary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[i.pce].Secondary(); v != 0 {
+			i.pce++
+			return v
+		}
+	}
+	return 0
+}
+
+func (i *iter) prevSecondary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[len(i.Elems)-i.pce-1].Secondary(); v != 0 {
+			i.pce++
+			return v
+		}
+	}
+	return 0
+}
+
+func (i *iter) nextTertiary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[i.pce].Tertiary(); v != 0 {
+			i.pce++
+			return int(v)
+		}
+	}
+	return 0
+}
+
+func (i *iter) nextQuaternary() int {
+	for ; i.pce < len(i.Elems); i.pce++ {
+		if v := i.Elems[i.pce].Quaternary(); v != 0 {
+			i.pce++
+			return v
+		}
+	}
+	return 0
+}
+
+func appendPrimary(key []byte, p int) []byte {
+	// Convert to variable length encoding; supports up to 23 bits.
+	if p <= 0x7FFF {
+		key = append(key, uint8(p>>8), uint8(p))
+	} else {
+		key = append(key, uint8(p>>16)|0x80, uint8(p>>8), uint8(p))
+	}
+	return key
+}
+
+// keyFromElems converts the weights ws to a compact sequence of bytes.
+// The result will be appended to the byte buffer in buf.
+func (c *Collator) keyFromElems(buf *Buffer, ws []colltab.Elem) {
+	for _, v := range ws {
+		if w := v.Primary(); w > 0 {
+			buf.key = appendPrimary(buf.key, w)
+		}
+	}
+	if !c.ignore[colltab.Secondary] {
+		buf.key = append(buf.key, 0, 0)
+		// TODO: we can use one 0 if we can guarantee that all non-zero weights are > 0xFF.
+		if !c.backwards {
+			for _, v := range ws {
+				if w := v.Secondary(); w > 0 {
+					buf.key = append(buf.key, uint8(w>>8), uint8(w))
+				}
+			}
+		} else {
+			for i := len(ws) - 1; i >= 0; i-- {
+				if w := ws[i].Secondary(); w > 0 {
+					buf.key = append(buf.key, uint8(w>>8), uint8(w))
+				}
+			}
+		}
+	} else if c.caseLevel {
+		buf.key = append(buf.key, 0, 0)
+	}
+	if !c.ignore[colltab.Tertiary] || c.caseLevel {
+		buf.key = append(buf.key, 0, 0)
+		for _, v := range ws {
+			if w := v.Tertiary(); w > 0 {
+				buf.key = append(buf.key, uint8(w))
+			}
+		}
+		// Derive the quaternary weights from the options and other levels.
+		// Note that we represent MaxQuaternary as 0xFF. The first byte of the
+		// representation of a primary weight is always smaller than 0xFF,
+		// so using this single byte value will compare correctly.
+		if !c.ignore[colltab.Quaternary] && c.alternate >= altShifted {
+			if c.alternate == altShiftTrimmed {
+				lastNonFFFF := len(buf.key)
+				buf.key = append(buf.key, 0)
+				for _, v := range ws {
+					if w := v.Quaternary(); w == colltab.MaxQuaternary {
+						buf.key = append(buf.key, 0xFF)
+					} else if w > 0 {
+						buf.key = appendPrimary(buf.key, w)
+						lastNonFFFF = len(buf.key)
+					}
+				}
+				buf.key = buf.key[:lastNonFFFF]
+			} else {
+				buf.key = append(buf.key, 0)
+				for _, v := range ws {
+					if w := v.Quaternary(); w == colltab.MaxQuaternary {
+						buf.key = append(buf.key, 0xFF)
+					} else if w > 0 {
+						buf.key = appendPrimary(buf.key, w)
+					}
+				}
+			}
+		}
+	}
+}
+
+func processWeights(vw alternateHandling, top uint32, wa []colltab.Elem) {
+	ignore := false
+	vtop := int(top)
+	switch vw {
+	case altShifted, altShiftTrimmed:
+		for i := range wa {
+			if p := wa[i].Primary(); p <= vtop && p != 0 {
+				wa[i] = colltab.MakeQuaternary(p)
+				ignore = true
+			} else if p == 0 {
+				if ignore {
+					wa[i] = colltab.Ignore
+				}
+			} else {
+				ignore = false
+			}
+		}
+	case altBlanked:
+		for i := range wa {
+			if p := wa[i].Primary(); p <= vtop && (ignore || p != 0) {
+				wa[i] = colltab.Ignore
+				ignore = true
+			} else {
+				ignore = false
+			}
+		}
+	}
+}

vendor/golang.org/x/text/collate/index.go

@@ -0,0 +1,32 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import "golang.org/x/text/internal/colltab"
+
+const blockSize = 64
+
+func getTable(t tableIndex) *colltab.Table {
+	return &colltab.Table{
+		Index: colltab.Trie{
+			Index0:  mainLookup[:][blockSize*t.lookupOffset:],
+			Values0: mainValues[:][blockSize*t.valuesOffset:],
+			Index:   mainLookup[:],
+			Values:  mainValues[:],
+		},
+		ExpandElem:     mainExpandElem[:],
+		ContractTries:  colltab.ContractTrieSet(mainCTEntries[:]),
+		ContractElem:   mainContractElem[:],
+		MaxContractLen: 18,
+		VariableTop:    varTop,
+	}
+}
+
+// tableIndex holds information for constructing a table
+// for a certain locale based on the main table.
+type tableIndex struct {
+	lookupOffset uint32
+	valuesOffset uint32
+}

vendor/golang.org/x/text/collate/maketables.go

@@ -0,0 +1,553 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build ignore
+
+// Collation table generator.
+// Data read from the web.
+
+package main
+
+import (
+	"archive/zip"
+	"bufio"
+	"bytes"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"unicode/utf8"
+
+	"golang.org/x/text/collate"
+	"golang.org/x/text/collate/build"
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/internal/gen"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/cldr"
+)
+
+var (
+	test = flag.Bool("test", false,
+		"test existing tables; can be used to compare web data with package data.")
+	short = flag.Bool("short", false, `Use "short" alternatives, when available.`)
+	draft = flag.Bool("draft", false, `Use draft versions, when available.`)
+	tags  = flag.String("tags", "", "build tags to be included after +build directive")
+	pkg   = flag.String("package", "collate",
+		"the name of the package in which the generated file is to be included")
+
+	tables = flagStringSetAllowAll("tables", "collate", "collate,chars",
+		"comma-spearated list of tables to generate.")
+	exclude = flagStringSet("exclude", "zh2", "",
+		"comma-separated list of languages to exclude.")
+	include = flagStringSet("include", "", "",
+		"comma-separated list of languages to include. Include trumps exclude.")
+	// TODO: Not included: unihan gb2312han zhuyin big5han (for size reasons)
+	// TODO: Not included: traditional (buggy for Bengali)
+	types = flagStringSetAllowAll("types", "standard,phonebook,phonetic,reformed,pinyin,stroke", "",
+		"comma-separated list of types that should be included.")
+)
+
+// stringSet implements an ordered set based on a list.  It implements flag.Value
+// to allow a set to be specified as a comma-separated list.
+type stringSet struct {
+	s        []string
+	allowed  *stringSet
+	dirty    bool // needs compaction if true
+	all      bool
+	allowAll bool
+}
+
+func flagStringSet(name, def, allowed, usage string) *stringSet {
+	ss := &stringSet{}
+	if allowed != "" {
+		usage += fmt.Sprintf(" (allowed values: any of %s)", allowed)
+		ss.allowed = &stringSet{}
+		failOnError(ss.allowed.Set(allowed))
+	}
+	ss.Set(def)
+	flag.Var(ss, name, usage)
+	return ss
+}
+
+func flagStringSetAllowAll(name, def, allowed, usage string) *stringSet {
+	ss := &stringSet{allowAll: true}
+	if allowed == "" {
+		flag.Var(ss, name, usage+fmt.Sprintf(` Use "all" to select all.`))
+	} else {
+		ss.allowed = &stringSet{}
+		failOnError(ss.allowed.Set(allowed))
+		flag.Var(ss, name, usage+fmt.Sprintf(` (allowed values: "all" or any of %s)`, allowed))
+	}
+	ss.Set(def)
+	return ss
+}
+
+func (ss stringSet) Len() int {
+	return len(ss.s)
+}
+
+func (ss stringSet) String() string {
+	return strings.Join(ss.s, ",")
+}
+
+func (ss *stringSet) Set(s string) error {
+	if ss.allowAll && s == "all" {
+		ss.s = nil
+		ss.all = true
+		return nil
+	}
+	ss.s = ss.s[:0]
+	for _, s := range strings.Split(s, ",") {
+		if s := strings.TrimSpace(s); s != "" {
+			if ss.allowed != nil && !ss.allowed.contains(s) {
+				return fmt.Errorf("unsupported value %q; must be one of %s", s, ss.allowed)
+			}
+			ss.add(s)
+		}
+	}
+	ss.compact()
+	return nil
+}
+
+func (ss *stringSet) add(s string) {
+	ss.s = append(ss.s, s)
+	ss.dirty = true
+}
+
+func (ss *stringSet) values() []string {
+	ss.compact()
+	return ss.s
+}
+
+func (ss *stringSet) contains(s string) bool {
+	if ss.all {
+		return true
+	}
+	for _, v := range ss.s {
+		if v == s {
+			return true
+		}
+	}
+	return false
+}
+
+func (ss *stringSet) compact() {
+	if !ss.dirty {
+		return
+	}
+	a := ss.s
+	sort.Strings(a)
+	k := 0
+	for i := 1; i < len(a); i++ {
+		if a[k] != a[i] {
+			a[k+1] = a[i]
+			k++
+		}
+	}
+	ss.s = a[:k+1]
+	ss.dirty = false
+}
+
+func skipLang(l string) bool {
+	if include.Len() > 0 {
+		return !include.contains(l)
+	}
+	return exclude.contains(l)
+}
+
+// altInclude returns a list of alternatives (for the LDML alt attribute)
+// in order of preference.  An empty string in this list indicates the
+// default entry.
+func altInclude() []string {
+	l := []string{}
+	if *short {
+		l = append(l, "short")
+	}
+	l = append(l, "")
+	// TODO: handle draft using cldr.SetDraftLevel
+	if *draft {
+		l = append(l, "proposed")
+	}
+	return l
+}
+
+func failOnError(e error) {
+	if e != nil {
+		log.Panic(e)
+	}
+}
+
+func openArchive() *zip.Reader {
+	f := gen.OpenCLDRCoreZip()
+	buffer, err := ioutil.ReadAll(f)
+	f.Close()
+	failOnError(err)
+	archive, err := zip.NewReader(bytes.NewReader(buffer), int64(len(buffer)))
+	failOnError(err)
+	return archive
+}
+
+// parseUCA parses a Default Unicode Collation Element Table of the format
+// specified in https://www.unicode.org/reports/tr10/#File_Format.
+// It returns the variable top.
+func parseUCA(builder *build.Builder) {
+	var r io.ReadCloser
+	var err error
+	for _, f := range openArchive().File {
+		if strings.HasSuffix(f.Name, "allkeys_CLDR.txt") {
+			r, err = f.Open()
+		}
+	}
+	if r == nil {
+		log.Fatal("File allkeys_CLDR.txt not found in archive.")
+	}
+	failOnError(err)
+	defer r.Close()
+	scanner := bufio.NewScanner(r)
+	colelem := regexp.MustCompile(`\[([.*])([0-9A-F.]+)\]`)
+	for i := 1; scanner.Scan(); i++ {
+		line := scanner.Text()
+		if len(line) == 0 || line[0] == '#' {
+			continue
+		}
+		if line[0] == '@' {
+			// parse properties
+			switch {
+			case strings.HasPrefix(line[1:], "version "):
+				a := strings.Split(line[1:], " ")
+				if a[1] != gen.UnicodeVersion() {
+					log.Fatalf("incompatible version %s; want %s", a[1], gen.UnicodeVersion())
+				}
+			case strings.HasPrefix(line[1:], "backwards "):
+				log.Fatalf("%d: unsupported option backwards", i)
+			default:
+				log.Printf("%d: unknown option %s", i, line[1:])
+			}
+		} else {
+			// parse entries
+			part := strings.Split(line, " ; ")
+			if len(part) != 2 {
+				log.Fatalf("%d: production rule without ';': %v", i, line)
+			}
+			lhs := []rune{}
+			for _, v := range strings.Split(part[0], " ") {
+				if v == "" {
+					continue
+				}
+				lhs = append(lhs, rune(convHex(i, v)))
+			}
+			var n int
+			var vars []int
+			rhs := [][]int{}
+			for i, m := range colelem.FindAllStringSubmatch(part[1], -1) {
+				n += len(m[0])
+				elem := []int{}
+				for _, h := range strings.Split(m[2], ".") {
+					elem = append(elem, convHex(i, h))
+				}
+				if m[1] == "*" {
+					vars = append(vars, i)
+				}
+				rhs = append(rhs, elem)
+			}
+			if len(part[1]) < n+3 || part[1][n+1] != '#' {
+				log.Fatalf("%d: expected comment; found %s", i, part[1][n:])
+			}
+			if *test {
+				testInput.add(string(lhs))
+			}
+			failOnError(builder.Add(lhs, rhs, vars))
+		}
+	}
+	if scanner.Err() != nil {
+		log.Fatal(scanner.Err())
+	}
+}
+
+func convHex(line int, s string) int {
+	r, e := strconv.ParseInt(s, 16, 32)
+	if e != nil {
+		log.Fatalf("%d: %v", line, e)
+	}
+	return int(r)
+}
+
+var testInput = stringSet{}
+
+var charRe = regexp.MustCompile(`&#x([0-9A-F]*);`)
+var tagRe = regexp.MustCompile(`<([a-z_]*)  */>`)
+
+var mainLocales = []string{}
+
+// charsets holds a list of exemplar characters per category.
+type charSets map[string][]string
+
+func (p charSets) fprint(w io.Writer) {
+	fmt.Fprintln(w, "[exN]string{")
+	for i, k := range []string{"", "contractions", "punctuation", "auxiliary", "currencySymbol", "index"} {
+		if set := p[k]; len(set) != 0 {
+			fmt.Fprintf(w, "\t\t%d: %q,\n", i, strings.Join(set, " "))
+		}
+	}
+	fmt.Fprintln(w, "\t},")
+}
+
+var localeChars = make(map[string]charSets)
+
+const exemplarHeader = `
+type exemplarType int
+const (
+	exCharacters exemplarType = iota
+	exContractions
+	exPunctuation
+	exAuxiliary
+	exCurrency
+	exIndex
+	exN
+)
+`
+
+func printExemplarCharacters(w io.Writer) {
+	fmt.Fprintln(w, exemplarHeader)
+	fmt.Fprintln(w, "var exemplarCharacters = map[string][exN]string{")
+	for _, loc := range mainLocales {
+		fmt.Fprintf(w, "\t%q: ", loc)
+		localeChars[loc].fprint(w)
+	}
+	fmt.Fprintln(w, "}")
+}
+
+func decodeCLDR(d *cldr.Decoder) *cldr.CLDR {
+	r := gen.OpenCLDRCoreZip()
+	data, err := d.DecodeZip(r)
+	failOnError(err)
+	return data
+}
+
+// parseMain parses XML files in the main directory of the CLDR core.zip file.
+func parseMain() {
+	d := &cldr.Decoder{}
+	d.SetDirFilter("main")
+	d.SetSectionFilter("characters")
+	data := decodeCLDR(d)
+	for _, loc := range data.Locales() {
+		x := data.RawLDML(loc)
+		if skipLang(x.Identity.Language.Type) {
+			continue
+		}
+		if x.Characters != nil {
+			x, _ = data.LDML(loc)
+			loc = language.Make(loc).String()
+			for _, ec := range x.Characters.ExemplarCharacters {
+				if ec.Draft != "" {
+					continue
+				}
+				if _, ok := localeChars[loc]; !ok {
+					mainLocales = append(mainLocales, loc)
+					localeChars[loc] = make(charSets)
+				}
+				localeChars[loc][ec.Type] = parseCharacters(ec.Data())
+			}
+		}
+	}
+}
+
+func parseCharacters(chars string) []string {
+	parseSingle := func(s string) (r rune, tail string, escaped bool) {
+		if s[0] == '\\' {
+			return rune(s[1]), s[2:], true
+		}
+		r, sz := utf8.DecodeRuneInString(s)
+		return r, s[sz:], false
+	}
+	chars = strings.TrimSpace(chars)
+	if n := len(chars) - 1; chars[n] == ']' && chars[0] == '[' {
+		chars = chars[1:n]
+	}
+	list := []string{}
+	var r, last, end rune
+	for len(chars) > 0 {
+		if chars[0] == '{' { // character sequence
+			buf := []rune{}
+			for chars = chars[1:]; len(chars) > 0; {
+				r, chars, _ = parseSingle(chars)
+				if r == '}' {
+					break
+				}
+				if r == ' ' {
+					log.Fatalf("space not supported in sequence %q", chars)
+				}
+				buf = append(buf, r)
+			}
+			list = append(list, string(buf))
+			last = 0
+		} else { // single character
+			escaped := false
+			r, chars, escaped = parseSingle(chars)
+			if r != ' ' {
+				if r == '-' && !escaped {
+					if last == 0 {
+						log.Fatal("'-' should be preceded by a character")
+					}
+					end, chars, _ = parseSingle(chars)
+					for ; last <= end; last++ {
+						list = append(list, string(last))
+					}
+					last = 0
+				} else {
+					list = append(list, string(r))
+					last = r
+				}
+			}
+		}
+	}
+	return list
+}
+
+var fileRe = regexp.MustCompile(`.*/collation/(.*)\.xml`)
+
+// typeMap translates legacy type keys to their BCP47 equivalent.
+var typeMap = map[string]string{
+	"phonebook":   "phonebk",
+	"traditional": "trad",
+}
+
+// parseCollation parses XML files in the collation directory of the CLDR core.zip file.
+func parseCollation(b *build.Builder) {
+	d := &cldr.Decoder{}
+	d.SetDirFilter("collation")
+	data := decodeCLDR(d)
+	for _, loc := range data.Locales() {
+		x, err := data.LDML(loc)
+		failOnError(err)
+		if skipLang(x.Identity.Language.Type) {
+			continue
+		}
+		cs := x.Collations.Collation
+		sl := cldr.MakeSlice(&cs)
+		if len(types.s) == 0 {
+			sl.SelectAnyOf("type", x.Collations.Default())
+		} else if !types.all {
+			sl.SelectAnyOf("type", types.s...)
+		}
+		sl.SelectOnePerGroup("alt", altInclude())
+
+		for _, c := range cs {
+			id, err := language.Parse(loc)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "invalid locale: %q", err)
+				continue
+			}
+			// Support both old- and new-style defaults.
+			d := c.Type
+			if x.Collations.DefaultCollation == nil {
+				d = x.Collations.Default()
+			} else {
+				d = x.Collations.DefaultCollation.Data()
+			}
+			// We assume tables are being built either for search or collation,
+			// but not both. For search the default is always "search".
+			if d != c.Type && c.Type != "search" {
+				typ := c.Type
+				if len(c.Type) > 8 {
+					typ = typeMap[c.Type]
+				}
+				id, err = id.SetTypeForKey("co", typ)
+				failOnError(err)
+			}
+			t := b.Tailoring(id)
+			c.Process(processor{t})
+		}
+	}
+}
+
+type processor struct {
+	t *build.Tailoring
+}
+
+func (p processor) Reset(anchor string, before int) (err error) {
+	if before != 0 {
+		err = p.t.SetAnchorBefore(anchor)
+	} else {
+		err = p.t.SetAnchor(anchor)
+	}
+	failOnError(err)
+	return nil
+}
+
+func (p processor) Insert(level int, str, context, extend string) error {
+	str = context + str
+	if *test {
+		testInput.add(str)
+	}
+	// TODO: mimic bug in old maketables: remove.
+	err := p.t.Insert(colltab.Level(level-1), str, context+extend)
+	failOnError(err)
+	return nil
+}
+
+func (p processor) Index(id string) {
+}
+
+func testCollator(c *collate.Collator) {
+	c0 := collate.New(language.Und)
+
+	// iterator over all characters for all locales and check
+	// whether Key is equal.
+	buf := collate.Buffer{}
+
+	// Add all common and not too uncommon runes to the test set.
+	for i := rune(0); i < 0x30000; i++ {
+		testInput.add(string(i))
+	}
+	for i := rune(0xE0000); i < 0xF0000; i++ {
+		testInput.add(string(i))
+	}
+	for _, str := range testInput.values() {
+		k0 := c0.KeyFromString(&buf, str)
+		k := c.KeyFromString(&buf, str)
+		if !bytes.Equal(k0, k) {
+			failOnError(fmt.Errorf("test:%U: keys differ (%x vs %x)", []rune(str), k0, k))
+		}
+		buf.Reset()
+	}
+	fmt.Println("PASS")
+}
+
+func main() {
+	gen.Init()
+	b := build.NewBuilder()
+	parseUCA(b)
+	if tables.contains("chars") {
+		parseMain()
+	}
+	parseCollation(b)
+
+	c, err := b.Build()
+	failOnError(err)
+
+	if *test {
+		testCollator(collate.NewFromTable(c))
+	} else {
+		w := &bytes.Buffer{}
+
+		gen.WriteUnicodeVersion(w)
+		gen.WriteCLDRVersion(w)
+
+		if tables.contains("collate") {
+			_, err = b.Print(w)
+			failOnError(err)
+		}
+		if tables.contains("chars") {
+			printExemplarCharacters(w)
+		}
+		gen.WriteGoFile("tables.go", *pkg, w.Bytes())
+	}
+}

vendor/golang.org/x/text/collate/option.go

@@ -0,0 +1,239 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"sort"
+
+	"golang.org/x/text/internal/colltab"
+	"golang.org/x/text/language"
+	"golang.org/x/text/unicode/norm"
+)
+
+// newCollator creates a new collator with default options configured.
+func newCollator(t colltab.Weighter) *Collator {
+	// Initialize a collator with default options.
+	c := &Collator{
+		options: options{
+			ignore: [colltab.NumLevels]bool{
+				colltab.Quaternary: true,
+				colltab.Identity:   true,
+			},
+			f: norm.NFD,
+			t: t,
+		},
+	}
+
+	// TODO: store vt in tags or remove.
+	c.variableTop = t.Top()
+
+	return c
+}
+
+// An Option is used to change the behavior of a Collator. Options override the
+// settings passed through the locale identifier.
+type Option struct {
+	priority int
+	f        func(o *options)
+}
+
+type prioritizedOptions []Option
+
+func (p prioritizedOptions) Len() int {
+	return len(p)
+}
+
+func (p prioritizedOptions) Swap(i, j int) {
+	p[i], p[j] = p[j], p[i]
+}
+
+func (p prioritizedOptions) Less(i, j int) bool {
+	return p[i].priority < p[j].priority
+}
+
+type options struct {
+	// ignore specifies which levels to ignore.
+	ignore [colltab.NumLevels]bool
+
+	// caseLevel is true if there is an additional level of case matching
+	// between the secondary and tertiary levels.
+	caseLevel bool
+
+	// backwards specifies the order of sorting at the secondary level.
+	// This option exists predominantly to support reverse sorting of accents in French.
+	backwards bool
+
+	// numeric specifies whether any sequence of decimal digits (category is Nd)
+	// is sorted at a primary level with its numeric value.
+	// For example, "A-21" < "A-123".
+	// This option is set by wrapping the main Weighter with NewNumericWeighter.
+	numeric bool
+
+	// alternate specifies an alternative handling of variables.
+	alternate alternateHandling
+
+	// variableTop is the largest primary value that is considered to be
+	// variable.
+	variableTop uint32
+
+	t colltab.Weighter
+
+	f norm.Form
+}
+
+func (o *options) setOptions(opts []Option) {
+	sort.Sort(prioritizedOptions(opts))
+	for _, x := range opts {
+		x.f(o)
+	}
+}
+
+// OptionsFromTag extracts the BCP47 collation options from the tag and
+// configures a collator accordingly. These options are set before any other
+// option.
+func OptionsFromTag(t language.Tag) Option {
+	return Option{0, func(o *options) {
+		o.setFromTag(t)
+	}}
+}
+
+func (o *options) setFromTag(t language.Tag) {
+	o.caseLevel = ldmlBool(t, o.caseLevel, "kc")
+	o.backwards = ldmlBool(t, o.backwards, "kb")
+	o.numeric = ldmlBool(t, o.numeric, "kn")
+
+	// Extract settings from the BCP47 u extension.
+	switch t.TypeForKey("ks") { // strength
+	case "level1":
+		o.ignore[colltab.Secondary] = true
+		o.ignore[colltab.Tertiary] = true
+	case "level2":
+		o.ignore[colltab.Tertiary] = true
+	case "level3", "":
+		// The default.
+	case "level4":
+		o.ignore[colltab.Quaternary] = false
+	case "identic":
+		o.ignore[colltab.Quaternary] = false
+		o.ignore[colltab.Identity] = false
+	}
+
+	switch t.TypeForKey("ka") {
+	case "shifted":
+		o.alternate = altShifted
+	// The following two types are not official BCP47, but we support them to
+	// give access to this otherwise hidden functionality. The name blanked is
+	// derived from the LDML name blanked and posix reflects the main use of
+	// the shift-trimmed option.
+	case "blanked":
+		o.alternate = altBlanked
+	case "posix":
+		o.alternate = altShiftTrimmed
+	}
+
+	// TODO: caseFirst ("kf"), reorder ("kr"), and maybe variableTop ("vt").
+
+	// Not used:
+	// - normalization ("kk", not necessary for this implementation)
+	// - hiraganaQuatenary ("kh", obsolete)
+}
+
+func ldmlBool(t language.Tag, old bool, key string) bool {
+	switch t.TypeForKey(key) {
+	case "true":
+		return true
+	case "false":
+		return false
+	default:
+		return old
+	}
+}
+
+var (
+	// IgnoreCase sets case-insensitive comparison.
+	IgnoreCase Option = ignoreCase
+	ignoreCase        = Option{3, ignoreCaseF}
+
+	// IgnoreDiacritics causes diacritical marks to be ignored. ("o" == "ö").
+	IgnoreDiacritics Option = ignoreDiacritics
+	ignoreDiacritics        = Option{3, ignoreDiacriticsF}
+
+	// IgnoreWidth causes full-width characters to match their half-width
+	// equivalents.
+	IgnoreWidth Option = ignoreWidth
+	ignoreWidth        = Option{2, ignoreWidthF}
+
+	// Loose sets the collator to ignore diacritics, case and width.
+	Loose Option = loose
+	loose        = Option{4, looseF}
+
+	// Force ordering if strings are equivalent but not equal.
+	Force Option = force
+	force        = Option{5, forceF}
+
+	// Numeric specifies that numbers should sort numerically ("2" < "12").
+	Numeric Option = numeric
+	numeric        = Option{5, numericF}
+)
+
+func ignoreWidthF(o *options) {
+	o.ignore[colltab.Tertiary] = true
+	o.caseLevel = true
+}
+
+func ignoreDiacriticsF(o *options) {
+	o.ignore[colltab.Secondary] = true
+}
+
+func ignoreCaseF(o *options) {
+	o.ignore[colltab.Tertiary] = true
+	o.caseLevel = false
+}
+
+func looseF(o *options) {
+	ignoreWidthF(o)
+	ignoreDiacriticsF(o)
+	ignoreCaseF(o)
+}
+
+func forceF(o *options) {
+	o.ignore[colltab.Identity] = false
+}
+
+func numericF(o *options) { o.numeric = true }
+
+// Reorder overrides the pre-defined ordering of scripts and character sets.
+func Reorder(s ...string) Option {
+	// TODO: need fractional weights to implement this.
+	panic("TODO: implement")
+}
+
+// TODO: consider making these public again. These options cannot be fully
+// specified in BCP47, so an API interface seems warranted. Still a higher-level
+// interface would be nice (e.g. a POSIX option for enabling altShiftTrimmed)
+
+// alternateHandling identifies the various ways in which variables are handled.
+// A rune with a primary weight lower than the variable top is considered a
+// variable.
+// See https://www.unicode.org/reports/tr10/#Variable_Weighting for details.
+type alternateHandling int
+
+const (
+	// altNonIgnorable turns off special handling of variables.
+	altNonIgnorable alternateHandling = iota
+
+	// altBlanked sets variables and all subsequent primary ignorables to be
+	// ignorable at all levels. This is identical to removing all variables
+	// and subsequent primary ignorables from the input.
+	altBlanked
+
+	// altShifted sets variables to be ignorable for levels one through three and
+	// adds a fourth level based on the values of the ignored levels.
+	altShifted
+
+	// altShiftTrimmed is a slight variant of altShifted that is used to
+	// emulate POSIX.
+	altShiftTrimmed
+)

vendor/golang.org/x/text/collate/sort.go

@@ -0,0 +1,81 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package collate
+
+import (
+	"bytes"
+	"sort"
+)
+
+const (
+	maxSortBuffer  = 40960
+	maxSortEntries = 4096
+)
+
+type swapper interface {
+	Swap(i, j int)
+}
+
+type sorter struct {
+	buf  *Buffer
+	keys [][]byte
+	src  swapper
+}
+
+func (s *sorter) init(n int) {
+	if s.buf == nil {
+		s.buf = &Buffer{}
+		s.buf.init()
+	}
+	if cap(s.keys) < n {
+		s.keys = make([][]byte, n)
+	}
+	s.keys = s.keys[0:n]
+}
+
+func (s *sorter) sort(src swapper) {
+	s.src = src
+	sort.Sort(s)
+}
+
+func (s sorter) Len() int {
+	return len(s.keys)
+}
+
+func (s sorter) Less(i, j int) bool {
+	return bytes.Compare(s.keys[i], s.keys[j]) == -1
+}
+
+func (s sorter) Swap(i, j int) {
+	s.keys[i], s.keys[j] = s.keys[j], s.keys[i]
+	s.src.Swap(i, j)
+}
+
+// A Lister can be sorted by Collator's Sort method.
+type Lister interface {
+	Len() int
+	Swap(i, j int)
+	// Bytes returns the bytes of the text at index i.
+	Bytes(i int) []byte
+}
+
+// Sort uses sort.Sort to sort the strings represented by x using the rules of c.
+func (c *Collator) Sort(x Lister) {
+	n := x.Len()
+	c.sorter.init(n)
+	for i := 0; i < n; i++ {
+		c.sorter.keys[i] = c.Key(c.sorter.buf, x.Bytes(i))
+	}
+	c.sorter.sort(x)
+}
+
+// SortStrings uses sort.Sort to sort the strings in x using the rules of c.
+func (c *Collator) SortStrings(x []string) {
+	c.sorter.init(len(x))
+	for i, s := range x {
+		c.sorter.keys[i] = c.KeyFromString(c.sorter.buf, s)
+	}
+	c.sorter.sort(sort.StringSlice(x))
+}