mirror of
https://github.com/openfaas/faasd.git
synced 2025-06-19 12:36:38 +00:00
Update dependencies
* Updates netlink/netns * Updates x/sys, arkade and apimachinery Build passes, minor updates. Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>
This commit is contained in:
Alex Ellis (OpenFaaS Ltd)
309
vendor/github.com/klauspost/compress/zstd/seqdec.go
generated
vendored
309
vendor/github.com/klauspost/compress/zstd/seqdec.go
generated
vendored
@ -20,6 +20,10 @@ type seq struct {
|
||||
llCode, mlCode, ofCode uint8
|
||||
}
|
||||
|
||||
type seqVals struct {
|
||||
ll, ml, mo int
|
||||
}
|
||||
|
||||
func (s seq) String() string {
|
||||
if s.offset <= 3 {
|
||||
if s.offset == 0 {
|
||||
@ -61,16 +65,19 @@ type sequenceDecs struct {
|
||||
offsets sequenceDec
|
||||
matchLengths sequenceDec
|
||||
prevOffset [3]int
|
||||
hist []byte
|
||||
dict []byte
|
||||
literals []byte
|
||||
out []byte
|
||||
nSeqs int
|
||||
br *bitReader
|
||||
seqSize int
|
||||
windowSize int
|
||||
maxBits uint8
|
||||
maxSyncLen uint64
|
||||
}
|
||||
|
||||
// initialize all 3 decoders from the stream input.
|
||||
func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []byte) error {
|
||||
func (s *sequenceDecs) initialize(br *bitReader, hist *history, out []byte) error {
|
||||
if err := s.litLengths.init(br); err != nil {
|
||||
return errors.New("litLengths:" + err.Error())
|
||||
}
|
||||
@ -80,8 +87,7 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []
|
||||
if err := s.matchLengths.init(br); err != nil {
|
||||
return errors.New("matchLengths:" + err.Error())
|
||||
}
|
||||
s.literals = literals
|
||||
s.hist = hist.b
|
||||
s.br = br
|
||||
s.prevOffset = hist.recentOffsets
|
||||
s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits
|
||||
s.windowSize = hist.windowSize
|
||||
@ -93,12 +99,127 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out []
|
||||
return nil
|
||||
}
|
||||
|
||||
// execute will execute the decoded sequence with the provided history.
|
||||
// The sequence must be evaluated before being sent.
|
||||
func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error {
|
||||
if len(s.dict) == 0 {
|
||||
return s.executeSimple(seqs, hist)
|
||||
}
|
||||
|
||||
// Ensure we have enough output size...
|
||||
if len(s.out)+s.seqSize > cap(s.out) {
|
||||
addBytes := s.seqSize + len(s.out)
|
||||
s.out = append(s.out, make([]byte, addBytes)...)
|
||||
s.out = s.out[:len(s.out)-addBytes]
|
||||
}
|
||||
|
||||
if debugDecoder {
|
||||
printf("Execute %d seqs with hist %d, dict %d, literals: %d into %d bytes\n", len(seqs), len(hist), len(s.dict), len(s.literals), s.seqSize)
|
||||
}
|
||||
|
||||
var t = len(s.out)
|
||||
out := s.out[:t+s.seqSize]
|
||||
|
||||
for _, seq := range seqs {
|
||||
// Add literals
|
||||
copy(out[t:], s.literals[:seq.ll])
|
||||
t += seq.ll
|
||||
s.literals = s.literals[seq.ll:]
|
||||
|
||||
// Copy from dictionary...
|
||||
if seq.mo > t+len(hist) || seq.mo > s.windowSize {
|
||||
if len(s.dict) == 0 {
|
||||
return fmt.Errorf("match offset (%d) bigger than current history (%d)", seq.mo, t+len(hist))
|
||||
}
|
||||
|
||||
// we may be in dictionary.
|
||||
dictO := len(s.dict) - (seq.mo - (t + len(hist)))
|
||||
if dictO < 0 || dictO >= len(s.dict) {
|
||||
return fmt.Errorf("match offset (%d) bigger than current history+dict (%d)", seq.mo, t+len(hist)+len(s.dict))
|
||||
}
|
||||
end := dictO + seq.ml
|
||||
if end > len(s.dict) {
|
||||
n := len(s.dict) - dictO
|
||||
copy(out[t:], s.dict[dictO:])
|
||||
t += n
|
||||
seq.ml -= n
|
||||
} else {
|
||||
copy(out[t:], s.dict[dictO:end])
|
||||
t += end - dictO
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Copy from history.
|
||||
if v := seq.mo - t; v > 0 {
|
||||
// v is the start position in history from end.
|
||||
start := len(hist) - v
|
||||
if seq.ml > v {
|
||||
// Some goes into current block.
|
||||
// Copy remainder of history
|
||||
copy(out[t:], hist[start:])
|
||||
t += v
|
||||
seq.ml -= v
|
||||
} else {
|
||||
copy(out[t:], hist[start:start+seq.ml])
|
||||
t += seq.ml
|
||||
continue
|
||||
}
|
||||
}
|
||||
// We must be in current buffer now
|
||||
if seq.ml > 0 {
|
||||
start := t - seq.mo
|
||||
if seq.ml <= t-start {
|
||||
// No overlap
|
||||
copy(out[t:], out[start:start+seq.ml])
|
||||
t += seq.ml
|
||||
continue
|
||||
} else {
|
||||
// Overlapping copy
|
||||
// Extend destination slice and copy one byte at the time.
|
||||
src := out[start : start+seq.ml]
|
||||
dst := out[t:]
|
||||
dst = dst[:len(src)]
|
||||
t += len(src)
|
||||
// Destination is the space we just added.
|
||||
for i := range src {
|
||||
dst[i] = src[i]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add final literals
|
||||
copy(out[t:], s.literals)
|
||||
if debugDecoder {
|
||||
t += len(s.literals)
|
||||
if t != len(out) {
|
||||
panic(fmt.Errorf("length mismatch, want %d, got %d, ss: %d", len(out), t, s.seqSize))
|
||||
}
|
||||
}
|
||||
s.out = out
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// decode sequences from the stream with the provided history.
|
||||
func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
func (s *sequenceDecs) decodeSync(hist []byte) error {
|
||||
supported, err := s.decodeSyncSimple(hist)
|
||||
if supported {
|
||||
return err
|
||||
}
|
||||
|
||||
br := s.br
|
||||
seqs := s.nSeqs
|
||||
startSize := len(s.out)
|
||||
// Grab full sizes tables, to avoid bounds checks.
|
||||
llTable, mlTable, ofTable := s.litLengths.fse.dt[:maxTablesize], s.matchLengths.fse.dt[:maxTablesize], s.offsets.fse.dt[:maxTablesize]
|
||||
llState, mlState, ofState := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state
|
||||
out := s.out
|
||||
maxBlockSize := maxCompressedBlockSize
|
||||
if s.windowSize < maxBlockSize {
|
||||
maxBlockSize = s.windowSize
|
||||
}
|
||||
|
||||
for i := seqs - 1; i >= 0; i-- {
|
||||
if br.overread() {
|
||||
@ -151,7 +272,7 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
|
||||
if temp == 0 {
|
||||
// 0 is not valid; input is corrupted; force offset to 1
|
||||
println("temp was 0")
|
||||
println("WARNING: temp was 0")
|
||||
temp = 1
|
||||
}
|
||||
|
||||
@ -176,51 +297,49 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
if ll > len(s.literals) {
|
||||
return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals))
|
||||
}
|
||||
size := ll + ml + len(s.out)
|
||||
size := ll + ml + len(out)
|
||||
if size-startSize > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size", size)
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", size-startSize, maxBlockSize)
|
||||
}
|
||||
if size > cap(s.out) {
|
||||
if size > cap(out) {
|
||||
// Not enough size, which can happen under high volume block streaming conditions
|
||||
// but could be if destination slice is too small for sync operations.
|
||||
// over-allocating here can create a large amount of GC pressure so we try to keep
|
||||
// it as contained as possible
|
||||
used := len(s.out) - startSize
|
||||
used := len(out) - startSize
|
||||
addBytes := 256 + ll + ml + used>>2
|
||||
// Clamp to max block size.
|
||||
if used+addBytes > maxBlockSize {
|
||||
addBytes = maxBlockSize - used
|
||||
}
|
||||
s.out = append(s.out, make([]byte, addBytes)...)
|
||||
s.out = s.out[:len(s.out)-addBytes]
|
||||
out = append(out, make([]byte, addBytes)...)
|
||||
out = out[:len(out)-addBytes]
|
||||
}
|
||||
if ml > maxMatchLen {
|
||||
return fmt.Errorf("match len (%d) bigger than max allowed length", ml)
|
||||
}
|
||||
|
||||
// Add literals
|
||||
s.out = append(s.out, s.literals[:ll]...)
|
||||
out = append(out, s.literals[:ll]...)
|
||||
s.literals = s.literals[ll:]
|
||||
out := s.out
|
||||
|
||||
if mo == 0 && ml > 0 {
|
||||
return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml)
|
||||
}
|
||||
|
||||
if mo > len(s.out)+len(hist) || mo > s.windowSize {
|
||||
if mo > len(out)+len(hist) || mo > s.windowSize {
|
||||
if len(s.dict) == 0 {
|
||||
return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist))
|
||||
return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize)
|
||||
}
|
||||
|
||||
// we may be in dictionary.
|
||||
dictO := len(s.dict) - (mo - (len(s.out) + len(hist)))
|
||||
dictO := len(s.dict) - (mo - (len(out) + len(hist)))
|
||||
if dictO < 0 || dictO >= len(s.dict) {
|
||||
return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist))
|
||||
return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(out)+len(hist)-startSize)
|
||||
}
|
||||
end := dictO + ml
|
||||
if end > len(s.dict) {
|
||||
out = append(out, s.dict[dictO:]...)
|
||||
mo -= len(s.dict) - dictO
|
||||
ml -= len(s.dict) - dictO
|
||||
} else {
|
||||
out = append(out, s.dict[dictO:end]...)
|
||||
@ -231,26 +350,25 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
|
||||
// Copy from history.
|
||||
// TODO: Blocks without history could be made to ignore this completely.
|
||||
if v := mo - len(s.out); v > 0 {
|
||||
if v := mo - len(out); v > 0 {
|
||||
// v is the start position in history from end.
|
||||
start := len(s.hist) - v
|
||||
start := len(hist) - v
|
||||
if ml > v {
|
||||
// Some goes into current block.
|
||||
// Copy remainder of history
|
||||
out = append(out, s.hist[start:]...)
|
||||
mo -= v
|
||||
out = append(out, hist[start:]...)
|
||||
ml -= v
|
||||
} else {
|
||||
out = append(out, s.hist[start:start+ml]...)
|
||||
out = append(out, hist[start:start+ml]...)
|
||||
ml = 0
|
||||
}
|
||||
}
|
||||
// We must be in current buffer now
|
||||
if ml > 0 {
|
||||
start := len(s.out) - mo
|
||||
if ml <= len(s.out)-start {
|
||||
start := len(out) - mo
|
||||
if ml <= len(out)-start {
|
||||
// No overlap
|
||||
out = append(out, s.out[start:start+ml]...)
|
||||
out = append(out, out[start:start+ml]...)
|
||||
} else {
|
||||
// Overlapping copy
|
||||
// Extend destination slice and copy one byte at the time.
|
||||
@ -264,7 +382,6 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
}
|
||||
}
|
||||
}
|
||||
s.out = out
|
||||
if i == 0 {
|
||||
// This is the last sequence, so we shouldn't update state.
|
||||
break
|
||||
@ -278,7 +395,8 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
mlState = mlTable[mlState.newState()&maxTableMask]
|
||||
ofState = ofTable[ofState.newState()&maxTableMask]
|
||||
} else {
|
||||
bits := br.getBitsFast(nBits)
|
||||
bits := br.get32BitsFast(nBits)
|
||||
|
||||
lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31))
|
||||
llState = llTable[(llState.newState()+lowBits)&maxTableMask]
|
||||
|
||||
@ -291,19 +409,14 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Add final literals
|
||||
s.out = append(s.out, s.literals...)
|
||||
return nil
|
||||
}
|
||||
// Check if space for literals
|
||||
if size := len(s.literals) + len(s.out) - startSize; size > maxBlockSize {
|
||||
return fmt.Errorf("output (%d) bigger than max block size (%d)", size, maxBlockSize)
|
||||
}
|
||||
|
||||
// update states, at least 27 bits must be available.
|
||||
func (s *sequenceDecs) update(br *bitReader) {
|
||||
// Max 8 bits
|
||||
s.litLengths.state.next(br)
|
||||
// Max 9 bits
|
||||
s.matchLengths.state.next(br)
|
||||
// Max 8 bits
|
||||
s.offsets.state.next(br)
|
||||
// Add final literals
|
||||
s.out = append(out, s.literals...)
|
||||
return br.close()
|
||||
}
|
||||
|
||||
var bitMask [16]uint16
|
||||
@ -314,87 +427,6 @@ func init() {
|
||||
}
|
||||
}
|
||||
|
||||
// update states, at least 27 bits must be available.
|
||||
func (s *sequenceDecs) updateAlt(br *bitReader) {
|
||||
// Update all 3 states at once. Approx 20% faster.
|
||||
a, b, c := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state
|
||||
|
||||
nBits := a.nbBits() + b.nbBits() + c.nbBits()
|
||||
if nBits == 0 {
|
||||
s.litLengths.state.state = s.litLengths.state.dt[a.newState()]
|
||||
s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()]
|
||||
s.offsets.state.state = s.offsets.state.dt[c.newState()]
|
||||
return
|
||||
}
|
||||
bits := br.getBitsFast(nBits)
|
||||
lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31))
|
||||
s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits]
|
||||
|
||||
lowBits = uint16(bits >> (c.nbBits() & 31))
|
||||
lowBits &= bitMask[b.nbBits()&15]
|
||||
s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()+lowBits]
|
||||
|
||||
lowBits = uint16(bits) & bitMask[c.nbBits()&15]
|
||||
s.offsets.state.state = s.offsets.state.dt[c.newState()+lowBits]
|
||||
}
|
||||
|
||||
// nextFast will return new states when there are at least 4 unused bytes left on the stream when done.
|
||||
func (s *sequenceDecs) nextFast(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) {
|
||||
// Final will not read from stream.
|
||||
ll, llB := llState.final()
|
||||
ml, mlB := mlState.final()
|
||||
mo, moB := ofState.final()
|
||||
|
||||
// extra bits are stored in reverse order.
|
||||
br.fillFast()
|
||||
mo += br.getBits(moB)
|
||||
if s.maxBits > 32 {
|
||||
br.fillFast()
|
||||
}
|
||||
ml += br.getBits(mlB)
|
||||
ll += br.getBits(llB)
|
||||
|
||||
if moB > 1 {
|
||||
s.prevOffset[2] = s.prevOffset[1]
|
||||
s.prevOffset[1] = s.prevOffset[0]
|
||||
s.prevOffset[0] = mo
|
||||
return
|
||||
}
|
||||
// mo = s.adjustOffset(mo, ll, moB)
|
||||
// Inlined for rather big speedup
|
||||
if ll == 0 {
|
||||
// There is an exception though, when current sequence's literals_length = 0.
|
||||
// In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2,
|
||||
// an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte.
|
||||
mo++
|
||||
}
|
||||
|
||||
if mo == 0 {
|
||||
mo = s.prevOffset[0]
|
||||
return
|
||||
}
|
||||
var temp int
|
||||
if mo == 3 {
|
||||
temp = s.prevOffset[0] - 1
|
||||
} else {
|
||||
temp = s.prevOffset[mo]
|
||||
}
|
||||
|
||||
if temp == 0 {
|
||||
// 0 is not valid; input is corrupted; force offset to 1
|
||||
println("temp was 0")
|
||||
temp = 1
|
||||
}
|
||||
|
||||
if mo != 1 {
|
||||
s.prevOffset[2] = s.prevOffset[1]
|
||||
}
|
||||
s.prevOffset[1] = s.prevOffset[0]
|
||||
s.prevOffset[0] = temp
|
||||
mo = temp
|
||||
return
|
||||
}
|
||||
|
||||
func (s *sequenceDecs) next(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) {
|
||||
// Final will not read from stream.
|
||||
ll, llB := llState.final()
|
||||
@ -457,36 +489,3 @@ func (s *sequenceDecs) adjustOffset(offset, litLen int, offsetB uint8) int {
|
||||
s.prevOffset[0] = temp
|
||||
return temp
|
||||
}
|
||||
|
||||
// mergeHistory will merge history.
|
||||
func (s *sequenceDecs) mergeHistory(hist *sequenceDecs) (*sequenceDecs, error) {
|
||||
for i := uint(0); i < 3; i++ {
|
||||
var sNew, sHist *sequenceDec
|
||||
switch i {
|
||||
default:
|
||||
// same as "case 0":
|
||||
sNew = &s.litLengths
|
||||
sHist = &hist.litLengths
|
||||
case 1:
|
||||
sNew = &s.offsets
|
||||
sHist = &hist.offsets
|
||||
case 2:
|
||||
sNew = &s.matchLengths
|
||||
sHist = &hist.matchLengths
|
||||
}
|
||||
if sNew.repeat {
|
||||
if sHist.fse == nil {
|
||||
return nil, fmt.Errorf("sequence stream %d, repeat requested, but no history", i)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if sNew.fse == nil {
|
||||
return nil, fmt.Errorf("sequence stream %d, no fse found", i)
|
||||
}
|
||||
if sHist.fse != nil && !sHist.fse.preDefined {
|
||||
fseDecoderPool.Put(sHist.fse)
|
||||
}
|
||||
sHist.fse = sNew.fse
|
||||
}
|
||||
return hist, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user