Migrate to containerd v1.7.0 and update dependencies

* Updates containerd to v1.7.0 and new binary for 32-bit
Arm OSes.
* Updates Go dependencies - openfaas and external

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

vendor/github.com/Microsoft/hcsshim/internal/hcs/callback.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (

vendor/github.com/Microsoft/hcsshim/internal/hcs/doc.go

@@ -0,0 +1 @@
+package hcs

vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (
@@ -51,6 +53,9 @@ var (
 	// ErrUnexpectedValue is an error encountered when hcs returns an invalid value
 	ErrUnexpectedValue = errors.New("unexpected value returned from hcs")
 
+	// ErrOperationDenied is an error when hcs attempts an operation that is explicitly denied
+	ErrOperationDenied = errors.New("operation denied")
+
 	// ErrVmcomputeAlreadyStopped is an error encountered when a shutdown or terminate request is made on a stopped container
 	ErrVmcomputeAlreadyStopped = syscall.Errno(0xc0370110)
 
@@ -82,7 +87,7 @@ var (
 	// ErrProcessAlreadyStopped is returned by hcs if the process we're trying to kill has already been stopped.
 	ErrProcessAlreadyStopped = syscall.Errno(0x8037011f)
 
-	// ErrInvalidHandle is an error that can be encountrered when querying the properties of a compute system when the handle to that
+	// ErrInvalidHandle is an error that can be encountered when querying the properties of a compute system when the handle to that
 	// compute system has already been closed.
 	ErrInvalidHandle = syscall.Errno(0x6)
 )
@@ -152,33 +157,38 @@ func (e *HcsError) Error() string {
 	return s
 }
 
+func (e *HcsError) Is(target error) bool {
+	return errors.Is(e.Err, target)
+}
+
+// unwrap isnt really needed, but helpful convince function
+
+func (e *HcsError) Unwrap() error {
+	return e.Err
+}
+
+// Deprecated: net.Error.Temporary is deprecated.
 func (e *HcsError) Temporary() bool {
-	err, ok := e.Err.(net.Error)
-	return ok && err.Temporary()
+	err := e.netError()
+	return (err != nil) && err.Temporary()
 }
 
 func (e *HcsError) Timeout() bool {
-	err, ok := e.Err.(net.Error)
-	return ok && err.Timeout()
+	err := e.netError()
+	return (err != nil) && err.Timeout()
 }
 
-// ProcessError is an error encountered in HCS during an operation on a Process object
-type ProcessError struct {
-	SystemID string
-	Pid      int
-	Op       string
-	Err      error
-	Events   []ErrorEvent
+func (e *HcsError) netError() (err net.Error) {
+	if errors.As(e.Unwrap(), &err) {
+		return err
+	}
+	return nil
 }
 
-var _ net.Error = &ProcessError{}
-
 // SystemError is an error encountered in HCS during an operation on a Container object
 type SystemError struct {
-	ID     string
-	Op     string
-	Err    error
-	Events []ErrorEvent
+	HcsError
+	ID string
 }
 
 var _ net.Error = &SystemError{}
@@ -191,29 +201,32 @@ func (e *SystemError) Error() string {
 	return s
 }
 
-func (e *SystemError) Temporary() bool {
-	err, ok := e.Err.(net.Error)
-	return ok && err.Temporary()
-}
-
-func (e *SystemError) Timeout() bool {
-	err, ok := e.Err.(net.Error)
-	return ok && err.Timeout()
-}
-
 func makeSystemError(system *System, op string, err error, events []ErrorEvent) error {
 	// Don't double wrap errors
-	if _, ok := err.(*SystemError); ok {
+	var e *SystemError
+	if errors.As(err, &e) {
 		return err
 	}
+
 	return &SystemError{
-		ID:     system.ID(),
-		Op:     op,
-		Err:    err,
-		Events: events,
+		ID: system.ID(),
+		HcsError: HcsError{
+			Op:     op,
+			Err:    err,
+			Events: events,
+		},
 	}
 }
 
+// ProcessError is an error encountered in HCS during an operation on a Process object
+type ProcessError struct {
+	HcsError
+	SystemID string
+	Pid      int
+}
+
+var _ net.Error = &ProcessError{}
+
 func (e *ProcessError) Error() string {
 	s := fmt.Sprintf("%s %s:%d: %s", e.Op, e.SystemID, e.Pid, e.Err.Error())
 	for _, ev := range e.Events {
@@ -222,27 +235,20 @@ func (e *ProcessError) Error() string {
 	return s
 }
 
-func (e *ProcessError) Temporary() bool {
-	err, ok := e.Err.(net.Error)
-	return ok && err.Temporary()
-}
-
-func (e *ProcessError) Timeout() bool {
-	err, ok := e.Err.(net.Error)
-	return ok && err.Timeout()
-}
-
 func makeProcessError(process *Process, op string, err error, events []ErrorEvent) error {
 	// Don't double wrap errors
-	if _, ok := err.(*ProcessError); ok {
+	var e *ProcessError
+	if errors.As(err, &e) {
 		return err
 	}
 	return &ProcessError{
 		Pid:      process.Pid(),
 		SystemID: process.SystemID(),
-		Op:       op,
-		Err:      err,
-		Events:   events,
+		HcsError: HcsError{
+			Op:     op,
+			Err:    err,
+			Events: events,
+		},
 	}
 }
 
@@ -251,41 +257,41 @@ func makeProcessError(process *Process, op string, err error, events []ErrorEven
 // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
 // will currently return true when the error is ErrElementNotFound.
 func IsNotExist(err error) bool {
-	err = getInnerError(err)
-	return err == ErrComputeSystemDoesNotExist ||
-		err == ErrElementNotFound
+	return IsAny(err, ErrComputeSystemDoesNotExist, ErrElementNotFound)
 }
 
 // IsErrorInvalidHandle checks whether the error is the result of an operation carried
 // out on a handle that is invalid/closed. This error popped up while trying to query
 // stats on a container in the process of being stopped.
 func IsErrorInvalidHandle(err error) bool {
-	err = getInnerError(err)
-	return err == ErrInvalidHandle
+	return errors.Is(err, ErrInvalidHandle)
 }
 
 // IsAlreadyClosed checks if an error is caused by the Container or Process having been
 // already closed by a call to the Close() method.
 func IsAlreadyClosed(err error) bool {
-	err = getInnerError(err)
-	return err == ErrAlreadyClosed
+	return errors.Is(err, ErrAlreadyClosed)
 }
 
 // IsPending returns a boolean indicating whether the error is that
 // the requested operation is being completed in the background.
 func IsPending(err error) bool {
-	err = getInnerError(err)
-	return err == ErrVmcomputeOperationPending
+	return errors.Is(err, ErrVmcomputeOperationPending)
 }
 
 // IsTimeout returns a boolean indicating whether the error is caused by
 // a timeout waiting for the operation to complete.
 func IsTimeout(err error) bool {
-	if err, ok := err.(net.Error); ok && err.Timeout() {
+	// HcsError and co. implement Timeout regardless of whether the errors they wrap do,
+	// so `errors.As(err, net.Error)`` will always be true.
+	// Using `errors.As(err.Unwrap(), net.Err)` wont work for general errors.
+	// So first check if there an `ErrTimeout` in the chain, then convert to a net error.
+	if errors.Is(err, ErrTimeout) {
 		return true
 	}
-	err = getInnerError(err)
-	return err == ErrTimeout
+
+	var nerr net.Error
+	return errors.As(err, &nerr) && nerr.Timeout()
 }
 
 // IsAlreadyStopped returns a boolean indicating whether the error is caused by
@@ -294,10 +300,7 @@ func IsTimeout(err error) bool {
 // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
 // will currently return true when the error is ErrElementNotFound.
 func IsAlreadyStopped(err error) bool {
-	err = getInnerError(err)
-	return err == ErrVmcomputeAlreadyStopped ||
-		err == ErrProcessAlreadyStopped ||
-		err == ErrElementNotFound
+	return IsAny(err, ErrVmcomputeAlreadyStopped, ErrProcessAlreadyStopped, ErrElementNotFound)
 }
 
 // IsNotSupported returns a boolean indicating whether the error is caused by
@@ -306,38 +309,28 @@ func IsAlreadyStopped(err error) bool {
 // ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported or ErrVmcomputeUnknownMessage
 // is thrown from the Platform
 func IsNotSupported(err error) bool {
-	err = getInnerError(err)
 	// If Platform doesn't recognize or support the request sent, below errors are seen
-	return err == ErrVmcomputeInvalidJSON ||
-		err == ErrInvalidData ||
-		err == ErrNotSupported ||
-		err == ErrVmcomputeUnknownMessage
+	return IsAny(err, ErrVmcomputeInvalidJSON, ErrInvalidData, ErrNotSupported, ErrVmcomputeUnknownMessage)
 }
 
 // IsOperationInvalidState returns true when err is caused by
 // `ErrVmcomputeOperationInvalidState`.
 func IsOperationInvalidState(err error) bool {
-	err = getInnerError(err)
-	return err == ErrVmcomputeOperationInvalidState
+	return errors.Is(err, ErrVmcomputeOperationInvalidState)
 }
 
 // IsAccessIsDenied returns true when err is caused by
 // `ErrVmcomputeOperationAccessIsDenied`.
 func IsAccessIsDenied(err error) bool {
-	err = getInnerError(err)
-	return err == ErrVmcomputeOperationAccessIsDenied
+	return errors.Is(err, ErrVmcomputeOperationAccessIsDenied)
 }
 
-func getInnerError(err error) error {
-	switch pe := err.(type) {
-	case nil:
-		return nil
-	case *HcsError:
-		err = pe.Err
-	case *SystemError:
-		err = pe.Err
-	case *ProcessError:
-		err = pe.Err
+// IsAny is a vectorized version of [errors.Is], it returns true if err is one of targets.
+func IsAny(err error, targets ...error) bool {
+	for _, e := range targets {
+		if errors.Is(err, e) {
+			return true
+		}
 	}
-	return err
+	return false
 }

vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (
@@ -10,6 +12,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/Microsoft/hcsshim/internal/cow"
 	"github.com/Microsoft/hcsshim/internal/log"
 	"github.com/Microsoft/hcsshim/internal/oc"
 	"github.com/Microsoft/hcsshim/internal/vmcompute"
@@ -36,6 +39,8 @@ type Process struct {
 	waitError      error
 }
 
+var _ cow.Process = &Process{}
+
 func newProcess(process vmcompute.HcsProcess, processID int, computeSystem *System) *Process {
 	return &Process{
 		handle:    process,
@@ -89,10 +94,7 @@ func (process *Process) processSignalResult(ctx context.Context, err error) (boo
 	case nil:
 		return true, nil
 	case ErrVmcomputeOperationInvalidState, ErrComputeSystemDoesNotExist, ErrElementNotFound:
-		select {
-		case <-process.waitBlock:
-			// The process exit notification has already arrived.
-		default:
+		if !process.stopped() {
 			// The process should be gone, but we have not received the notification.
 			// After a second, force unblock the process wait to work around a possible
 			// deadlock in the HCS.
@@ -114,9 +116,9 @@ func (process *Process) processSignalResult(ctx context.Context, err error) (boo
 
 // Signal signals the process with `options`.
 //
-// For LCOW `guestrequest.SignalProcessOptionsLCOW`.
+// For LCOW `guestresource.SignalProcessOptionsLCOW`.
 //
-// For WCOW `guestrequest.SignalProcessOptionsWCOW`.
+// For WCOW `guestresource.SignalProcessOptionsWCOW`.
 func (process *Process) Signal(ctx context.Context, options interface{}) (bool, error) {
 	process.handleLock.RLock()
 	defer process.handleLock.RUnlock()
@@ -152,6 +154,10 @@ func (process *Process) Kill(ctx context.Context) (bool, error) {
 		return false, makeProcessError(process, operation, ErrAlreadyClosed, nil)
 	}
 
+	if process.stopped() {
+		return false, makeProcessError(process, operation, ErrProcessAlreadyStopped, nil)
+	}
+
 	if process.killSignalDelivered {
 		// A kill signal has already been sent to this process. Sending a second
 		// one offers no real benefit, as processes cannot stop themselves from
@@ -161,7 +167,39 @@ func (process *Process) Kill(ctx context.Context) (bool, error) {
 		return true, nil
 	}
 
-	resultJSON, err := vmcompute.HcsTerminateProcess(ctx, process.handle)
+	// HCS serializes the signals sent to a target pid per compute system handle.
+	// To avoid SIGKILL being serialized behind other signals, we open a new compute
+	// system handle to deliver the kill signal.
+	// If the calls to opening a new compute system handle fail, we forcefully
+	// terminate the container itself so that no container is left behind
+	hcsSystem, err := OpenComputeSystem(ctx, process.system.id)
+	if err != nil {
+		// log error and force termination of container
+		log.G(ctx).WithField("err", err).Error("OpenComputeSystem() call failed")
+		err = process.system.Terminate(ctx)
+		// if the Terminate() call itself ever failed, log and return error
+		if err != nil {
+			log.G(ctx).WithField("err", err).Error("Terminate() call failed")
+			return false, err
+		}
+		process.system.Close()
+		return true, nil
+	}
+	defer hcsSystem.Close()
+
+	newProcessHandle, err := hcsSystem.OpenProcess(ctx, process.Pid())
+	if err != nil {
+		// Return true only if the target process has either already
+		// exited, or does not exist.
+		if IsAlreadyStopped(err) {
+			return true, nil
+		} else {
+			return false, err
+		}
+	}
+	defer newProcessHandle.Close()
+
+	resultJSON, err := vmcompute.HcsTerminateProcess(ctx, newProcessHandle.handle)
 	if err != nil {
 		// We still need to check these two cases, as processes may still be killed by an
 		// external actor (human operator, OOM, random script etc).
@@ -185,9 +223,9 @@ func (process *Process) Kill(ctx context.Context) (bool, error) {
 		}
 	}
 	events := processHcsResult(ctx, resultJSON)
-	delivered, err := process.processSignalResult(ctx, err)
+	delivered, err := newProcessHandle.processSignalResult(ctx, err)
 	if err != nil {
-		err = makeProcessError(process, operation, err, events)
+		err = makeProcessError(newProcessHandle, operation, err, events)
 	}
 
 	process.killSignalDelivered = delivered
@@ -201,7 +239,7 @@ func (process *Process) Kill(ctx context.Context) (bool, error) {
 // call multiple times.
 func (process *Process) waitBackground() {
 	operation := "hcs::Process::waitBackground"
-	ctx, span := trace.StartSpan(context.Background(), operation)
+	ctx, span := oc.StartSpan(context.Background(), operation)
 	defer span.End()
 	span.AddAttributes(
 		trace.StringAttribute("cid", process.SystemID()),
@@ -227,12 +265,12 @@ func (process *Process) waitBackground() {
 			propertiesJSON, resultJSON, err = vmcompute.HcsGetProcessProperties(ctx, process.handle)
 			events := processHcsResult(ctx, resultJSON)
 			if err != nil {
-				err = makeProcessError(process, operation, err, events) //nolint:ineffassign
+				err = makeProcessError(process, operation, err, events)
 			} else {
 				properties := &processStatus{}
 				err = json.Unmarshal([]byte(propertiesJSON), properties)
 				if err != nil {
-					err = makeProcessError(process, operation, err, nil) //nolint:ineffassign
+					err = makeProcessError(process, operation, err, nil)
 				} else {
 					if properties.LastWaitResult != 0 {
 						log.G(ctx).WithField("wait-result", properties.LastWaitResult).Warning("non-zero last wait result")
@@ -254,12 +292,22 @@ func (process *Process) waitBackground() {
 }
 
 // Wait waits for the process to exit. If the process has already exited returns
-// the pervious error (if any).
+// the previous error (if any).
 func (process *Process) Wait() error {
 	<-process.waitBlock
 	return process.waitError
 }
 
+// Exited returns if the process has stopped
+func (process *Process) stopped() bool {
+	select {
+	case <-process.waitBlock:
+		return true
+	default:
+		return false
+	}
+}
+
 // ResizeConsole resizes the console of the process.
 func (process *Process) ResizeConsole(ctx context.Context, width, height uint16) error {
 	process.handleLock.RLock()
@@ -296,15 +344,13 @@ func (process *Process) ResizeConsole(ctx context.Context, width, height uint16)
 // ExitCode returns the exit code of the process. The process must have
 // already terminated.
 func (process *Process) ExitCode() (int, error) {
-	select {
-	case <-process.waitBlock:
-		if process.waitError != nil {
-			return -1, process.waitError
-		}
-		return process.exitCode, nil
-	default:
+	if !process.stopped() {
 		return -1, makeProcessError(process, "hcs::Process::ExitCode", ErrInvalidProcessState, nil)
 	}
+	if process.waitError != nil {
+		return -1, process.waitError
+	}
+	return process.exitCode, nil
 }
 
 // StdioLegacy returns the stdin, stdout, and stderr pipes, respectively. Closing
@@ -312,7 +358,7 @@ func (process *Process) ExitCode() (int, error) {
 // are the responsibility of the caller to close.
 func (process *Process) StdioLegacy() (_ io.WriteCloser, _ io.ReadCloser, _ io.ReadCloser, err error) {
 	operation := "hcs::Process::StdioLegacy"
-	ctx, span := trace.StartSpan(context.Background(), operation)
+	ctx, span := oc.StartSpan(context.Background(), operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(
@@ -350,7 +396,7 @@ func (process *Process) StdioLegacy() (_ io.WriteCloser, _ io.ReadCloser, _ io.R
 }
 
 // Stdio returns the stdin, stdout, and stderr pipes, respectively.
-// To close them, close the process handle.
+// To close them, close the process handle, or use the `CloseStd*` functions.
 func (process *Process) Stdio() (stdin io.Writer, stdout, stderr io.Reader) {
 	process.stdioLock.Lock()
 	defer process.stdioLock.Unlock()
@@ -359,46 +405,57 @@ func (process *Process) Stdio() (stdin io.Writer, stdout, stderr io.Reader) {
 
 // CloseStdin closes the write side of the stdin pipe so that the process is
 // notified on the read side that there is no more data in stdin.
-func (process *Process) CloseStdin(ctx context.Context) error {
+func (process *Process) CloseStdin(ctx context.Context) (err error) {
+	operation := "hcs::Process::CloseStdin"
+	ctx, span := trace.StartSpan(ctx, operation)
+	defer span.End()
+	defer func() { oc.SetSpanStatus(span, err) }()
+	span.AddAttributes(
+		trace.StringAttribute("cid", process.SystemID()),
+		trace.Int64Attribute("pid", int64(process.processID)))
+
 	process.handleLock.RLock()
 	defer process.handleLock.RUnlock()
 
-	operation := "hcs::Process::CloseStdin"
-
 	if process.handle == 0 {
 		return makeProcessError(process, operation, ErrAlreadyClosed, nil)
 	}
 
-	modifyRequest := processModifyRequest{
-		Operation: modifyCloseHandle,
-		CloseHandle: &closeHandle{
-			Handle: stdIn,
-		},
-	}
-
-	modifyRequestb, err := json.Marshal(modifyRequest)
-	if err != nil {
-		return err
-	}
-
-	resultJSON, err := vmcompute.HcsModifyProcess(ctx, process.handle, string(modifyRequestb))
-	events := processHcsResult(ctx, resultJSON)
-	if err != nil {
-		return makeProcessError(process, operation, err, events)
-	}
-
 	process.stdioLock.Lock()
-	if process.stdin != nil {
-		process.stdin.Close()
-		process.stdin = nil
+	defer process.stdioLock.Unlock()
+	if process.stdin == nil {
+		return nil
 	}
-	process.stdioLock.Unlock()
+
+	//HcsModifyProcess request to close stdin will fail if the process has already exited
+	if !process.stopped() {
+		modifyRequest := processModifyRequest{
+			Operation: modifyCloseHandle,
+			CloseHandle: &closeHandle{
+				Handle: stdIn,
+			},
+		}
+
+		modifyRequestb, err := json.Marshal(modifyRequest)
+		if err != nil {
+			return err
+		}
+
+		resultJSON, err := vmcompute.HcsModifyProcess(ctx, process.handle, string(modifyRequestb))
+		events := processHcsResult(ctx, resultJSON)
+		if err != nil {
+			return makeProcessError(process, operation, err, events)
+		}
+	}
+
+	process.stdin.Close()
+	process.stdin = nil
 
 	return nil
 }
 
 func (process *Process) CloseStdout(ctx context.Context) (err error) {
-	ctx, span := trace.StartSpan(ctx, "hcs::Process::CloseStdout") //nolint:ineffassign,staticcheck
+	ctx, span := oc.StartSpan(ctx, "hcs::Process::CloseStdout") //nolint:ineffassign,staticcheck
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(
@@ -422,7 +479,7 @@ func (process *Process) CloseStdout(ctx context.Context) (err error) {
 }
 
 func (process *Process) CloseStderr(ctx context.Context) (err error) {
-	ctx, span := trace.StartSpan(ctx, "hcs::Process::CloseStderr") //nolint:ineffassign,staticcheck
+	ctx, span := oc.StartSpan(ctx, "hcs::Process::CloseStderr") //nolint:ineffassign,staticcheck
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(
@@ -441,7 +498,6 @@ func (process *Process) CloseStderr(ctx context.Context) (err error) {
 	if process.stderr != nil {
 		process.stderr.Close()
 		process.stderr = nil
-
 	}
 	return nil
 }
@@ -450,7 +506,7 @@ func (process *Process) CloseStderr(ctx context.Context) (err error) {
 // or wait on it.
 func (process *Process) Close() (err error) {
 	operation := "hcs::Process::Close"
-	ctx, span := trace.StartSpan(context.Background(), operation)
+	ctx, span := oc.StartSpan(context.Background(), operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema1/schema1.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package schema1
 
 import (
@@ -101,7 +103,7 @@ type ContainerConfig struct {
 	HvRuntime                   *HvRuntime          `json:",omitempty"` // Hyper-V container settings. Used by Hyper-V containers only. Format ImagePath=%root%\BaseLayerID\UtilityVM
 	Servicing                   bool                `json:",omitempty"` // True if this container is for servicing
 	AllowUnqualifiedDNSQuery    bool                `json:",omitempty"` // True to allow unqualified DNS name resolution
-	DNSSearchList               string              `json:",omitempty"` // Comma seperated list of DNS suffixes to use for name resolution
+	DNSSearchList               string              `json:",omitempty"` // Comma separated list of DNS suffixes to use for name resolution
 	ContainerType               string              `json:",omitempty"` // "Linux" for Linux containers on Windows. Omitted otherwise.
 	TerminateOnLastHandleClosed bool                `json:",omitempty"` // Should HCS terminate the container once all handles have been closed
 	MappedVirtualDisks          []MappedVirtualDisk `json:",omitempty"` // Array of virtual disks to mount at start

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/cpu_group_property.go

@@ -9,6 +9,14 @@
 
 package hcsschema
 
+type CPUGroupPropertyCode uint32
+
+const (
+	CPUCapacityProperty           = 0x00010000
+	CPUSchedulingPriorityProperty = 0x00020000
+	IdleLPReserveProperty         = 0x00030000
+)
+
 type CpuGroupProperty struct {
 	PropertyCode  uint32 `json:"PropertyCode,omitempty"`
 	PropertyValue uint32 `json:"PropertyValue,omitempty"`

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/debug_options.go

@@ -0,0 +1,22 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.1
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type DebugOptions struct {
+	// BugcheckSavedStateFileName is the path for the file in which the guest VM state will be saved when
+	// the guest crashes.
+	BugcheckSavedStateFileName string `json:"BugcheckSavedStateFileName,omitempty"`
+	// BugcheckNoCrashdumpSavedStateFileName is the path of the file in which the guest VM state will be
+	// saved when the guest crashes but the guest isn't able to generate the crash dump. This usually
+	// happens in early boot failures.
+	BugcheckNoCrashdumpSavedStateFileName string `json:"BugcheckNoCrashdumpSavedStateFileName,omitempty"`
+	TripleFaultSavedStateFileName         string `json:"TripleFaultSavedStateFileName,omitempty"`
+	FirmwareDumpFileName                  string `json:"FirmwareDumpFileName,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/guest_state.go

@@ -14,6 +14,9 @@ type GuestState struct {
 	//  The path to an existing file uses for persistent guest state storage.  An empty string indicates the system should initialize new transient, in-memory guest state.
 	GuestStateFilePath string `json:"GuestStateFilePath,omitempty"`
 
+	//  The guest state file type affected by different guest isolation modes - whether a file or block storage.
+	GuestStateFileType string `json:"GuestStateFileType,omitempty"`
+
 	//  The path to an existing file for persistent runtime state storage.  An empty string indicates the system should initialize new transient, in-memory runtime state.
 	RuntimeStateFilePath string `json:"RuntimeStateFilePath,omitempty"`
 

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/isolation_settings.go

@@ -0,0 +1,21 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type IsolationSettings struct {
+	// Guest isolation type options to decide virtual trust levels of virtual machine
+	IsolationType string `json:"IsolationType,omitempty"`
+	// Configuration to debug HCL layer for HCS VM TODO: Task 31102306: Miss the way to prevent the exposure of private debug configuration in HCS TODO: Think about the secret configurations which are private in VMMS VM (only edit by hvsedit)
+	DebugHost string `json:"DebugHost,omitempty"`
+	DebugPort int64  `json:"DebugPort,omitempty"`
+	// Optional data passed by host on isolated virtual machine start
+	LaunchData string `json:"LaunchData,omitempty"`
+	HclEnabled bool   `json:"HclEnabled,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/modify_setting_request.go

@@ -9,10 +9,12 @@
 
 package hcsschema
 
+import "github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
+
 type ModifySettingRequest struct {
 	ResourcePath string `json:"ResourcePath,omitempty"`
 
-	RequestType string `json:"RequestType,omitempty"`
+	RequestType guestrequest.RequestType `json:"RequestType,omitempty"` // NOTE: Swagger generated as string. Locally updated.
 
 	Settings interface{} `json:"Settings,omitempty"` // NOTE: Swagger generated as *interface{}. Locally updated
 

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/security_settings.go

@@ -0,0 +1,16 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.4
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type SecuritySettings struct {
+	// Enablement of Trusted Platform Module on the computer system
+	EnableTpm bool               `json:"EnableTpm,omitempty"`
+	Isolation *IsolationSettings `json:"Isolation,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/system_time.go

@@ -0,0 +1,28 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.1
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type SystemTime struct {
+	Year int32 `json:"Year,omitempty"`
+
+	Month int32 `json:"Month,omitempty"`
+
+	DayOfWeek int32 `json:"DayOfWeek,omitempty"`
+
+	Day int32 `json:"Day,omitempty"`
+
+	Hour int32 `json:"Hour,omitempty"`
+
+	Minute int32 `json:"Minute,omitempty"`
+
+	Second int32 `json:"Second,omitempty"`
+
+	Milliseconds int32 `json:"Milliseconds,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/time_zone_information.go

@@ -0,0 +1,26 @@
+/*
+ * HCS API
+ *
+ * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen)
+ *
+ * API version: 2.1
+ * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git)
+ */
+
+package hcsschema
+
+type TimeZoneInformation struct {
+	Bias int32 `json:"Bias,omitempty"`
+
+	StandardName string `json:"StandardName,omitempty"`
+
+	StandardDate *SystemTime `json:"StandardDate,omitempty"`
+
+	StandardBias int32 `json:"StandardBias,omitempty"`
+
+	DaylightName string `json:"DaylightName,omitempty"`
+
+	DaylightDate *SystemTime `json:"DaylightDate,omitempty"`
+
+	DaylightBias int32 `json:"DaylightBias,omitempty"`
+}

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/uefi.go

@@ -12,6 +12,8 @@ package hcsschema
 type Uefi struct {
 	EnableDebugger bool `json:"EnableDebugger,omitempty"`
 
+	ApplySecureBootTemplate string `json:"ApplySecureBootTemplate,omitempty"`
+
 	SecureBootTemplateId string `json:"SecureBootTemplateId,omitempty"`
 
 	BootThis *UefiBootEntry `json:"BootThis,omitempty"`

vendor/github.com/Microsoft/hcsshim/internal/hcs/schema2/virtual_machine.go

@@ -29,4 +29,8 @@ type VirtualMachine struct {
 	StorageQoS *StorageQoS `json:"StorageQoS,omitempty"`
 
 	GuestConnection *GuestConnection `json:"GuestConnection,omitempty"`
+
+	SecuritySettings *SecuritySettings `json:"SecuritySettings,omitempty"`
+
+	DebugOptions *DebugOptions `json:"DebugOptions,omitempty"`
 }

vendor/github.com/Microsoft/hcsshim/internal/hcs/service.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (

vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (
@@ -37,6 +39,9 @@ type System struct {
 	startTime      time.Time
 }
 
+var _ cow.Container = &System{}
+var _ cow.ProcessHost = &System{}
+
 func newSystem(id string) *System {
 	return &System{
 		id:        id,
@@ -55,7 +60,7 @@ func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface in
 
 	// hcsCreateComputeSystemContext is an async operation. Start the outer span
 	// here to measure the full create time.
-	ctx, span := trace.StartSpan(ctx, operation)
+	ctx, span := oc.StartSpan(ctx, operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute("cid", id))
@@ -89,7 +94,8 @@ func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface in
 		}
 	}
 
-	events, err := processAsyncHcsResult(ctx, createError, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemCreateCompleted, &timeout.SystemCreate)
+	events, err := processAsyncHcsResult(ctx, createError, resultJSON, computeSystem.callbackNumber,
+		hcsNotificationSystemCreateCompleted, &timeout.SystemCreate)
 	if err != nil {
 		if err == ErrTimeout {
 			// Terminate the compute system if it still exists. We're okay to
@@ -190,7 +196,7 @@ func (computeSystem *System) Start(ctx context.Context) (err error) {
 
 	// hcsStartComputeSystemContext is an async operation. Start the outer span
 	// here to measure the full start time.
-	ctx, span := trace.StartSpan(ctx, operation)
+	ctx, span := oc.StartSpan(ctx, operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
@@ -198,12 +204,15 @@ func (computeSystem *System) Start(ctx context.Context) (err error) {
 	computeSystem.handleLock.RLock()
 	defer computeSystem.handleLock.RUnlock()
 
+	// prevent starting an exited system because waitblock we do not recreate waitBlock
+	// or rerun waitBackground, so we have no way to be notified of it closing again
 	if computeSystem.handle == 0 {
 		return makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil)
 	}
 
 	resultJSON, err := vmcompute.HcsStartComputeSystem(ctx, computeSystem.handle, "")
-	events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart)
+	events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber,
+		hcsNotificationSystemStartCompleted, &timeout.SystemStart)
 	if err != nil {
 		return makeSystemError(computeSystem, operation, err, events)
 	}
@@ -223,7 +232,7 @@ func (computeSystem *System) Shutdown(ctx context.Context) error {
 
 	operation := "hcs::System::Shutdown"
 
-	if computeSystem.handle == 0 {
+	if computeSystem.handle == 0 || computeSystem.stopped() {
 		return nil
 	}
 
@@ -244,7 +253,7 @@ func (computeSystem *System) Terminate(ctx context.Context) error {
 
 	operation := "hcs::System::Terminate"
 
-	if computeSystem.handle == 0 {
+	if computeSystem.handle == 0 || computeSystem.stopped() {
 		return nil
 	}
 
@@ -265,7 +274,7 @@ func (computeSystem *System) Terminate(ctx context.Context) error {
 // safe to call multiple times.
 func (computeSystem *System) waitBackground() {
 	operation := "hcs::System::waitBackground"
-	ctx, span := trace.StartSpan(context.Background(), operation)
+	ctx, span := oc.StartSpan(context.Background(), operation)
 	defer span.End()
 	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
 
@@ -287,24 +296,40 @@ func (computeSystem *System) waitBackground() {
 	oc.SetSpanStatus(span, err)
 }
 
+func (computeSystem *System) WaitChannel() <-chan struct{} {
+	return computeSystem.waitBlock
+}
+
+func (computeSystem *System) WaitError() error {
+	return computeSystem.waitError
+}
+
 // Wait synchronously waits for the compute system to shutdown or terminate. If
 // the compute system has already exited returns the previous error (if any).
 func (computeSystem *System) Wait() error {
-	<-computeSystem.waitBlock
-	return computeSystem.waitError
+	<-computeSystem.WaitChannel()
+	return computeSystem.WaitError()
+}
+
+// stopped returns true if the compute system stopped.
+func (computeSystem *System) stopped() bool {
+	select {
+	case <-computeSystem.waitBlock:
+		return true
+	default:
+	}
+	return false
 }
 
 // ExitError returns an error describing the reason the compute system terminated.
 func (computeSystem *System) ExitError() error {
-	select {
-	case <-computeSystem.waitBlock:
-		if computeSystem.waitError != nil {
-			return computeSystem.waitError
-		}
-		return computeSystem.exitError
-	default:
+	if !computeSystem.stopped() {
 		return errors.New("container not exited")
 	}
+	if computeSystem.waitError != nil {
+		return computeSystem.waitError
+	}
+	return computeSystem.exitError
 }
 
 // Properties returns the requested container properties targeting a V1 schema container.
@@ -314,6 +339,10 @@ func (computeSystem *System) Properties(ctx context.Context, types ...schema1.Pr
 
 	operation := "hcs::System::Properties"
 
+	if computeSystem.handle == 0 {
+		return nil, makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil)
+	}
+
 	queryBytes, err := json.Marshal(schema1.PropertyQuery{PropertyTypes: types})
 	if err != nil {
 		return nil, makeSystemError(computeSystem, operation, err, nil)
@@ -341,7 +370,11 @@ func (computeSystem *System) Properties(ctx context.Context, types ...schema1.Pr
 // failed to be queried they will be tallied up and returned in as the first return value. Failures on
 // query are NOT considered errors; the only failure case for this method is if the containers job object
 // cannot be opened.
-func (computeSystem *System) queryInProc(ctx context.Context, props *hcsschema.Properties, types []hcsschema.PropertyType) ([]hcsschema.PropertyType, error) {
+func (computeSystem *System) queryInProc(
+	ctx context.Context,
+	props *hcsschema.Properties,
+	types []hcsschema.PropertyType,
+) ([]hcsschema.PropertyType, error) {
 	// In the future we can make use of some new functionality in the HCS that allows you
 	// to pass a job object for HCS to use for the container. Currently, the only way we'll
 	// be able to open the job/silo is if we're running as SYSTEM.
@@ -407,7 +440,7 @@ func (computeSystem *System) statisticsInProc(job *jobobject.JobObject) (*hcssch
 	// as well which isn't great and is wasted work to fetch.
 	//
 	// HCS only let's you grab statistics in an all or nothing fashion, so we can't just grab the private
-	// working set ourselves and ask for everything else seperately. The optimization we can make here is
+	// working set ourselves and ask for everything else separately. The optimization we can make here is
 	// to open the silo ourselves and do the same queries for the rest of the info, as well as calculating
 	// the private working set in a more efficient manner by:
 	//
@@ -447,6 +480,10 @@ func (computeSystem *System) statisticsInProc(job *jobobject.JobObject) (*hcssch
 func (computeSystem *System) hcsPropertiesV2Query(ctx context.Context, types []hcsschema.PropertyType) (*hcsschema.Properties, error) {
 	operation := "hcs::System::PropertiesV2"
 
+	if computeSystem.handle == 0 {
+		return nil, makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil)
+	}
+
 	queryBytes, err := json.Marshal(hcsschema.PropertyQuery{PropertyTypes: types})
 	if err != nil {
 		return nil, makeSystemError(computeSystem, operation, err, nil)
@@ -495,7 +532,7 @@ func (computeSystem *System) PropertiesV2(ctx context.Context, types ...hcsschem
 	if err == nil && len(fallbackTypes) == 0 {
 		return properties, nil
 	} else if err != nil {
-		logEntry.WithError(fmt.Errorf("failed to query compute system properties in-proc: %w", err))
+		logEntry = logEntry.WithError(fmt.Errorf("failed to query compute system properties in-proc: %w", err))
 		fallbackTypes = types
 	}
 
@@ -527,9 +564,9 @@ func (computeSystem *System) PropertiesV2(ctx context.Context, types ...hcsschem
 func (computeSystem *System) Pause(ctx context.Context) (err error) {
 	operation := "hcs::System::Pause"
 
-	// hcsPauseComputeSystemContext is an async peration. Start the outer span
+	// hcsPauseComputeSystemContext is an async operation. Start the outer span
 	// here to measure the full pause time.
-	ctx, span := trace.StartSpan(ctx, operation)
+	ctx, span := oc.StartSpan(ctx, operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
@@ -542,7 +579,8 @@ func (computeSystem *System) Pause(ctx context.Context) (err error) {
 	}
 
 	resultJSON, err := vmcompute.HcsPauseComputeSystem(ctx, computeSystem.handle, "")
-	events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause)
+	events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber,
+		hcsNotificationSystemPauseCompleted, &timeout.SystemPause)
 	if err != nil {
 		return makeSystemError(computeSystem, operation, err, events)
 	}
@@ -556,7 +594,7 @@ func (computeSystem *System) Resume(ctx context.Context) (err error) {
 
 	// hcsResumeComputeSystemContext is an async operation. Start the outer span
 	// here to measure the full restore time.
-	ctx, span := trace.StartSpan(ctx, operation)
+	ctx, span := oc.StartSpan(ctx, operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
@@ -569,7 +607,8 @@ func (computeSystem *System) Resume(ctx context.Context) (err error) {
 	}
 
 	resultJSON, err := vmcompute.HcsResumeComputeSystem(ctx, computeSystem.handle, "")
-	events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume)
+	events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber,
+		hcsNotificationSystemResumeCompleted, &timeout.SystemResume)
 	if err != nil {
 		return makeSystemError(computeSystem, operation, err, events)
 	}
@@ -581,9 +620,9 @@ func (computeSystem *System) Resume(ctx context.Context) (err error) {
 func (computeSystem *System) Save(ctx context.Context, options interface{}) (err error) {
 	operation := "hcs::System::Save"
 
-	// hcsSaveComputeSystemContext is an async peration. Start the outer span
+	// hcsSaveComputeSystemContext is an async operation. Start the outer span
 	// here to measure the full save time.
-	ctx, span := trace.StartSpan(ctx, operation)
+	ctx, span := oc.StartSpan(ctx, operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
@@ -601,7 +640,8 @@ func (computeSystem *System) Save(ctx context.Context, options interface{}) (err
 	}
 
 	result, err := vmcompute.HcsSaveComputeSystem(ctx, computeSystem.handle, string(saveOptions))
-	events, err := processAsyncHcsResult(ctx, err, result, computeSystem.callbackNumber, hcsNotificationSystemSaveCompleted, &timeout.SystemSave)
+	events, err := processAsyncHcsResult(ctx, err, result, computeSystem.callbackNumber,
+		hcsNotificationSystemSaveCompleted, &timeout.SystemSave)
 	if err != nil {
 		return makeSystemError(computeSystem, operation, err, events)
 	}
@@ -626,6 +666,11 @@ func (computeSystem *System) createProcess(ctx context.Context, operation string
 	processInfo, processHandle, resultJSON, err := vmcompute.HcsCreateProcess(ctx, computeSystem.handle, configuration)
 	events := processHcsResult(ctx, resultJSON)
 	if err != nil {
+		if v2, ok := c.(*hcsschema.ProcessParameters); ok {
+			operation += ": " + v2.CommandLine
+		} else if v1, ok := c.(*schema1.ProcessConfig); ok {
+			operation += ": " + v1.CommandLine
+		}
 		return nil, nil, makeSystemError(computeSystem, operation, err, events)
 	}
 
@@ -692,7 +737,7 @@ func (computeSystem *System) OpenProcess(ctx context.Context, pid int) (*Process
 // Close cleans up any state associated with the compute system but does not terminate or wait for it.
 func (computeSystem *System) Close() (err error) {
 	operation := "hcs::System::Close"
-	ctx, span := trace.StartSpan(context.Background(), operation)
+	ctx, span := oc.StartSpan(context.Background(), operation)
 	defer span.End()
 	defer func() { oc.SetSpanStatus(span, err) }()
 	span.AddAttributes(trace.StringAttribute("cid", computeSystem.id))
@@ -735,7 +780,8 @@ func (computeSystem *System) registerCallback(ctx context.Context) error {
 	callbackMap[callbackNumber] = callbackContext
 	callbackMapLock.Unlock()
 
-	callbackHandle, err := vmcompute.HcsRegisterComputeSystemCallback(ctx, computeSystem.handle, notificationWatcherCallback, callbackNumber)
+	callbackHandle, err := vmcompute.HcsRegisterComputeSystemCallback(ctx, computeSystem.handle,
+		notificationWatcherCallback, callbackNumber)
 	if err != nil {
 		return err
 	}
@@ -762,7 +808,7 @@ func (computeSystem *System) unregisterCallback(ctx context.Context) error {
 		return nil
 	}
 
-	// hcsUnregisterComputeSystemCallback has its own syncronization
+	// hcsUnregisterComputeSystemCallback has its own synchronization
 	// to wait for all callbacks to complete. We must NOT hold the callbackMapLock.
 	err := vmcompute.HcsUnregisterComputeSystemCallback(ctx, handle)
 	if err != nil {

vendor/github.com/Microsoft/hcsshim/internal/hcs/utils.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (

vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go

@@ -1,3 +1,5 @@
+//go:build windows
+
 package hcs
 
 import (
@@ -7,7 +9,14 @@ import (
 	"github.com/Microsoft/hcsshim/internal/log"
 )
 
-func processAsyncHcsResult(ctx context.Context, err error, resultJSON string, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) ([]ErrorEvent, error) {
+func processAsyncHcsResult(
+	ctx context.Context,
+	err error,
+	resultJSON string,
+	callbackNumber uintptr,
+	expectedNotification hcsNotification,
+	timeout *time.Duration,
+) ([]ErrorEvent, error) {
 	events := processHcsResult(ctx, resultJSON)
 	if IsPending(err) {
 		return nil, waitForNotification(ctx, callbackNumber, expectedNotification, timeout)
@@ -16,7 +25,12 @@ func processAsyncHcsResult(ctx context.Context, err error, resultJSON string, ca
 	return events, err
 }
 
-func waitForNotification(ctx context.Context, callbackNumber uintptr, expectedNotification hcsNotification, timeout *time.Duration) error {
+func waitForNotification(
+	ctx context.Context,
+	callbackNumber uintptr,
+	expectedNotification hcsNotification,
+	timeout *time.Duration,
+) error {
 	callbackMapLock.RLock()
 	if _, ok := callbackMap[callbackNumber]; !ok {
 		callbackMapLock.RUnlock()