Fix unhandled range error and extend deletion timeout

Fixes a bug when attempting to access a non-existant IP from
GetIPfromPID called via the list API.

Renames the provider from faas-containerd

Updates function deletion grace period to 30s to prevent any
errors in the REST API during a long-running deletion.

Tested on Linux with the figlet function which by default takes
around 5s to delete due to its write_timeout value, the deletion
now blocks rather than throwing an error.

Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

Makefile

@@ -2,7 +2,7 @@ Version := $(shell git describe --tags --dirty)
 GitCommit := $(shell git rev-parse HEAD)
 LDFLAGS := "-s -w -X main.Version=$(Version) -X main.GitCommit=$(GitCommit)"
 CONTAINERD_VER := 1.3.2
-CNI_VERSION := v0.8.4
+CNI_VERSION := v0.8.5
 ARCH := amd64
 
 .PHONY: all
@@ -22,7 +22,6 @@ prepare-test:
 	curl -sLSf https://github.com/containerd/containerd/releases/download/v$(CONTAINERD_VER)/containerd-$(CONTAINERD_VER).linux-amd64.tar.gz > /tmp/containerd.tar.gz && sudo tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
 	curl -SLfs https://raw.githubusercontent.com/containerd/containerd/v1.3.2/containerd.service | sudo tee /etc/systemd/system/containerd.service
 	sudo systemctl daemon-reload && sudo systemctl start containerd
-	sudo curl -fSLs "https://github.com/genuinetools/netns/releases/download/v0.5.3/netns-linux-amd64" --output "/usr/local/bin/netns" && sudo chmod a+x "/usr/local/bin/netns"
 	sudo /sbin/sysctl -w net.ipv4.conf.all.forwarding=1
 	sudo mkdir -p /opt/cni/bin
 	curl -sSL https://github.com/containernetworking/plugins/releases/download/$(CNI_VERSION)/cni-plugins-linux-$(ARCH)-$(CNI_VERSION).tgz | sudo tar -xz -C /opt/cni/bin
@@ -37,7 +36,7 @@ prepare-test:
 
 .PHONY: test-e2e
 test-e2e:
-	sudo cat /run/faasd/secrets/basic-auth-password | /usr/local/bin/faas-cli login --password-stdin
+	sudo cat /var/lib/faasd/secrets/basic-auth-password | /usr/local/bin/faas-cli login --password-stdin
 	/usr/local/bin/faas-cli store deploy figlet --env write_timeout=1s --env read_timeout=1s
 	sleep 2
 	/usr/local/bin/faas-cli list -v

README.md

@@ -52,7 +52,7 @@ Other operations are pending development in the provider such as:
 
     For Windows users, install [Git Bash](https://git-scm.com/downloads) along with multipass or vagrant. You can also use WSL1 or WSL2 which provides a Linux environment.
 
-    You will also need [containerd v1.3.2](https://github.com/containerd/containerd) and the [CNI plugins v0.8.4](https://github.com/containernetworking/plugins)
+    You will also need [containerd v1.3.2](https://github.com/containerd/containerd) and the [CNI plugins v0.8.5](https://github.com/containernetworking/plugins)
 
     [faas-cli](https://github.com/openfaas/faas-cli) is optional, but recommended.
 
@@ -75,7 +75,7 @@ Done:
 * [x] Clear / remove containers and tasks with SIGTERM / SIGINT
 * [x] Determine armhf/arm64 containers to run for gateway
 * [x] Configure `basic_auth` to protect the OpenFaaS gateway and faasd-provider HTTP API
-* [x] Setup custom working directory for faasd `/run/faasd/`
+* [x] Setup custom working directory for faasd `/var/lib/faasd/`
 * [x] Use CNI to create network namespaces and adapters
 
 ## Tutorial: Get started on armhf / Raspberry Pi
@@ -84,9 +84,114 @@ You can run this tutorial on your Raspberry Pi, or adapt the steps for a regular
 
 * [faasd - lightweight Serverless for your Raspberry Pi](https://blog.alexellis.io/faasd-for-lightweight-serverless/)
 
+## Get containerd
+
+You have three options - binaries for PC, binaries for armhf, or build from source.
+
+* Install containerd `x86_64` only
+
+```sh
+export VER=1.3.2
+curl -sLSf https://github.com/containerd/containerd/releases/download/v$VER/containerd-$VER.linux-amd64.tar.gz > /tmp/containerd.tar.gz \
+  && sudo tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
+
+containerd -version
+```
+
+* Or get my containerd binaries for armhf
+
+Building containerd on armhf is extremely slow.
+
+```sh
+curl -sSL https://github.com/alexellis/containerd-armhf/releases/download/v1.3.2/containerd.tgz | sudo tar -xvz --strip-components=2 -C /usr/local/bin/
+```
+
+* Or clone / build / install [containerd](https://github.com/containerd/containerd) from source:
+
+```sh
+export GOPATH=$HOME/go/
+mkdir -p $GOPATH/src/github.com/containerd
+cd $GOPATH/src/github.com/containerd
+git clone https://github.com/containerd/containerd
+cd containerd
+git fetch origin --tags
+git checkout v1.3.2
+
+make
+sudo make install
+
+containerd --version
+```
+
+Kill any old containerd version:
+
+```sh
+# Kill any old version
+sudo killall containerd
+sudo systemctl disable containerd
+```
+
+Start containerd in a new terminal:
+
+```sh
+sudo containerd &
+```
+### Enable forwarding
+
+> This is required to allow containers in containerd to access the Internet via your computer's primary network interface.
+
+```sh
+sudo /sbin/sysctl -w net.ipv4.conf.all.forwarding=1
+```
+
+Make the setting permanent:
+
+```sh
+echo "net.ipv4.conf.all.forwarding=1" | sudo tee -a /etc/sysctl.conf
+```
+
 ## Hacking (build from source)
 
-Install the CNI plugins:
+### Get build packages
+
+```sh
+sudo apt update \
+  && sudo apt install -qy \
+    runc \
+    bridge-utils
+```
+
+You may find alternatives for CentOS and other distributions.
+
+### Install Go 1.13 (x86_64)
+
+```sh
+curl -sSLf https://dl.google.com/go/go1.13.6.linux-amd64.tar.gz > go.tgz
+sudo rm -rf /usr/local/go/
+sudo mkdir -p /usr/local/go/
+sudo tar -xvf go.tgz -C /usr/local/go/ --strip-components=1
+
+export GOPATH=$HOME/go/
+export PATH=$PATH:/usr/local/go/bin/
+
+go version
+```
+
+### Or on Raspberry Pi (armhf)
+
+```sh
+curl -SLsf https://dl.google.com/go/go1.13.6.linux-armv6l.tar.gz > go.tgz
+sudo rm -rf /usr/local/go/
+sudo mkdir -p /usr/local/go/
+sudo tar -xvf go.tgz -C /usr/local/go/ --strip-components=1
+
+export GOPATH=$HOME/go/
+export PATH=$PATH:/usr/local/go/bin/
+
+go version
+```
+
+### Install the CNI plugins:
 
 * For PC run `export ARCH=amd64`
 * For RPi/armhf run `export ARCH=arm`
@@ -95,7 +200,9 @@ Install the CNI plugins:
 Then run:
 
 ```sh
-export CNI_VERSION=v0.8.4
+export ARCH=amd64
+export CNI_VERSION=v0.8.5
+
 sudo mkdir -p /opt/cni/bin
 curl -sSL https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz | sudo tar -xz -C /opt/cni/bin
 ```
@@ -113,7 +220,7 @@ go build
 # sudo ./faasd up
 ```
 
-### Build and run (binaries)
+### Build and run `faasd` (binaries)
 
 ```sh
 # For x86_64
@@ -134,7 +241,7 @@ sudo curl -fSLs "https://github.com/alexellis/faasd/releases/download/0.4.4/faas
 
 ### At run-time
 
-Look in `hosts` in the current working folder or in `/run/faasd/` to get the IP for the gateway or Prometheus
+Look in `hosts` in the current working folder or in `/var/lib/faasd/` to get the IP for the gateway or Prometheus
 
 ```sh
 127.0.0.1      localhost
@@ -148,7 +255,7 @@ Look in `hosts` in the current working folder or in `/run/faasd/` to get the IP
 
 The IP addresses are dynamic and may change on every launch.
 
-Since faasd-provider uses containerd heavily it is not running as a container, but as a stand-alone process. Its port is available via the bridge interface, i.e. openfaas0.
+Since faasd-provider uses containerd heavily it is not running as a container, but as a stand-alone process. Its port is available via the bridge interface, i.e. `openfaas0`
 
 * Prometheus will run on the Prometheus IP plus port 8080 i.e. http://[prometheus_ip]:9090/targets
 
@@ -158,8 +265,8 @@ Since faasd-provider uses containerd heavily it is not running as a container, b
 
 * basic-auth
 
-    You will then need to get the basic-auth password, it is written to `/run/faasd/secrets/basic-auth-password` if you followed the above instructions.
-The default Basic Auth username is `admin`, which is written to `/run/faasd/secrets/basic-auth-user`, if you wish to use a non-standard user then create this file and add your username (no newlines or other characters) 
+    You will then need to get the basic-auth password, it is written to `/var/lib/faasd/secrets/basic-auth-password` if you followed the above instructions.
+The default Basic Auth username is `admin`, which is written to `/var/lib/faasd/secrets/basic-auth-user`, if you wish to use a non-standard user then create this file and add your username (no newlines or other characters) 
 
 #### Installation with systemd
 

cloud-config.txt

@@ -0,0 +1,27 @@
+#cloud-config
+ssh_authorized_keys:
+  - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8Q/aUYUr3P1XKVucnO9mlWxOjJm+K01lHJR90MkHC9zbfTqlp8P7C3J26zKAuzHXOeF+VFxETRr6YedQKW9zp5oP7sN+F2gr/pO7GV3VmOqHMV7uKfyUQfq7H1aVzLfCcI7FwN2Zekv3yB7kj35pbsMa1Za58aF6oHRctZU6UWgXXbRxP+B04DoVU7jTstQ4GMoOCaqYhgPHyjEAS3DW0kkPW6HzsvJHkxvVcVlZ/wNJa1Ie/yGpzOzWIN0Ol0t2QT/RSWOhfzO1A2P0XbPuZ04NmriBonO9zR7T1fMNmmtTuK7WazKjQT3inmYRAqU6pe8wfX8WIWNV7OowUjUsv alex@alexr.local
+
+package_update: true
+
+packages:
+ - runc
+
+runcmd:
+- curl -sLSf https://github.com/containerd/containerd/releases/download/v1.3.2/containerd-1.3.2.linux-amd64.tar.gz > /tmp/containerd.tar.gz && tar -xvf /tmp/containerd.tar.gz -C /usr/local/bin/ --strip-components=1
+- curl -SLfs https://raw.githubusercontent.com/containerd/containerd/v1.3.2/containerd.service | tee /etc/systemd/system/containerd.service
+- systemctl daemon-reload && systemctl start containerd
+- /sbin/sysctl -w net.ipv4.conf.all.forwarding=1
+- mkdir -p /opt/cni/bin
+- curl -sSL https://github.com/containernetworking/plugins/releases/download/v0.8.5/cni-plugins-linux-amd64-v0.8.5.tgz | tar -xz -C /opt/cni/bin
+- mkdir -p /go/src/github.com/alexellis/
+- cd /go/src/github.com/alexellis/ && git clone https://github.com/alexellis/faasd
+- curl -fSLs "https://github.com/alexellis/faasd/releases/download/0.6.1/faasd" --output "/usr/local/bin/faasd" && chmod a+x "/usr/local/bin/faasd"
+- cd /go/src/github.com/alexellis/faasd/ && /usr/local/bin/faasd install
+- systemctl status -l containerd --no-pager
+- journalctl -u faasd-provider --no-pager
+- systemctl status -l faasd-provider --no-pager
+- systemctl status -l faasd --no-pager
+- curl -sSLf https://cli.openfaas.com | sh
+- sleep 5 && journalctl -u faasd --no-pager
+- cat /var/lib/faasd/secrets/basic-auth-password | /usr/local/bin/faas-cli login --password-stdin

cmd/install.go

@@ -18,8 +18,8 @@ var installCmd = &cobra.Command{
 	RunE:  runInstall,
 }
 
-const faasdwd = "/run/faasd"
-const faasdProviderWd = "/run/faasd-provider"
+const faasdwd = "/var/lib/faasd"
+const faasdProviderWd = "/var/lib/faasd-provider"
 
 func runInstall(_ *cobra.Command, _ []string) error {
 
@@ -87,7 +87,7 @@ func runInstall(_ *cobra.Command, _ []string) error {
 	}
 
 	fmt.Println(`Login with:
-  sudo cat /run/faasd/secrets/basic-auth-password | faas-cli login -s`)
+  sudo cat /var/lib/faasd/secrets/basic-auth-password | faas-cli login -s`)
 
 	return nil
 }

cmd/up.go

@@ -53,7 +53,7 @@ var defaultCNIConf = fmt.Sprintf(`
 }
 `, pkg.DefaultNetworkName, pkg.DefaultBridgeName, pkg.DefaultSubnet)
 
-const secretMountDir = "/run/secrets"
+const containerSecretMountDir = "/run/secrets"
 
 func runUp(_ *cobra.Command, _ []string) error {
 
@@ -208,18 +208,18 @@ func makeServiceDefinitions(archSuffix string) []pkg.Service {
 			Image: "docker.io/openfaas/basic-auth-plugin:0.18.10" + archSuffix,
 			Env: []string{
 				"port=8080",
-				"secret_mount_path=" + secretMountDir,
+				"secret_mount_path=" + containerSecretMountDir,
 				"user_filename=basic-auth-user",
 				"pass_filename=basic-auth-password",
 			},
 			Mounts: []pkg.Mount{
 				pkg.Mount{
 					Src:  path.Join(path.Join(wd, "secrets"), "basic-auth-password"),
-					Dest: path.Join(secretMountDir, "basic-auth-password"),
+					Dest: path.Join(containerSecretMountDir, "basic-auth-password"),
 				},
 				pkg.Mount{
 					Src:  path.Join(path.Join(wd, "secrets"), "basic-auth-user"),
-					Dest: path.Join(secretMountDir, "basic-auth-user"),
+					Dest: path.Join(containerSecretMountDir, "basic-auth-user"),
 				},
 			},
 			Caps: []string{"CAP_NET_RAW"},
@@ -257,18 +257,18 @@ func makeServiceDefinitions(archSuffix string) []pkg.Service {
 				"faas_nats_port=4222",
 				"auth_proxy_url=http://basic-auth-plugin:8080/validate",
 				"auth_proxy_pass_body=false",
-				"secret_mount_path=" + secretMountDir,
+				"secret_mount_path=" + containerSecretMountDir,
 				"scale_from_zero=true",
 			},
 			Image: "docker.io/openfaas/gateway:0.18.8" + archSuffix,
 			Mounts: []pkg.Mount{
 				pkg.Mount{
 					Src:  path.Join(path.Join(wd, "secrets"), "basic-auth-password"),
-					Dest: path.Join(secretMountDir, "basic-auth-password"),
+					Dest: path.Join(containerSecretMountDir, "basic-auth-password"),
 				},
 				pkg.Mount{
 					Src:  path.Join(path.Join(wd, "secrets"), "basic-auth-user"),
-					Dest: path.Join(secretMountDir, "basic-auth-user"),
+					Dest: path.Join(containerSecretMountDir, "basic-auth-user"),
 				},
 			},
 			Caps: []string{"CAP_NET_RAW"},
@@ -284,17 +284,17 @@ func makeServiceDefinitions(archSuffix string) []pkg.Service {
 				"max_inflight=1",
 				"write_debug=false",
 				"basic_auth=true",
-				"secret_mount_path=" + secretMountDir,
+				"secret_mount_path=" + containerSecretMountDir,
 			},
 			Image: "docker.io/openfaas/queue-worker:0.9.0",
 			Mounts: []pkg.Mount{
 				pkg.Mount{
 					Src:  path.Join(path.Join(wd, "secrets"), "basic-auth-password"),
-					Dest: path.Join(secretMountDir, "basic-auth-password"),
+					Dest: path.Join(containerSecretMountDir, "basic-auth-password"),
 				},
 				pkg.Mount{
 					Src:  path.Join(path.Join(wd, "secrets"), "basic-auth-user"),
-					Dest: path.Join(secretMountDir, "basic-auth-user"),
+					Dest: path.Join(containerSecretMountDir, "basic-auth-user"),
 				},
 			},
 			Caps: []string{"CAP_NET_RAW"},

pkg/provider/handlers/cni_network.go

@@ -161,8 +161,12 @@ func GetIPfromPID(pid int) (*net.IP, error) {
 	if addrsErr != nil {
 		return nil, fmt.Errorf("unable to find address for veth pair using: %v %s", peerIDs, addrsErr)
 	}
-	return &addrs[0].CIDRs[0].IP, nil
 
+	if len(addrs) > 0 && len(addrs[0].CIDRs) > 0 {
+		return &addrs[0].CIDRs[0].IP, nil
+	}
+
+	return nil, fmt.Errorf("no IP found for function")
 }
 
 // NetID generates the network IF based on task name and task PID

pkg/provider/handlers/containers.go

@@ -60,8 +60,13 @@ func GetFunction(client *containerd.Client, name string) (Function, error) {
 			if svc.Status == "running" {
 				replicas = 1
 				f.pid = task.Pid()
+
 				// Get container IP address
-				ip, _ := GetIPfromPID(int(task.Pid()))
+				ip, getIPErr := GetIPfromPID(int(task.Pid()))
+				if getIPErr != nil {
+					return Function{}, getIPErr
+				}
+
 				f.IP = ip.String()
 			}
 		} else {

pkg/provider/handlers/delete.go

@@ -49,6 +49,8 @@ func MakeDeleteHandler(client *containerd.Client, cni gocni.CNI) func(w http.Res
 		}
 
 		ctx := namespaces.WithNamespace(context.Background(), FunctionNamespace)
+
+		// TODO: this needs to still happen if the task is paused
 		if function.replicas != 0 {
 			err = DeleteCNINetwork(ctx, cni, client, name)
 			if err != nil {

pkg/provider/handlers/info.go

@@ -8,10 +8,11 @@ import (
 )
 
 const (
-	//OrchestrationIdentifier identifier string for provider orchestration
+	// OrchestrationIdentifier identifier string for provider orchestration
 	OrchestrationIdentifier = "containerd"
-	//ProviderName name of the provider
-	ProviderName = "faas-containerd"
+
+	// ProviderName name of the provider
+	ProviderName = "faasd"
 )
 
 //MakeInfoHandler creates handler for /system/info endpoint

pkg/service/service.go

@@ -53,8 +53,11 @@ func Remove(ctx context.Context, client *containerd.Client, name string) error {
 	return nil
 }
 
-// From Stellar
+// Adapted from Stellar - https://github.com/stellar
 func killTask(ctx context.Context, task containerd.Task) error {
+
+	killTimeout := 30 * time.Second
+
 	wg := &sync.WaitGroup{}
 	wg.Add(1)
 	var err error
@@ -69,11 +72,12 @@ func killTask(ctx context.Context, task containerd.Task) error {
 			if err := task.Kill(ctx, unix.SIGTERM, containerd.WithKillAll); err != nil {
 				log.Printf("error killing container task: %s", err)
 			}
+
 			select {
 			case <-wait:
 				task.Delete(ctx)
 				return
-			case <-time.After(5 * time.Second):
+			case <-time.After(killTimeout):
 				if err := task.Kill(ctx, unix.SIGKILL, containerd.WithKillAll); err != nil {
 					log.Printf("error force killing container task: %s", err)
 				}