From e8c2eeb05282fa04e3e89e3c75526fecd823fccb Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Fri, 19 Feb 2021 12:12:14 +0000 Subject: [PATCH] Use CNI cache to find container IP This is an optimization that uses the results cache created by CNI on the filesystem to store and fetch IP addresses for containers in the core services and for functions. As part of the change, the dependency on the syscall code from Weave net has been removed, and the code should compile on MacOS again. Updates and rebases the work in #38 by carlosedp Tested in the original PR, further testing in the incoming PR. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- pkg/cninetwork/cni_network.go | 86 +++++++++++---------- pkg/cninetwork/weave.go | 118 ----------------------------- pkg/cninetwork/weave_darwin.go | 10 --- pkg/cninetwork/weave_linux.go | 19 ----- pkg/provider/handlers/deploy.go | 7 +- pkg/provider/handlers/functions.go | 7 +- pkg/supervisor.go | 6 +- 7 files changed, 57 insertions(+), 196 deletions(-) delete mode 100644 pkg/cninetwork/weave.go delete mode 100644 pkg/cninetwork/weave_darwin.go delete mode 100644 pkg/cninetwork/weave_linux.go diff --git a/pkg/cninetwork/cni_network.go b/pkg/cninetwork/cni_network.go index bca6914..3971b9f 100644 --- a/pkg/cninetwork/cni_network.go +++ b/pkg/cninetwork/cni_network.go @@ -1,6 +1,7 @@ package cninetwork import ( + "bufio" "context" "fmt" "io" @@ -10,6 +11,7 @@ import ( "os" "path" "path/filepath" + "strings" "github.com/containerd/containerd" gocni "github.com/containerd/go-cni" @@ -19,21 +21,31 @@ import ( const ( // CNIBinDir describes the directory where the CNI binaries are stored CNIBinDir = "/opt/cni/bin" + // CNIConfDir describes the directory where the CNI plugin's configuration is stored CNIConfDir = "/etc/cni/net.d" + // NetNSPathFmt gives the path to the a process network namespace, given the pid NetNSPathFmt = "/proc/%d/ns/net" - // CNIResultsDir is the directory CNI stores allocated IP for containers - CNIResultsDir = "/var/lib/cni/results" + + // CNIDataDir is the directory CNI stores allocated IP for containers + CNIDataDir = "/var/run/cni" + // defaultCNIConfFilename is the vanity filename of default CNI configuration file defaultCNIConfFilename = "10-openfaas.conflist" + // defaultNetworkName names the "docker-bridge"-like CNI plugin-chain installed when no other CNI configuration is present. // This value appears in iptables comments created by CNI. defaultNetworkName = "openfaas-cni-bridge" + // defaultBridgeName is the default bridge device name used in the defaultCNIConf defaultBridgeName = "openfaas0" + // defaultSubnet is the default subnet used in the defaultCNIConf -- this value is set to not collide with common container networking subnets: defaultSubnet = "10.62.0.0/16" + + // defaultIfPrefix is the interface name to be created in the container + defaultIfPrefix = "eth" ) // defaultCNIConf is a CNI configuration that enables network access to containers (docker-bridge style) @@ -50,6 +62,7 @@ var defaultCNIConf = fmt.Sprintf(` "ipam": { "type": "host-local", "subnet": "%s", + "dataDir": "%s", "routes": [ { "dst": "0.0.0.0/0" } ] @@ -60,7 +73,7 @@ var defaultCNIConf = fmt.Sprintf(` } ] } -`, defaultNetworkName, defaultBridgeName, defaultSubnet) +`, defaultNetworkName, defaultBridgeName, defaultSubnet, CNIDataDir) // InitNetwork writes configlist file and initializes CNI network func InitNetwork() (gocni.CNI, error) { @@ -75,11 +88,14 @@ func InitNetwork() (gocni.CNI, error) { netConfig := path.Join(CNIConfDir, defaultCNIConfFilename) if err := ioutil.WriteFile(netConfig, []byte(defaultCNIConf), 644); err != nil { return nil, fmt.Errorf("cannot write network config: %s", defaultCNIConfFilename) - } + // Initialize CNI library - cni, err := gocni.New(gocni.WithPluginConfDir(CNIConfDir), - gocni.WithPluginDir([]string{CNIBinDir})) + cni, err := gocni.New( + gocni.WithPluginConfDir(CNIConfDir), + gocni.WithPluginDir([]string{CNIBinDir}), + gocni.WithInterfacePrefix(defaultIfPrefix), + ) if err != nil { return nil, fmt.Errorf("error initializing cni: %s", err) @@ -131,43 +147,33 @@ func DeleteCNINetwork(ctx context.Context, cni gocni.CNI, client *containerd.Cli return errors.Wrapf(containerErr, "Unable to find container: %s, error: %s", name, containerErr) } -// GetIPAddress returns the IP address of the created container -func GetIPAddress(result *gocni.CNIResult, task containerd.Task) (net.IP, error) { - // Get the IP of the created interface - var ip net.IP - for ifName, config := range result.Interfaces { - if config.Sandbox == netNamespace(task) { - for _, ipConfig := range config.IPConfigs { - if ifName != "lo" && ipConfig.IP.To4() != nil { - ip = ipConfig.IP - } +// GetIPAddress returns the IP address from container based on name and PID +func GetIPAddress(name string, PID uint32) (string, error) { + processName := fmt.Sprintf("%s-%d", name, PID) + CNIDir := path.Join(CNIDataDir, defaultNetworkName) + + files, err := ioutil.ReadDir(CNIDir) + if err != nil { + return "", fmt.Errorf("failed to read CNI dir for container %s: %v", name, err) + } + + for _, file := range files { + f, err := os.Open(filepath.Join(CNIDir, file.Name())) + if err != nil { + return "", fmt.Errorf("failed to open CNI IP file for %s/%s: %v", CNIDir, file.Name(), err) + } + reader := bufio.NewReader(f) + text, _ := reader.ReadString('\n') + if strings.Contains(text, processName) { + i, _ := reader.ReadString('\n') + if strings.Contains(i, defaultIfPrefix) { + f.Close() + return file.Name(), nil } } + f.Close() } - if ip == nil { - return nil, fmt.Errorf("unable to get IP address for: %s", task.ID()) - } - return ip, nil -} - -func GetIPfromPID(pid int) (*net.IP, error) { - // https://github.com/weaveworks/weave/blob/master/net/netdev.go - - peerIDs, err := ConnectedToBridgeVethPeerIds(defaultBridgeName) - if err != nil { - return nil, fmt.Errorf("unable to find peers on: %s %s", defaultBridgeName, err) - } - - addrs, addrsErr := GetNetDevsByVethPeerIds(pid, peerIDs) - if addrsErr != nil { - return nil, fmt.Errorf("unable to find address for veth pair using: %v %s", peerIDs, addrsErr) - } - - if len(addrs) > 0 && len(addrs[0].CIDRs) > 0 { - return &addrs[0].CIDRs[0].IP, nil - } - - return nil, fmt.Errorf("no IP found for function") + return "", fmt.Errorf("unable to get IP address for container %s", name) } // CNIGateway returns the gateway for default subnet diff --git a/pkg/cninetwork/weave.go b/pkg/cninetwork/weave.go deleted file mode 100644 index 71b4fca..0000000 --- a/pkg/cninetwork/weave.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Weaveworks -// github.com/weaveworks/weave/net - -package cninetwork - -import ( - "fmt" - "net" - "os" - - "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" -) - -type Dev struct { - Name string `json:"Name,omitempty"` - MAC net.HardwareAddr `json:"MAC,omitempty"` - CIDRs []*net.IPNet `json:"CIDRs,omitempty"` -} - -// ConnectedToBridgeVethPeerIds returns peer indexes of veth links connected to -// the given bridge. The peer index is used to query from a container netns -// whether the container is connected to the bridge. -func ConnectedToBridgeVethPeerIds(bridgeName string) ([]int, error) { - var ids []int - - br, err := netlink.LinkByName(bridgeName) - if err != nil { - return nil, err - } - links, err := netlink.LinkList() - if err != nil { - return nil, err - } - - for _, link := range links { - if _, isveth := link.(*netlink.Veth); isveth && link.Attrs().MasterIndex == br.Attrs().Index { - peerID := link.Attrs().ParentIndex - if peerID == 0 { - // perhaps running on an older kernel where ParentIndex doesn't work. - // as fall-back, assume the peers are consecutive - peerID = link.Attrs().Index - 1 - } - ids = append(ids, peerID) - } - } - - return ids, nil -} - -// Lookup the weave interface of a container -func GetWeaveNetDevs(processID int) ([]Dev, error) { - peerIDs, err := ConnectedToBridgeVethPeerIds("weave") - if err != nil { - return nil, err - } - - return GetNetDevsByVethPeerIds(processID, peerIDs) -} - -func GetNetDevsByVethPeerIds(processID int, peerIDs []int) ([]Dev, error) { - // Bail out if this container is running in the root namespace - netnsRoot, err := netns.GetFromPid(1) - if err != nil { - return nil, fmt.Errorf("unable to open root namespace: %s", err) - } - defer netnsRoot.Close() - netnsContainer, err := netns.GetFromPid(processID) - if err != nil { - // Unable to find a namespace for this process - just return nothing - if os.IsNotExist(err) { - return nil, nil - } - return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err) - } - defer netnsContainer.Close() - if netnsRoot.Equal(netnsContainer) { - return nil, nil - } - - // convert list of peerIDs into a map for faster lookup - indexes := make(map[int]struct{}) - for _, id := range peerIDs { - indexes[id] = struct{}{} - } - - var netdevs []Dev - err = WithNetNS(netnsContainer, func() error { - links, err := netlink.LinkList() - if err != nil { - return err - } - - for _, link := range links { - if _, found := indexes[link.Attrs().Index]; found { - netdev, err := linkToNetDev(link) - if err != nil { - return err - } - netdevs = append(netdevs, netdev) - } - } - return nil - }) - - return netdevs, err -} - -// Get the weave bridge interface. -// NB: Should be called from the root network namespace. -func GetBridgeNetDev(bridgeName string) (Dev, error) { - link, err := netlink.LinkByName(bridgeName) - if err != nil { - return Dev{}, err - } - - return linkToNetDev(link) -} diff --git a/pkg/cninetwork/weave_darwin.go b/pkg/cninetwork/weave_darwin.go deleted file mode 100644 index 983e705..0000000 --- a/pkg/cninetwork/weave_darwin.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build darwin - -package cninetwork - -import "github.com/vishvananda/netlink" - -func linkToNetDev(link netlink.Link) (Dev, error) { - - return Dev{}, nil -} diff --git a/pkg/cninetwork/weave_linux.go b/pkg/cninetwork/weave_linux.go deleted file mode 100644 index 3950ab2..0000000 --- a/pkg/cninetwork/weave_linux.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build linux - -package cninetwork - -import "github.com/vishvananda/netlink" - -func linkToNetDev(link netlink.Link) (Dev, error) { - - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return Dev{}, err - } - - netDev := Dev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr} - for _, addr := range addrs { - netDev.CIDRs = append(netDev.CIDRs, addr.IPNet) - } - return netDev, nil -} diff --git a/pkg/provider/handlers/deploy.go b/pkg/provider/handlers/deploy.go index 4fa9358..6d5452c 100644 --- a/pkg/provider/handlers/deploy.go +++ b/pkg/provider/handlers/deploy.go @@ -200,17 +200,18 @@ func createTask(ctx context.Context, client *containerd.Client, container contai log.Printf("Container ID: %s\tTask ID %s:\tTask PID: %d\t\n", name, task.ID(), task.Pid()) labels := map[string]string{} - network, err := cninetwork.CreateCNINetwork(ctx, cni, task, labels) + _, err := cninetwork.CreateCNINetwork(ctx, cni, task, labels) if err != nil { return err } - ip, err := cninetwork.GetIPAddress(network, task) + ip, err := cninetwork.GetIPAddress(name, task.Pid()) if err != nil { return err } - log.Printf("%s has IP: %s.\n", name, ip.String()) + + log.Printf("%s has IP: %s.\n", name, ip) _, waitErr := task.Wait(ctx) if waitErr != nil { diff --git a/pkg/provider/handlers/functions.go b/pkg/provider/handlers/functions.go index 457cba8..064e619 100644 --- a/pkg/provider/handlers/functions.go +++ b/pkg/provider/handlers/functions.go @@ -3,10 +3,11 @@ package handlers import ( "context" "fmt" - "github.com/opencontainers/runtime-spec/specs-go" "log" "strings" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/containerd/containerd" "github.com/containerd/containerd/namespaces" "github.com/openfaas/faasd/pkg/cninetwork" @@ -106,11 +107,11 @@ func GetFunction(client *containerd.Client, name string) (Function, error) { fn.pid = task.Pid() // Get container IP address - ip, err := cninetwork.GetIPfromPID(int(task.Pid())) + ip, err := cninetwork.GetIPAddress(name, task.Pid()) if err != nil { return Function{}, err } - fn.IP = ip.String() + fn.IP = ip } } else { replicas = 0 diff --git a/pkg/supervisor.go b/pkg/supervisor.go index 3da2272..3d34ef9 100644 --- a/pkg/supervisor.go +++ b/pkg/supervisor.go @@ -194,19 +194,19 @@ func (s *Supervisor) Start(svcs []Service) error { } labels := map[string]string{} - network, err := cninetwork.CreateCNINetwork(ctx, s.cni, task, labels) + _, err = cninetwork.CreateCNINetwork(ctx, s.cni, task, labels) if err != nil { log.Printf("Error creating CNI for %s: %s", svc.Name, err) return err } - ip, err := cninetwork.GetIPAddress(network, task) + ip, err := cninetwork.GetIPAddress(svc.Name, task.Pid()) if err != nil { log.Printf("Error getting IP for %s: %s", svc.Name, err) return err } - log.Printf("%s has IP: %s\n", newContainer.ID(), ip.String()) + log.Printf("%s has IP: %s\n", newContainer.ID(), ip) hosts, err := ioutil.ReadFile("hosts") if err != nil {