diff --git a/pkg/cninetwork/cni_network.go b/pkg/cninetwork/cni_network.go index bca6914..3971b9f 100644 --- a/pkg/cninetwork/cni_network.go +++ b/pkg/cninetwork/cni_network.go @@ -1,6 +1,7 @@ package cninetwork import ( + "bufio" "context" "fmt" "io" @@ -10,6 +11,7 @@ import ( "os" "path" "path/filepath" + "strings" "github.com/containerd/containerd" gocni "github.com/containerd/go-cni" @@ -19,21 +21,31 @@ import ( const ( // CNIBinDir describes the directory where the CNI binaries are stored CNIBinDir = "/opt/cni/bin" + // CNIConfDir describes the directory where the CNI plugin's configuration is stored CNIConfDir = "/etc/cni/net.d" + // NetNSPathFmt gives the path to the a process network namespace, given the pid NetNSPathFmt = "/proc/%d/ns/net" - // CNIResultsDir is the directory CNI stores allocated IP for containers - CNIResultsDir = "/var/lib/cni/results" + + // CNIDataDir is the directory CNI stores allocated IP for containers + CNIDataDir = "/var/run/cni" + // defaultCNIConfFilename is the vanity filename of default CNI configuration file defaultCNIConfFilename = "10-openfaas.conflist" + // defaultNetworkName names the "docker-bridge"-like CNI plugin-chain installed when no other CNI configuration is present. // This value appears in iptables comments created by CNI. defaultNetworkName = "openfaas-cni-bridge" + // defaultBridgeName is the default bridge device name used in the defaultCNIConf defaultBridgeName = "openfaas0" + // defaultSubnet is the default subnet used in the defaultCNIConf -- this value is set to not collide with common container networking subnets: defaultSubnet = "10.62.0.0/16" + + // defaultIfPrefix is the interface name to be created in the container + defaultIfPrefix = "eth" ) // defaultCNIConf is a CNI configuration that enables network access to containers (docker-bridge style) @@ -50,6 +62,7 @@ var defaultCNIConf = fmt.Sprintf(` "ipam": { "type": "host-local", "subnet": "%s", + "dataDir": "%s", "routes": [ { "dst": "0.0.0.0/0" } ] @@ -60,7 +73,7 @@ var defaultCNIConf = fmt.Sprintf(` } ] } -`, defaultNetworkName, defaultBridgeName, defaultSubnet) +`, defaultNetworkName, defaultBridgeName, defaultSubnet, CNIDataDir) // InitNetwork writes configlist file and initializes CNI network func InitNetwork() (gocni.CNI, error) { @@ -75,11 +88,14 @@ func InitNetwork() (gocni.CNI, error) { netConfig := path.Join(CNIConfDir, defaultCNIConfFilename) if err := ioutil.WriteFile(netConfig, []byte(defaultCNIConf), 644); err != nil { return nil, fmt.Errorf("cannot write network config: %s", defaultCNIConfFilename) - } + // Initialize CNI library - cni, err := gocni.New(gocni.WithPluginConfDir(CNIConfDir), - gocni.WithPluginDir([]string{CNIBinDir})) + cni, err := gocni.New( + gocni.WithPluginConfDir(CNIConfDir), + gocni.WithPluginDir([]string{CNIBinDir}), + gocni.WithInterfacePrefix(defaultIfPrefix), + ) if err != nil { return nil, fmt.Errorf("error initializing cni: %s", err) @@ -131,43 +147,33 @@ func DeleteCNINetwork(ctx context.Context, cni gocni.CNI, client *containerd.Cli return errors.Wrapf(containerErr, "Unable to find container: %s, error: %s", name, containerErr) } -// GetIPAddress returns the IP address of the created container -func GetIPAddress(result *gocni.CNIResult, task containerd.Task) (net.IP, error) { - // Get the IP of the created interface - var ip net.IP - for ifName, config := range result.Interfaces { - if config.Sandbox == netNamespace(task) { - for _, ipConfig := range config.IPConfigs { - if ifName != "lo" && ipConfig.IP.To4() != nil { - ip = ipConfig.IP - } +// GetIPAddress returns the IP address from container based on name and PID +func GetIPAddress(name string, PID uint32) (string, error) { + processName := fmt.Sprintf("%s-%d", name, PID) + CNIDir := path.Join(CNIDataDir, defaultNetworkName) + + files, err := ioutil.ReadDir(CNIDir) + if err != nil { + return "", fmt.Errorf("failed to read CNI dir for container %s: %v", name, err) + } + + for _, file := range files { + f, err := os.Open(filepath.Join(CNIDir, file.Name())) + if err != nil { + return "", fmt.Errorf("failed to open CNI IP file for %s/%s: %v", CNIDir, file.Name(), err) + } + reader := bufio.NewReader(f) + text, _ := reader.ReadString('\n') + if strings.Contains(text, processName) { + i, _ := reader.ReadString('\n') + if strings.Contains(i, defaultIfPrefix) { + f.Close() + return file.Name(), nil } } + f.Close() } - if ip == nil { - return nil, fmt.Errorf("unable to get IP address for: %s", task.ID()) - } - return ip, nil -} - -func GetIPfromPID(pid int) (*net.IP, error) { - // https://github.com/weaveworks/weave/blob/master/net/netdev.go - - peerIDs, err := ConnectedToBridgeVethPeerIds(defaultBridgeName) - if err != nil { - return nil, fmt.Errorf("unable to find peers on: %s %s", defaultBridgeName, err) - } - - addrs, addrsErr := GetNetDevsByVethPeerIds(pid, peerIDs) - if addrsErr != nil { - return nil, fmt.Errorf("unable to find address for veth pair using: %v %s", peerIDs, addrsErr) - } - - if len(addrs) > 0 && len(addrs[0].CIDRs) > 0 { - return &addrs[0].CIDRs[0].IP, nil - } - - return nil, fmt.Errorf("no IP found for function") + return "", fmt.Errorf("unable to get IP address for container %s", name) } // CNIGateway returns the gateway for default subnet diff --git a/pkg/cninetwork/weave.go b/pkg/cninetwork/weave.go deleted file mode 100644 index 71b4fca..0000000 --- a/pkg/cninetwork/weave.go +++ /dev/null @@ -1,118 +0,0 @@ -// Copyright Weaveworks -// github.com/weaveworks/weave/net - -package cninetwork - -import ( - "fmt" - "net" - "os" - - "github.com/vishvananda/netlink" - "github.com/vishvananda/netns" -) - -type Dev struct { - Name string `json:"Name,omitempty"` - MAC net.HardwareAddr `json:"MAC,omitempty"` - CIDRs []*net.IPNet `json:"CIDRs,omitempty"` -} - -// ConnectedToBridgeVethPeerIds returns peer indexes of veth links connected to -// the given bridge. The peer index is used to query from a container netns -// whether the container is connected to the bridge. -func ConnectedToBridgeVethPeerIds(bridgeName string) ([]int, error) { - var ids []int - - br, err := netlink.LinkByName(bridgeName) - if err != nil { - return nil, err - } - links, err := netlink.LinkList() - if err != nil { - return nil, err - } - - for _, link := range links { - if _, isveth := link.(*netlink.Veth); isveth && link.Attrs().MasterIndex == br.Attrs().Index { - peerID := link.Attrs().ParentIndex - if peerID == 0 { - // perhaps running on an older kernel where ParentIndex doesn't work. - // as fall-back, assume the peers are consecutive - peerID = link.Attrs().Index - 1 - } - ids = append(ids, peerID) - } - } - - return ids, nil -} - -// Lookup the weave interface of a container -func GetWeaveNetDevs(processID int) ([]Dev, error) { - peerIDs, err := ConnectedToBridgeVethPeerIds("weave") - if err != nil { - return nil, err - } - - return GetNetDevsByVethPeerIds(processID, peerIDs) -} - -func GetNetDevsByVethPeerIds(processID int, peerIDs []int) ([]Dev, error) { - // Bail out if this container is running in the root namespace - netnsRoot, err := netns.GetFromPid(1) - if err != nil { - return nil, fmt.Errorf("unable to open root namespace: %s", err) - } - defer netnsRoot.Close() - netnsContainer, err := netns.GetFromPid(processID) - if err != nil { - // Unable to find a namespace for this process - just return nothing - if os.IsNotExist(err) { - return nil, nil - } - return nil, fmt.Errorf("unable to open process %d namespace: %s", processID, err) - } - defer netnsContainer.Close() - if netnsRoot.Equal(netnsContainer) { - return nil, nil - } - - // convert list of peerIDs into a map for faster lookup - indexes := make(map[int]struct{}) - for _, id := range peerIDs { - indexes[id] = struct{}{} - } - - var netdevs []Dev - err = WithNetNS(netnsContainer, func() error { - links, err := netlink.LinkList() - if err != nil { - return err - } - - for _, link := range links { - if _, found := indexes[link.Attrs().Index]; found { - netdev, err := linkToNetDev(link) - if err != nil { - return err - } - netdevs = append(netdevs, netdev) - } - } - return nil - }) - - return netdevs, err -} - -// Get the weave bridge interface. -// NB: Should be called from the root network namespace. -func GetBridgeNetDev(bridgeName string) (Dev, error) { - link, err := netlink.LinkByName(bridgeName) - if err != nil { - return Dev{}, err - } - - return linkToNetDev(link) -} diff --git a/pkg/cninetwork/weave_darwin.go b/pkg/cninetwork/weave_darwin.go deleted file mode 100644 index 983e705..0000000 --- a/pkg/cninetwork/weave_darwin.go +++ /dev/null @@ -1,10 +0,0 @@ -// +build darwin - -package cninetwork - -import "github.com/vishvananda/netlink" - -func linkToNetDev(link netlink.Link) (Dev, error) { - - return Dev{}, nil -} diff --git a/pkg/cninetwork/weave_linux.go b/pkg/cninetwork/weave_linux.go deleted file mode 100644 index 3950ab2..0000000 --- a/pkg/cninetwork/weave_linux.go +++ /dev/null @@ -1,19 +0,0 @@ -// +build linux - -package cninetwork - -import "github.com/vishvananda/netlink" - -func linkToNetDev(link netlink.Link) (Dev, error) { - - addrs, err := netlink.AddrList(link, netlink.FAMILY_V4) - if err != nil { - return Dev{}, err - } - - netDev := Dev{Name: link.Attrs().Name, MAC: link.Attrs().HardwareAddr} - for _, addr := range addrs { - netDev.CIDRs = append(netDev.CIDRs, addr.IPNet) - } - return netDev, nil -} diff --git a/pkg/provider/handlers/deploy.go b/pkg/provider/handlers/deploy.go index 4fa9358..6d5452c 100644 --- a/pkg/provider/handlers/deploy.go +++ b/pkg/provider/handlers/deploy.go @@ -200,17 +200,18 @@ func createTask(ctx context.Context, client *containerd.Client, container contai log.Printf("Container ID: %s\tTask ID %s:\tTask PID: %d\t\n", name, task.ID(), task.Pid()) labels := map[string]string{} - network, err := cninetwork.CreateCNINetwork(ctx, cni, task, labels) + _, err := cninetwork.CreateCNINetwork(ctx, cni, task, labels) if err != nil { return err } - ip, err := cninetwork.GetIPAddress(network, task) + ip, err := cninetwork.GetIPAddress(name, task.Pid()) if err != nil { return err } - log.Printf("%s has IP: %s.\n", name, ip.String()) + + log.Printf("%s has IP: %s.\n", name, ip) _, waitErr := task.Wait(ctx) if waitErr != nil { diff --git a/pkg/provider/handlers/functions.go b/pkg/provider/handlers/functions.go index 457cba8..064e619 100644 --- a/pkg/provider/handlers/functions.go +++ b/pkg/provider/handlers/functions.go @@ -3,10 +3,11 @@ package handlers import ( "context" "fmt" - "github.com/opencontainers/runtime-spec/specs-go" "log" "strings" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/containerd/containerd" "github.com/containerd/containerd/namespaces" "github.com/openfaas/faasd/pkg/cninetwork" @@ -106,11 +107,11 @@ func GetFunction(client *containerd.Client, name string) (Function, error) { fn.pid = task.Pid() // Get container IP address - ip, err := cninetwork.GetIPfromPID(int(task.Pid())) + ip, err := cninetwork.GetIPAddress(name, task.Pid()) if err != nil { return Function{}, err } - fn.IP = ip.String() + fn.IP = ip } } else { replicas = 0 diff --git a/pkg/supervisor.go b/pkg/supervisor.go index 3da2272..3d34ef9 100644 --- a/pkg/supervisor.go +++ b/pkg/supervisor.go @@ -194,19 +194,19 @@ func (s *Supervisor) Start(svcs []Service) error { } labels := map[string]string{} - network, err := cninetwork.CreateCNINetwork(ctx, s.cni, task, labels) + _, err = cninetwork.CreateCNINetwork(ctx, s.cni, task, labels) if err != nil { log.Printf("Error creating CNI for %s: %s", svc.Name, err) return err } - ip, err := cninetwork.GetIPAddress(network, task) + ip, err := cninetwork.GetIPAddress(svc.Name, task.Pid()) if err != nil { log.Printf("Error getting IP for %s: %s", svc.Name, err) return err } - log.Printf("%s has IP: %s\n", newContainer.ID(), ip.String()) + log.Printf("%s has IP: %s\n", newContainer.ID(), ip) hosts, err := ioutil.ReadFile("hosts") if err != nil {