Feat cni (#7)

* fmt

* 完善cni以及添加接口

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "nixEnvSelector.nixFile": "${workspaceFolder}/default.nix"
+}

Cargo.lock

@@ -513,6 +513,13 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
+[[package]]
+name = "cni"
+version = "0.1.0"
+dependencies = [
+ "serde_json",
+]
+
 [[package]]
 name = "config"
 version = "0.11.0"
@@ -2349,10 +2356,12 @@ dependencies = [
 name = "service"
 version = "0.1.0"
 dependencies = [
+ "cni",
  "containerd-client",
  "env_logger",
  "handlebars",
  "hex",
+ "lazy_static",
  "log",
  "my-workspace-hack",
  "oci-spec",

crates/cni/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "cni"
+version = "0.1.0"
+edition = "2024"
+# version.workspace = true
+# authors.workspace = true
+
+[dependencies]
+serde_json = "1.0"

crates/cni/src/cni_network.rs

@@ -0,0 +1,175 @@
+use crate::Err;
+use serde_json::Value;
+use std::{
+    fmt::Error,
+    fs::{self, File},
+    io::Write,
+    net::IpAddr,
+    path::Path,
+};
+
+const CNI_BIN_DIR: &str = "/opt/cni/bin";
+const CNI_CONF_DIR: &str = "/etc/cni/net.d";
+// const NET_NS_PATH_FMT: &str = "/proc/{}/ns/net";
+const CNI_DATA_DIR: &str = "/var/run/cni";
+const DEFAULT_CNI_CONF_FILENAME: &str = "10-faasrs.conflist";
+const DEFAULT_NETWORK_NAME: &str = "faasrs-cni-bridge";
+const DEFAULT_BRIDGE_NAME: &str = "faasrs0";
+const DEFAULT_SUBNET: &str = "10.66.0.0/16";
+// const DEFAULT_IF_PREFIX: &str = "eth";
+
+fn default_cni_conf() -> String {
+    format!(
+        r#"
+{{
+    "cniVersion": "0.4.0",
+    "name": "{}",
+    "plugins": [
+      {{
+        "type": "bridge",
+        "bridge": "{}",
+        "isGateway": true,
+        "ipMasq": true,
+        "ipam": {{
+            "type": "host-local",
+            "subnet": "{}",
+            "dataDir": "{}",
+            "routes": [
+                {{ "dst": "0.0.0.0/0" }}
+            ]
+        }}
+      }},
+      {{
+        "type": "firewall"
+      }}
+    ]
+}}
+"#,
+        DEFAULT_NETWORK_NAME, DEFAULT_BRIDGE_NAME, DEFAULT_SUBNET, CNI_DATA_DIR
+    )
+}
+
+pub fn init_net_work() -> Result<(), Err> {
+    if !dir_exists(Path::new(CNI_CONF_DIR)) {
+        fs::create_dir_all(CNI_CONF_DIR)?;
+    }
+    let net_config = Path::new(CNI_CONF_DIR).join(DEFAULT_CNI_CONF_FILENAME);
+    let mut file = File::create(&net_config)?;
+    file.write_all(default_cni_conf().as_bytes())?;
+
+    Ok(())
+}
+
+fn get_netns(ns: &str, cid: &str) -> String {
+    format!("{}-{}", ns, cid)
+}
+
+fn get_path(netns: &str) -> String {
+    format!("/var/run/netns/{}", netns)
+}
+
+//TODO: 创建网络和删除网络的错误处理
+pub fn create_cni_network(cid: String, ns: String) -> Result<(String, String), Err> {
+    // let netid = format!("{}-{}", cid, pid);
+    let netns = get_netns(ns.as_str(), cid.as_str());
+    let path = get_path(netns.as_str());
+    let mut ip = String::new();
+
+    let output = std::process::Command::new("ip")
+        .arg("netns")
+        .arg("add")
+        .arg(&netns)
+        .output()?;
+
+    if !output.status.success() {
+        return Err(Box::new(Error));
+    }
+
+    let add_command = format!(
+        "export CNI_PATH={} && cnitool add faasrs-cni-bridge {}",
+        CNI_BIN_DIR, path
+    );
+    let output = std::process::Command::new("sh")
+        .arg("-c")
+        .arg(&add_command)
+        .output();
+    match output {
+        Ok(output) => {
+            let stdout = String::from_utf8_lossy(&output.stdout);
+            let json: Value = match serde_json::from_str(&stdout) {
+                Ok(json) => json,
+                Err(e) => {
+                    return Err(Box::new(e));
+                }
+            };
+            if let Some(ips) = json.get("ips").and_then(|ips| ips.as_array()) {
+                if let Some(first_ip) = ips
+                    .get(0)
+                    .and_then(|ip| ip.get("address"))
+                    .and_then(|addr| addr.as_str())
+                {
+                    ip = first_ip.to_string();
+                } else {
+                }
+            } else {
+            }
+        }
+        Err(e) => {
+            return Err(Box::new(e));
+        }
+    }
+
+    Ok((ip, path))
+}
+
+pub fn delete_cni_network(ns: &str, cid: &str) {
+    let netns = get_netns(ns, cid);
+    let path = get_path(&netns);
+    let del_command = format!(
+        "export CNI_PATH={} && cnitool del faasrs-cni-bridge {}",
+        CNI_BIN_DIR, path
+    );
+    let _output_del = std::process::Command::new("sh")
+        .arg("-c")
+        .arg(&del_command)
+        .output()
+        .expect("Failed to execute del command");
+    let _output = std::process::Command::new("ip")
+        .arg("netns")
+        .arg("delete")
+        .arg(&netns)
+        .output();
+}
+
+fn dir_exists(dirname: &Path) -> bool {
+    path_exists(dirname).map_or(false, |info| info.is_dir())
+}
+
+fn path_exists(path: &Path) -> Option<fs::Metadata> {
+    match fs::metadata(path) {
+        Ok(metadata) => Some(metadata),
+        Err(_) => None,
+    }
+}
+
+#[allow(unused)]
+fn cni_gateway() -> Result<String, Err> {
+    let ip: IpAddr = DEFAULT_SUBNET.parse().unwrap();
+    if let IpAddr::V4(ip) = ip {
+        let octets = &mut ip.octets();
+        octets[3] = 1;
+        return Ok(ip.to_string());
+    }
+    Err(Box::new(Error))
+}
+
+#[allow(unused)]
+fn dir_empty(dirname: &Path) -> bool {
+    if !dir_exists(dirname) {
+        return false;
+    }
+    match fs::read_dir(dirname) {
+        Ok(mut entries) => entries.next().is_none(),
+        Err(_) => false,
+    }
+}

crates/cni/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod cni_network;
+
+type Err = Box<dyn std::error::Error>;

crates/service/Cargo.toml

@@ -16,4 +16,6 @@ oci-spec =  "0.6"
 sha2 = "0.10"
 hex = "0.4"
 my-workspace-hack = { version = "0.1", path = "../my-workspace-hack" }
-handlebars= "4.1.0"     
+cni = { version = "0.1", path = "../cni" }
+handlebars= "4.1.0"
+lazy_static = "1.4"

crates/service/src/lib.rs

@@ -23,16 +23,28 @@ use oci_spec::image::{Arch, ImageConfiguration, ImageIndex, ImageManifest, Media
 use prost_types::Any;
 use sha2::{Digest, Sha256};
 use spec::{DEFAULT_NAMESPACE, generate_spec};
-use std::{fs, sync::Arc, time::Duration, vec};
+use std::{
+    collections::HashMap,
+    fs,
+    sync::{Arc, RwLock},
+    time::Duration,
+    vec,
+};
 use tokio::time::timeout;
 
 // config.json,dockerhub密钥
 // const DOCKER_CONFIG_DIR: &str = "/var/lib/faasd/.docker/";
 
+type NetnsMap = Arc<RwLock<HashMap<String, (String, String)>>>;
+lazy_static::lazy_static! {
+    static ref GLOBAL_NETNS_MAP: NetnsMap = Arc::new(RwLock::new(HashMap::new()));
+}
+
 type Err = Box<dyn std::error::Error>;
 
 pub struct Service {
     client: Arc<Client>,
+    netns_map: NetnsMap,
 }
 
 impl Service {
@@ -40,9 +52,25 @@ impl Service {
         let client = Client::from_path(endpoint).await.unwrap();
         Ok(Service {
             client: Arc::new(client),
+            netns_map: GLOBAL_NETNS_MAP.clone(),
         })
     }
 
+    pub async fn save_netns_ip(&self, cid: &str, netns: &str, ip: &str) {
+        let mut map = self.netns_map.write().unwrap();
+        map.insert(cid.to_string(), (netns.to_string(), ip.to_string()));
+    }
+
+    pub async fn get_netns_ip(&self, cid: &str) -> Option<(String, String)> {
+        let map = self.netns_map.read().unwrap();
+        map.get(cid).cloned()
+    }
+
+    pub async fn remove_netns_ip(&self, cid: &str) {
+        let mut map = self.netns_map.write().unwrap();
+        map.remove(cid);
+    }
+
     async fn prepare_snapshot(&self, cid: &str, ns: &str) -> Result<Vec<Mount>, Err> {
         let parent_snapshot = self.get_parent_snapshot(cid, ns).await?;
         let req = PrepareSnapshotRequest {
@@ -69,8 +97,8 @@ impl Service {
         };
 
         let _mount = self.prepare_snapshot(cid, ns).await?;
-
-        let spec_path = generate_spec(&cid, ns).unwrap();
+        let (env, args) = self.get_env_and_args(image_name, ns).await?;
+        let spec_path = generate_spec(&cid, ns, args, env).unwrap();
         let spec = fs::read_to_string(spec_path).unwrap();
 
         let spec = Any {
@@ -96,10 +124,10 @@ impl Service {
             container: Some(container),
         };
 
-        let req = with_namespace!(req, namespace);
+        // let req = with_namespace!(req, namespace);
 
         let _resp = containers_client
-            .create(req)
+            .create(with_namespace!(req, namespace))
             .await
             .expect("Failed to create container");
 
@@ -162,6 +190,7 @@ impl Service {
                 .delete(with_namespace!(delete_request, namespace))
                 .await
                 .expect("Failed to delete container");
+            self.remove_netns_ip(cid).await;
 
             // println!("Container: {:?} deleted", cc);
         } else {
@@ -191,7 +220,8 @@ impl Service {
         Ok(())
     }
 
-    async fn create_task(&self, cid: &str, ns: &str) -> Result<(), Err> {
+    /// 返回任务的pid
+    async fn create_task(&self, cid: &str, ns: &str) -> Result<u32, Err> {
         let mut sc = self.client.snapshots();
         let req = MountsRequest {
             snapshotter: "overlayfs".to_string(),
@@ -203,15 +233,17 @@ impl Service {
             .into_inner()
             .mounts;
         drop(sc);
+        let (ip, path) = cni::cni_network::create_cni_network(cid.to_string(), ns.to_string())?;
+        self.save_netns_ip(cid, &path, &ip).await;
         let mut tc = self.client.tasks();
         let req = CreateTaskRequest {
             container_id: cid.to_string(),
             rootfs: mounts,
             ..Default::default()
         };
-        let _resp = tc.create(with_namespace!(req, ns)).await?;
-
-        Ok(())
+        let resp = tc.create(with_namespace!(req, ns)).await?;
+        let pid = resp.into_inner().pid;
+        Ok(pid)
     }
 
     async fn start_task(&self, cid: &str, ns: &str) -> Result<(), Err> {
@@ -578,6 +610,21 @@ impl Service {
         Ok(ret)
     }
 
+    async fn get_env_and_args(
+        &self,
+        name: &str,
+        ns: &str,
+    ) -> Result<(Vec<String>, Vec<String>), Err> {
+        let img_config = self.get_img_config(name, ns).await.unwrap();
+        if let Some(config) = img_config.config() {
+            let env = config.env().as_ref().map_or_else(Vec::new, |v| v.clone());
+            let args = config.cmd().as_ref().map_or_else(Vec::new, |v| v.clone());
+            Ok((env, args))
+        } else {
+            Err("No config found".into())
+        }
+    }
+
     fn check_namespace(&self, ns: &str) -> String {
         match ns {
             "" => DEFAULT_NAMESPACE.to_string(),

crates/service/src/spec.rs

@@ -103,6 +103,7 @@ pub struct LinuxDeviceCgroup {
 pub struct LinuxNamespace {
     #[serde(rename = "type")]
     pub type_: String,
+    pub path: Option<String>,
 }
 
 pub fn default_unix_caps() -> Vec<String> {
@@ -150,22 +151,27 @@ fn default_readonly_paths() -> Vec<String> {
     ]
 }
 
-fn default_unix_namespaces() -> Vec<LinuxNamespace> {
+fn default_unix_namespaces(ns: &str, cid: &str) -> Vec<LinuxNamespace> {
     vec![
         LinuxNamespace {
             type_: String::from(PID_NAMESPACE),
+            path: None,
         },
         LinuxNamespace {
             type_: String::from(IPC_NAMESPACE),
+            path: None,
         },
         LinuxNamespace {
             type_: String::from(UTS_NAMESPACE),
+            path: None,
         },
         LinuxNamespace {
             type_: String::from(MOUNT_NAMESPACE),
+            path: None,
         },
         LinuxNamespace {
             type_: String::from(NETWORK_NAMESPACE),
+            path: Some(format!("/var/run/netns/{}", get_netns(ns, cid))),
         },
     ]
 }
@@ -294,7 +300,7 @@ pub fn populate_default_unix_spec(id: &str, ns: &str) -> Spec {
                     access: RWM.to_string(),
                 }],
             },
-            namespaces: default_unix_namespaces(),
+            namespaces: default_unix_namespaces(ns, id),
         },
         mounts: default_mounts(),
     }
@@ -306,12 +312,23 @@ pub fn save_spec_to_file(spec: &Spec, path: &str) -> Result<(), std::io::Error>
     Ok(())
 }
 
-pub fn generate_spec(id: &str, ns: &str) -> Result<String, std::io::Error> {
+fn get_netns(ns: &str, cid: &str) -> String {
+    format!("{}-{}", ns, cid)
+}
+
+pub fn generate_spec(
+    id: &str,
+    ns: &str,
+    args: Vec<String>,
+    env: Vec<String>,
+) -> Result<String, std::io::Error> {
     let namespace = match ns {
         "" => DEFAULT_NAMESPACE,
         _ => ns,
     };
-    let spec = populate_default_unix_spec(id, ns);
+    let mut spec = populate_default_unix_spec(id, ns);
+    spec.process.args = args;
+    spec.process.env = env;
     let path = format!("{}/{}/{}.json", PATH_TO_SPEC_PREFIX, namespace, id);
     save_spec_to_file(&spec, &path)?;
     Ok(path)