diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..b793c5a --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "nixEnvSelector.nixFile": "${workspaceFolder}/default.nix" +} diff --git a/Cargo.lock b/Cargo.lock index 52e60cb..9a5f720 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -513,6 +513,13 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cni" +version = "0.1.0" +dependencies = [ + "serde_json", +] + [[package]] name = "config" version = "0.11.0" @@ -2349,10 +2356,12 @@ dependencies = [ name = "service" version = "0.1.0" dependencies = [ + "cni", "containerd-client", "env_logger", "handlebars", "hex", + "lazy_static", "log", "my-workspace-hack", "oci-spec", diff --git a/crates/cni/Cargo.toml b/crates/cni/Cargo.toml new file mode 100644 index 0000000..0cbb13d --- /dev/null +++ b/crates/cni/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "cni" +version = "0.1.0" +edition = "2024" +# version.workspace = true +# authors.workspace = true + +[dependencies] +serde_json = "1.0" \ No newline at end of file diff --git a/crates/cni/src/cni_network.rs b/crates/cni/src/cni_network.rs new file mode 100644 index 0000000..6b358fe --- /dev/null +++ b/crates/cni/src/cni_network.rs @@ -0,0 +1,175 @@ +use crate::Err; +use serde_json::Value; +use std::{ + fmt::Error, + fs::{self, File}, + io::Write, + net::IpAddr, + path::Path, +}; + +const CNI_BIN_DIR: &str = "/opt/cni/bin"; +const CNI_CONF_DIR: &str = "/etc/cni/net.d"; +// const NET_NS_PATH_FMT: &str = "/proc/{}/ns/net"; +const CNI_DATA_DIR: &str = "/var/run/cni"; +const DEFAULT_CNI_CONF_FILENAME: &str = "10-faasrs.conflist"; +const DEFAULT_NETWORK_NAME: &str = "faasrs-cni-bridge"; +const DEFAULT_BRIDGE_NAME: &str = "faasrs0"; +const DEFAULT_SUBNET: &str = "10.66.0.0/16"; +// const DEFAULT_IF_PREFIX: &str = "eth"; + +fn default_cni_conf() -> String { + format!( + r#" +{{ + "cniVersion": "0.4.0", + "name": "{}", + "plugins": [ + {{ + "type": "bridge", + "bridge": "{}", + "isGateway": true, + "ipMasq": true, + "ipam": {{ + "type": "host-local", + "subnet": "{}", + "dataDir": "{}", + "routes": [ + {{ "dst": "0.0.0.0/0" }} + ] + }} + }}, + {{ + "type": "firewall" + }} + ] +}} +"#, + DEFAULT_NETWORK_NAME, DEFAULT_BRIDGE_NAME, DEFAULT_SUBNET, CNI_DATA_DIR + ) +} + +pub fn init_net_work() -> Result<(), Err> { + if !dir_exists(Path::new(CNI_CONF_DIR)) { + fs::create_dir_all(CNI_CONF_DIR)?; + } + let net_config = Path::new(CNI_CONF_DIR).join(DEFAULT_CNI_CONF_FILENAME); + let mut file = File::create(&net_config)?; + file.write_all(default_cni_conf().as_bytes())?; + + Ok(()) +} + +fn get_netns(ns: &str, cid: &str) -> String { + format!("{}-{}", ns, cid) +} + +fn get_path(netns: &str) -> String { + format!("/var/run/netns/{}", netns) +} + +//TODO: 创建网络和删除网络的错误处理 +pub fn create_cni_network(cid: String, ns: String) -> Result<(String, String), Err> { + // let netid = format!("{}-{}", cid, pid); + let netns = get_netns(ns.as_str(), cid.as_str()); + let path = get_path(netns.as_str()); + let mut ip = String::new(); + + let output = std::process::Command::new("ip") + .arg("netns") + .arg("add") + .arg(&netns) + .output()?; + + if !output.status.success() { + return Err(Box::new(Error)); + } + + let add_command = format!( + "export CNI_PATH={} && cnitool add faasrs-cni-bridge {}", + CNI_BIN_DIR, path + ); + let output = std::process::Command::new("sh") + .arg("-c") + .arg(&add_command) + .output(); + match output { + Ok(output) => { + let stdout = String::from_utf8_lossy(&output.stdout); + let json: Value = match serde_json::from_str(&stdout) { + Ok(json) => json, + Err(e) => { + return Err(Box::new(e)); + } + }; + if let Some(ips) = json.get("ips").and_then(|ips| ips.as_array()) { + if let Some(first_ip) = ips + .get(0) + .and_then(|ip| ip.get("address")) + .and_then(|addr| addr.as_str()) + { + ip = first_ip.to_string(); + } else { + } + } else { + } + } + Err(e) => { + return Err(Box::new(e)); + } + } + + Ok((ip, path)) +} + +pub fn delete_cni_network(ns: &str, cid: &str) { + let netns = get_netns(ns, cid); + let path = get_path(&netns); + let del_command = format!( + "export CNI_PATH={} && cnitool del faasrs-cni-bridge {}", + CNI_BIN_DIR, path + ); + let _output_del = std::process::Command::new("sh") + .arg("-c") + .arg(&del_command) + .output() + .expect("Failed to execute del command"); + let _output = std::process::Command::new("ip") + .arg("netns") + .arg("delete") + .arg(&netns) + .output(); +} + +fn dir_exists(dirname: &Path) -> bool { + path_exists(dirname).map_or(false, |info| info.is_dir()) +} + +fn path_exists(path: &Path) -> Option { + match fs::metadata(path) { + Ok(metadata) => Some(metadata), + Err(_) => None, + } +} + +#[allow(unused)] +fn cni_gateway() -> Result { + let ip: IpAddr = DEFAULT_SUBNET.parse().unwrap(); + if let IpAddr::V4(ip) = ip { + let octets = &mut ip.octets(); + octets[3] = 1; + return Ok(ip.to_string()); + } + Err(Box::new(Error)) +} + +#[allow(unused)] +fn dir_empty(dirname: &Path) -> bool { + if !dir_exists(dirname) { + return false; + } + match fs::read_dir(dirname) { + Ok(mut entries) => entries.next().is_none(), + Err(_) => false, + } +} diff --git a/crates/cni/src/lib.rs b/crates/cni/src/lib.rs new file mode 100644 index 0000000..c358cd4 --- /dev/null +++ b/crates/cni/src/lib.rs @@ -0,0 +1,3 @@ +pub mod cni_network; + +type Err = Box; diff --git a/crates/service/Cargo.toml b/crates/service/Cargo.toml index 09c81c0..872590d 100644 --- a/crates/service/Cargo.toml +++ b/crates/service/Cargo.toml @@ -16,4 +16,6 @@ oci-spec = "0.6" sha2 = "0.10" hex = "0.4" my-workspace-hack = { version = "0.1", path = "../my-workspace-hack" } -handlebars= "4.1.0" \ No newline at end of file +cni = { version = "0.1", path = "../cni" } +handlebars= "4.1.0" +lazy_static = "1.4" \ No newline at end of file diff --git a/crates/service/src/lib.rs b/crates/service/src/lib.rs index b456835..59b3766 100644 --- a/crates/service/src/lib.rs +++ b/crates/service/src/lib.rs @@ -23,16 +23,28 @@ use oci_spec::image::{Arch, ImageConfiguration, ImageIndex, ImageManifest, Media use prost_types::Any; use sha2::{Digest, Sha256}; use spec::{DEFAULT_NAMESPACE, generate_spec}; -use std::{fs, sync::Arc, time::Duration, vec}; +use std::{ + collections::HashMap, + fs, + sync::{Arc, RwLock}, + time::Duration, + vec, +}; use tokio::time::timeout; // config.json,dockerhub密钥 // const DOCKER_CONFIG_DIR: &str = "/var/lib/faasd/.docker/"; +type NetnsMap = Arc>>; +lazy_static::lazy_static! { + static ref GLOBAL_NETNS_MAP: NetnsMap = Arc::new(RwLock::new(HashMap::new())); +} + type Err = Box; pub struct Service { client: Arc, + netns_map: NetnsMap, } impl Service { @@ -40,9 +52,25 @@ impl Service { let client = Client::from_path(endpoint).await.unwrap(); Ok(Service { client: Arc::new(client), + netns_map: GLOBAL_NETNS_MAP.clone(), }) } + pub async fn save_netns_ip(&self, cid: &str, netns: &str, ip: &str) { + let mut map = self.netns_map.write().unwrap(); + map.insert(cid.to_string(), (netns.to_string(), ip.to_string())); + } + + pub async fn get_netns_ip(&self, cid: &str) -> Option<(String, String)> { + let map = self.netns_map.read().unwrap(); + map.get(cid).cloned() + } + + pub async fn remove_netns_ip(&self, cid: &str) { + let mut map = self.netns_map.write().unwrap(); + map.remove(cid); + } + async fn prepare_snapshot(&self, cid: &str, ns: &str) -> Result, Err> { let parent_snapshot = self.get_parent_snapshot(cid, ns).await?; let req = PrepareSnapshotRequest { @@ -69,8 +97,8 @@ impl Service { }; let _mount = self.prepare_snapshot(cid, ns).await?; - - let spec_path = generate_spec(&cid, ns).unwrap(); + let (env, args) = self.get_env_and_args(image_name, ns).await?; + let spec_path = generate_spec(&cid, ns, args, env).unwrap(); let spec = fs::read_to_string(spec_path).unwrap(); let spec = Any { @@ -96,10 +124,10 @@ impl Service { container: Some(container), }; - let req = with_namespace!(req, namespace); + // let req = with_namespace!(req, namespace); let _resp = containers_client - .create(req) + .create(with_namespace!(req, namespace)) .await .expect("Failed to create container"); @@ -162,6 +190,7 @@ impl Service { .delete(with_namespace!(delete_request, namespace)) .await .expect("Failed to delete container"); + self.remove_netns_ip(cid).await; // println!("Container: {:?} deleted", cc); } else { @@ -191,7 +220,8 @@ impl Service { Ok(()) } - async fn create_task(&self, cid: &str, ns: &str) -> Result<(), Err> { + /// 返回任务的pid + async fn create_task(&self, cid: &str, ns: &str) -> Result { let mut sc = self.client.snapshots(); let req = MountsRequest { snapshotter: "overlayfs".to_string(), @@ -203,15 +233,17 @@ impl Service { .into_inner() .mounts; drop(sc); + let (ip, path) = cni::cni_network::create_cni_network(cid.to_string(), ns.to_string())?; + self.save_netns_ip(cid, &path, &ip).await; let mut tc = self.client.tasks(); let req = CreateTaskRequest { container_id: cid.to_string(), rootfs: mounts, ..Default::default() }; - let _resp = tc.create(with_namespace!(req, ns)).await?; - - Ok(()) + let resp = tc.create(with_namespace!(req, ns)).await?; + let pid = resp.into_inner().pid; + Ok(pid) } async fn start_task(&self, cid: &str, ns: &str) -> Result<(), Err> { @@ -578,6 +610,21 @@ impl Service { Ok(ret) } + async fn get_env_and_args( + &self, + name: &str, + ns: &str, + ) -> Result<(Vec, Vec), Err> { + let img_config = self.get_img_config(name, ns).await.unwrap(); + if let Some(config) = img_config.config() { + let env = config.env().as_ref().map_or_else(Vec::new, |v| v.clone()); + let args = config.cmd().as_ref().map_or_else(Vec::new, |v| v.clone()); + Ok((env, args)) + } else { + Err("No config found".into()) + } + } + fn check_namespace(&self, ns: &str) -> String { match ns { "" => DEFAULT_NAMESPACE.to_string(), diff --git a/crates/service/src/spec.rs b/crates/service/src/spec.rs index 6469009..0594a07 100644 --- a/crates/service/src/spec.rs +++ b/crates/service/src/spec.rs @@ -103,6 +103,7 @@ pub struct LinuxDeviceCgroup { pub struct LinuxNamespace { #[serde(rename = "type")] pub type_: String, + pub path: Option, } pub fn default_unix_caps() -> Vec { @@ -150,22 +151,27 @@ fn default_readonly_paths() -> Vec { ] } -fn default_unix_namespaces() -> Vec { +fn default_unix_namespaces(ns: &str, cid: &str) -> Vec { vec![ LinuxNamespace { type_: String::from(PID_NAMESPACE), + path: None, }, LinuxNamespace { type_: String::from(IPC_NAMESPACE), + path: None, }, LinuxNamespace { type_: String::from(UTS_NAMESPACE), + path: None, }, LinuxNamespace { type_: String::from(MOUNT_NAMESPACE), + path: None, }, LinuxNamespace { type_: String::from(NETWORK_NAMESPACE), + path: Some(format!("/var/run/netns/{}", get_netns(ns, cid))), }, ] } @@ -294,7 +300,7 @@ pub fn populate_default_unix_spec(id: &str, ns: &str) -> Spec { access: RWM.to_string(), }], }, - namespaces: default_unix_namespaces(), + namespaces: default_unix_namespaces(ns, id), }, mounts: default_mounts(), } @@ -306,12 +312,23 @@ pub fn save_spec_to_file(spec: &Spec, path: &str) -> Result<(), std::io::Error> Ok(()) } -pub fn generate_spec(id: &str, ns: &str) -> Result { +fn get_netns(ns: &str, cid: &str) -> String { + format!("{}-{}", ns, cid) +} + +pub fn generate_spec( + id: &str, + ns: &str, + args: Vec, + env: Vec, +) -> Result { let namespace = match ns { "" => DEFAULT_NAMESPACE, _ => ns, }; - let spec = populate_default_unix_spec(id, ns); + let mut spec = populate_default_unix_spec(id, ns); + spec.process.args = args; + spec.process.env = env; let path = format!("{}/{}/{}.json", PATH_TO_SPEC_PREFIX, namespace, id); save_spec_to_file(&spec, &path)?; Ok(path)