Switch to EFI boot and use official release for QEMU and GDB

runner/src/machine/microvm.rs

@@ -0,0 +1,74 @@
+use std::{
+    fs::OpenOptions,
+    io::{Seek, SeekFrom, Write},
+    path::PathBuf,
+    process::Command,
+};
+
+pub const MACHINE_ARGS: &[&str] = &[
+    "-machine",
+    "microvm,pit=on,pic=off,rtc=on",
+    "-nodefaults",
+    "-no-user-config",
+];
+
+pub const DEVICE_ARGS: &[&str] = &[
+    "-device",
+    "virtio-blk-device,drive=x0",
+    "-device",
+    "virtio-keyboard-device",
+    "-device",
+    "virtio-net-device,netdev=net01",
+];
+
+pub fn create_bootdev_image(path: PathBuf) -> PathBuf {
+    let dir = path.parent().unwrap();
+    let name = path.file_name().unwrap().to_str().unwrap().to_string();
+    let elf_path = dir.join(name.clone()).to_str().unwrap().to_string();
+    let strip_elf_path = dir
+        .join(name.clone() + ".stripped.elf")
+        .to_str()
+        .unwrap()
+        .to_string();
+
+    // We use rust-strip to reduce the kernel image size.
+    let status = Command::new("rust-strip")
+        .arg(&elf_path)
+        .arg("-o")
+        .arg(&strip_elf_path)
+        .status();
+
+    match status {
+        Ok(status) => {
+            if !status.success() {
+                panic!("Failed to strip kernel elf.");
+            }
+        }
+        Err(err) => match err.kind() {
+            std::io::ErrorKind::NotFound => panic!(
+                "Not find rust-strip command, 
+                try `cargo install cargo-binutils` and then rerun."
+            ),
+            _ => panic!("Strip kernel elf failed, err:{:#?}", err),
+        },
+    }
+
+    // Because QEMU denies a x86_64 multiboot ELF file (GRUB2 accept it, btw),
+    // modify `em_machine` to pretend to be an x86 (32-bit) ELF image,
+    //
+    // https://github.com/qemu/qemu/blob/950c4e6c94b15cd0d8b63891dddd7a8dbf458e6a/hw/i386/multiboot.c#L197
+    // Set EM_386 (0x0003) to em_machine.
+    let mut file = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .open(&strip_elf_path)
+        .unwrap();
+
+    let bytes: [u8; 2] = [0x03, 0x00];
+
+    file.seek(SeekFrom::Start(18)).unwrap();
+    file.write_all(&bytes).unwrap();
+    file.flush().unwrap();
+
+    strip_elf_path.into()
+}

runner/src/machine/mod.rs

@@ -0,0 +1,2 @@
+pub mod microvm;
+pub mod qemu_grub_efi;

runner/src/machine/qemu_grub_efi.rs

@@ -0,0 +1,207 @@
+use std::{
+    fs::{self, File},
+    io::{Read, Write},
+    path::{Path, PathBuf},
+};
+
+use crate::BootProtocol;
+
+use glob::glob;
+
+macro_rules! ovmf_prefix {
+    () => {
+        // There are 3 optional OVMF builds at your service in the dev image
+        "/root/ovmf/release/"
+        // "/root/ovmf/debug/"
+        // "/usr/share/OVMF/"
+    };
+}
+
+pub const MACHINE_ARGS: &[&str] = &[
+    "-machine",
+    "q35,kernel-irqchip=split",
+    "-drive",
+    concat!(
+        "if=pflash,format=raw,unit=0,readonly=on,file=",
+        ovmf_prefix!(),
+        "OVMF_CODE.fd"
+    ),
+    "-drive",
+    concat!(
+        "if=pflash,format=raw,unit=1,file=",
+        ovmf_prefix!(),
+        "OVMF_VARS.fd"
+    ),
+];
+
+pub const NOIOMMU_DEVICE_ARGS: &[&str] = &[
+    "-device",
+    "virtio-blk-pci,bus=pcie.0,addr=0x6,drive=x0,disable-legacy=on,disable-modern=off",
+    "-device",
+    "virtio-keyboard-pci,disable-legacy=on,disable-modern=off",
+    "-device",
+    "virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off",
+];
+
+pub const IOMMU_DEVICE_ARGS: &[&str] = &[
+    "-device",
+    "virtio-blk-pci,bus=pcie.0,addr=0x6,drive=x0,disable-legacy=on,disable-modern=off,iommu_platform=on,ats=on",
+    "-device",
+    "virtio-keyboard-pci,disable-legacy=on,disable-modern=off,iommu_platform=on,ats=on",
+    "-device",
+    "virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off,iommu_platform=on,ats=on",
+    "-device",
+    "intel-iommu,intremap=on,device-iotlb=on",
+    "-device",
+    "ioh3420,id=pcie.0,chassis=1",
+];
+
+pub const GRUB_LIB_PREFIX: &str = "/usr/lib/grub";
+pub const GRUB_VERSION: &str = "x86_64-efi";
+
+pub fn create_bootdev_image(
+    jinux_path: PathBuf,
+    initramfs_path: PathBuf,
+    grub_cfg: String,
+    protocol: BootProtocol,
+) -> PathBuf {
+    let target_dir = jinux_path.parent().unwrap();
+    let iso_root = target_dir.join("iso_root");
+
+    // Clear or make the iso dir.
+    if iso_root.exists() {
+        fs::remove_dir_all(&iso_root).unwrap();
+    }
+    fs::create_dir_all(iso_root.join("boot").join("grub")).unwrap();
+
+    // Copy the initramfs to the boot directory.
+    fs::copy(
+        initramfs_path,
+        iso_root.join("boot").join("initramfs.cpio.gz"),
+    )
+    .unwrap();
+
+    let target_path = match protocol {
+        BootProtocol::Linux => {
+            // Find the setup header in the build script output directory.
+            let bs_out_dir = glob("target/x86_64-custom/debug/build/jinux-frame-*").unwrap();
+            let header_bin = Path::new(bs_out_dir.into_iter().next().unwrap().unwrap().as_path())
+                .join("out")
+                .join("bin")
+                .join("jinux-frame-x86-boot-setup.bin");
+            // Make the `zimage`-compatible kernel image and place it in the boot directory.
+            let target_path = iso_root.join("boot").join("jinuz");
+            make_zimage(&target_path, &jinux_path.as_path(), &header_bin.as_path()).unwrap();
+            target_path
+        }
+        BootProtocol::Multiboot | BootProtocol::Multiboot2 => {
+            // Copy the kernel image to the boot directory.
+            let target_path = iso_root.join("boot").join("jinux");
+            fs::copy(&jinux_path, &target_path).unwrap();
+            target_path
+        }
+    };
+    let target_name = target_path.file_name().unwrap().to_str().unwrap();
+
+    // Write the grub.cfg file
+    let grub_cfg_path = iso_root.join("boot").join("grub").join("grub.cfg");
+    fs::write(&grub_cfg_path, grub_cfg).unwrap();
+
+    // Make the boot device CDROM image.
+    let iso_path = target_dir.join(target_name.to_string() + ".iso");
+    let mut cmd = std::process::Command::new("grub-mkrescue");
+    cmd.arg("--output").arg(&iso_path).arg(iso_root.as_os_str());
+    if !cmd.status().unwrap().success() {
+        panic!("Failed to run `{:?}`.", cmd);
+    }
+
+    iso_path.into()
+}
+
+pub fn generate_grub_cfg(
+    template_filename: &str,
+    kcmdline: &str,
+    skip_grub_menu: bool,
+    protocol: BootProtocol,
+) -> String {
+    let mut buffer = String::new();
+
+    // Read the contents of the file.
+    fs::File::open(template_filename)
+        .unwrap()
+        .read_to_string(&mut buffer)
+        .unwrap();
+
+    // Delete the first two lines that notes the file a template file.
+    let buffer = buffer.lines().skip(2).collect::<Vec<&str>>().join("\n");
+    // Set the timout style and timeout.
+    let buffer = buffer
+        .replace(
+            "#GRUB_TIMEOUT_STYLE#",
+            if skip_grub_menu { "hidden" } else { "menu" },
+        )
+        .replace("#GRUB_TIMEOUT#", if skip_grub_menu { "0" } else { "1" });
+    // Replace all occurrences of "#KERNEL_COMMAND_LINE#" with the desired value.
+    let buffer = buffer.replace("#KERNEL_COMMAND_LINE#", kcmdline);
+    // Replace the grub commands according to the protocol selected.
+    let buffer = match protocol {
+        BootProtocol::Multiboot => buffer
+            .replace("#GRUB_CMD_KERNEL#", "multiboot")
+            .replace("#KERNEL#", "/boot/jinux")
+            .replace("#GRUB_CMD_INITRAMFS#", "module --nounzip"),
+        BootProtocol::Multiboot2 => buffer
+            .replace("#GRUB_CMD_KERNEL#", "multiboot2")
+            .replace("#KERNEL#", "/boot/jinux")
+            .replace("#GRUB_CMD_INITRAMFS#", "module2 --nounzip"),
+        BootProtocol::Linux => buffer
+            .replace("#GRUB_CMD_KERNEL#", "linux")
+            .replace("#KERNEL#", "/boot/jinuz")
+            .replace("#GRUB_CMD_INITRAMFS#", "initrd"),
+    };
+
+    buffer
+}
+
+/// This function sould be used when generating the Linux x86 Boot setup header.
+/// Some fields in the Linux x86 Boot setup header should be filled after assembled.
+/// And the filled fields must have the bytes with values of 0xAB. See
+/// `framework/jinux-frame/src/arch/x86/boot/linux_boot/setup/src/header.S` for more
+/// info on this mechanism.
+fn fill_header_field(header: &mut [u8], offset: usize, value: &[u8]) {
+    let size = value.len();
+    assert_eq!(
+        &header[offset..offset + size],
+        vec![0xABu8; size].as_slice()
+    );
+    header[offset..offset + size].copy_from_slice(value);
+}
+
+fn make_zimage(path: &Path, kernel_path: &Path, header_path: &Path) -> std::io::Result<()> {
+    let mut header = Vec::new();
+    File::open(header_path)?.read_to_end(&mut header)?;
+    // Pad the header to let the payload starts with 8-byte alignment.
+    header.resize((header.len() + 7) & !7, 0x00);
+
+    let mut kernel = Vec::new();
+    File::open(kernel_path)?.read_to_end(&mut kernel)?;
+
+    let header_len = header.len();
+    let kernel_len = kernel.len();
+
+    fill_header_field(
+        &mut header,
+        0x248, /* payload_offset */
+        &(header_len as u32).to_le_bytes(),
+    );
+    fill_header_field(
+        &mut header,
+        0x24C, /* payload_length */
+        &(kernel_len as u32).to_le_bytes(),
+    );
+
+    let mut kernel_image = File::create(path)?;
+    kernel_image.write_all(&header)?;
+    kernel_image.write_all(&kernel)?;
+
+    Ok(())
+}

runner/src/main.rs

@@ -0,0 +1,286 @@
+//! jinux-runner is the Jinux runner script to ease the pain of running
+//! and testing Jinux inside a QEMU VM. It should be built and run as the
+//! cargo runner: https://doc.rust-lang.org/cargo/reference/config.html
+//!
+//! The runner will generate the filesystem image for starting Jinux. If
+//! we should use the runner in the default mode, which invokes QEMU with
+//! a GRUB boot device image, the runner would be responsible for generating
+//! the appropriate kernel image and the boot device image. It also supports
+//! to directly boot the kernel image without GRUB using the QEMU microvm
+//! machine type.
+//!
+
+pub mod machine;
+
+use std::{
+    fs::OpenOptions,
+    io::Write,
+    path::{Path, PathBuf},
+    process::Command,
+};
+
+use clap::{Parser, ValueEnum};
+
+use crate::machine::{microvm, qemu_grub_efi};
+
+#[derive(Debug, Clone, Copy, PartialEq, ValueEnum)]
+enum BootMethod {
+    QemuGrub,
+    Microvm,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, ValueEnum)]
+pub enum BootProtocol {
+    Multiboot,
+    Multiboot2,
+    Linux,
+}
+/// The CLI of this runner.
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    // Positional arguments.
+    /// The Jinux binary path.
+    path: PathBuf,
+
+    /// Provide the kernel commandline, which specifies
+    /// the init process.
+    kcmdline: String,
+
+    // Optional arguments.
+    /// Boot method. Can be one of the following items:
+    ///  - `qemu-grub`;
+    ///  - `microvm`.
+    #[arg(long, value_enum, default_value_t = BootMethod::QemuGrub)]
+    boot_method: BootMethod,
+
+    /// Boot protocol. Can be one of the following items:
+    ///  - `multiboot`;
+    ///  - `multiboot2`;
+    ///  - `linux`.
+    #[arg(long, value_enum, default_value_t = BootProtocol::Multiboot2)]
+    boot_protocol: BootProtocol,
+
+    /// Enable KVM when running QEMU.
+    #[arg(long, default_value_t = false)]
+    enable_kvm: bool,
+
+    /// Emulate Intel IOMMU by QEMU.
+    #[arg(long, default_value_t = false)]
+    emulate_iommu: bool,
+
+    /// Run QEMU as a GDB server.
+    #[arg(long, default_value_t = false)]
+    halt_for_gdb: bool,
+
+    /// Boot without displaying the GRUB menu.
+    #[arg(long, default_value_t = false)]
+    skip_grub_menu: bool,
+
+    /// Run a GDB client instead of running the kernel.
+    #[arg(long, default_value_t = false)]
+    run_gdb_client: bool,
+}
+
+pub const COMMON_ARGS: &[&str] = &[
+    "--no-reboot",
+    "-cpu",
+    "Icelake-Server,+x2apic",
+    "-m",
+    "2G",
+    "-nographic", // TODO: figure out why grub can't shown up without it
+    "-monitor",
+    "vc",
+    "-serial",
+    "mon:stdio",
+    "-display",
+    "none",
+    "-device",
+    "isa-debug-exit,iobase=0xf4,iosize=0x04",
+    "-object",
+    "filter-dump,id=filter0,netdev=net01,file=virtio-net.pcap",
+];
+
+pub fn random_hostfwd_ports() -> (u16, u16) {
+    let start = 32768u16;
+    let end = 61000u16;
+    let port1 = rand::random::<u16>() % (end - 1 - start) + start;
+    let port2 = rand::random::<u16>() % (end - port1) + port1;
+    (port1, port2)
+}
+
+pub const GDB_ARGS: &[&str] = &[
+    "-chardev",
+    "socket,path=/tmp/jinux-gdb-socket,server=on,wait=off,id=gdb0",
+    "-gdb",
+    "chardev:gdb0",
+    "-S",
+];
+
+fn run_gdb_client(path: &PathBuf, gdb_grub: bool) {
+    let path = std::fs::canonicalize(path).unwrap();
+    let mut gdb_cmd = Command::new("gdb");
+    // Set the architecture, otherwise GDB will complain about.
+    gdb_cmd.arg("-ex").arg("set arch i386:x86-64:intel");
+    let grub_script = "/tmp/jinux-gdb-grub-script";
+    if gdb_grub {
+        let grub_dir =
+            PathBuf::from(qemu_grub_efi::GRUB_LIB_PREFIX).join(qemu_grub_efi::GRUB_VERSION);
+        // Load symbols from GRUB using the provided grub gdb script.
+        // Read the contents from `gdb_grub` and
+        // replace the lines containing "target remote :1234".
+        gdb_cmd.current_dir(&grub_dir);
+        let grub_script_content = std::fs::read_to_string(grub_dir.join("gdb_grub")).unwrap();
+        let lines = grub_script_content.lines().collect::<Vec<_>>();
+        let mut f = OpenOptions::new()
+            .write(true)
+            .create(true)
+            .open(grub_script)
+            .unwrap();
+        for line in lines {
+            if line.contains("file kernel.exec") {
+                writeln!(f, "{}", line).unwrap();
+                // A horrible hack on GRUB EFI debugging.
+                // https://stackoverflow.com/questions/43872078/debug-grub2-efi-image-running-on-qemu
+                // Please use our custom built debug OVMF image to confirm the entrypoint address.
+                writeln!(f, "add-symbol-file kernel.exec 0x0007E69F000").unwrap();
+            } else if line.contains("target remote :1234") {
+                // Connect to the GDB server.
+                writeln!(f, "target remote /tmp/jinux-gdb-socket").unwrap();
+            } else {
+                writeln!(f, "{}", line).unwrap();
+            }
+        }
+        gdb_cmd.arg("-x").arg(grub_script);
+    } else {
+        // Load symbols from the kernel image.
+        gdb_cmd.arg("-ex").arg(format!("file {}", path.display()));
+        // Connect to the GDB server.
+        gdb_cmd
+            .arg("-ex")
+            .arg("target remote /tmp/jinux-gdb-socket");
+    }
+    // Connect to the GDB server and run.
+    println!("running:{:#?}", gdb_cmd);
+    gdb_cmd.status().unwrap();
+    if gdb_grub {
+        // Clean the temporary script file then return.
+        std::fs::remove_file(grub_script).unwrap();
+    }
+}
+
+fn main() {
+    let args = Args::parse();
+
+    if args.run_gdb_client {
+        let gdb_grub = args.boot_method == BootMethod::QemuGrub;
+        // You should comment out the next line if you want to debug grub instead
+        // of the kernel because this argument is not exposed by runner CLI.
+        let gdb_grub = gdb_grub && false;
+        run_gdb_client(&args.path, gdb_grub);
+        return;
+    }
+
+    let mut qemu_cmd = Command::new("qemu-system-x86_64");
+
+    qemu_cmd.args(COMMON_ARGS);
+
+    qemu_cmd.arg("-netdev");
+    let (port1, port2) = random_hostfwd_ports();
+    qemu_cmd.arg(format!(
+        "user,id=net01,hostfwd=tcp::{}-:22,hostfwd=tcp::{}-:8080",
+        port1, port2
+    ));
+    println!(
+        "[jinux-runner] Binding host ports to guest ports: ({} -> {}); ({} -> {}).",
+        port1, 22, port2, 8080
+    );
+
+    if args.halt_for_gdb {
+        if args.enable_kvm {
+            println!("[jinux-runner] Can't enable KVM when running QEMU as a GDB server. Abort.");
+            return;
+        }
+        qemu_cmd.args(GDB_ARGS);
+    }
+
+    if args.enable_kvm {
+        qemu_cmd.arg("-enable-kvm");
+    }
+    // Add machine-specific arguments
+    if args.boot_method == BootMethod::QemuGrub {
+        qemu_cmd.args(qemu_grub_efi::MACHINE_ARGS);
+    } else if args.boot_method == BootMethod::Microvm {
+        qemu_cmd.args(microvm::MACHINE_ARGS);
+    }
+    // Add device arguments
+    if args.boot_method == BootMethod::Microvm {
+        qemu_cmd.args(microvm::DEVICE_ARGS);
+    } else if args.emulate_iommu {
+        qemu_cmd.args(qemu_grub_efi::IOMMU_DEVICE_ARGS);
+    } else {
+        qemu_cmd.args(qemu_grub_efi::NOIOMMU_DEVICE_ARGS);
+    }
+
+    let fs_image = create_fs_image(args.path.as_path());
+    qemu_cmd.arg("-drive");
+    qemu_cmd.arg(fs_image);
+
+    if args.boot_method == BootMethod::Microvm {
+        let image = microvm::create_bootdev_image(args.path);
+        qemu_cmd.arg("-kernel");
+        qemu_cmd.arg(image.as_os_str());
+        qemu_cmd.arg("-append");
+        qemu_cmd.arg(&args.kcmdline);
+        qemu_cmd.arg("-initrd");
+        qemu_cmd.arg("regression/build/initramfs.cpio.gz");
+    } else if args.boot_method == BootMethod::QemuGrub {
+        let grub_cfg = qemu_grub_efi::generate_grub_cfg(
+            "runner/grub/grub.cfg.template",
+            &args.kcmdline,
+            args.skip_grub_menu,
+            args.boot_protocol,
+        );
+        let initramfs_path = PathBuf::from("regression/build/initramfs.cpio.gz");
+        let bootdev_image = qemu_grub_efi::create_bootdev_image(
+            args.path,
+            initramfs_path,
+            grub_cfg,
+            args.boot_protocol,
+        );
+        qemu_cmd.arg("-cdrom");
+        qemu_cmd.arg(bootdev_image.as_os_str());
+    }
+
+    println!("[jinux-runner] Running: {:#?}", qemu_cmd);
+
+    let exit_status = qemu_cmd.status().unwrap();
+    if !exit_status.success() {
+        // FIXME: Exit code manipulation is not needed when using non-x86 QEMU
+        let qemu_exit_code = exit_status.code().unwrap();
+        let kernel_exit_code = qemu_exit_code >> 1;
+        match kernel_exit_code {
+            0x10 /*jinux_frame::QemuExitCode::Success*/ => { std::process::exit(0); },
+            0x20 /*jinux_frame::QemuExitCode::Failed*/ => { std::process::exit(1); },
+            _ => { std::process::exit(qemu_exit_code) },
+        }
+    }
+}
+
+pub fn create_fs_image(path: &Path) -> String {
+    let mut fs_img_path = path.parent().unwrap().to_str().unwrap().to_string();
+    fs_img_path.push_str("/fs.img");
+    let path = Path::new(fs_img_path.as_str());
+    if path.exists() {
+        return format!("file={},if=none,format=raw,id=x0", fs_img_path.as_str());
+    }
+    let f = OpenOptions::new()
+        .read(true)
+        .write(true)
+        .create(true)
+        .open(fs_img_path.as_str())
+        .unwrap();
+    // 32MiB
+    f.set_len(64 * 1024 * 1024).unwrap();
+    format!("file={},if=none,format=raw,id=x0", fs_img_path.as_str())
+}