mirror of
https://github.com/asterinas/asterinas.git
synced 2025-06-09 05:16:47 +00:00
Make CPU-local and early ACPI initialization heap-less
This commit is contained in:
parent
7496b24da1
commit
92bc8cbbf7
4
Cargo.lock
generated
4
Cargo.lock
generated
@ -4,9 +4,9 @@ version = 4
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "acpi"
|
name = "acpi"
|
||||||
version = "5.1.0"
|
version = "5.2.0"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "9e42f25ac5fa51f4188d14baf8f387a97dcd8639644b2f3df948bf5f6dd7d6fa"
|
checksum = "94476c7ef97af4c4d998b3f422c1b01d5211aad57c80ed200baf148d1f1efab6"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bit_field",
|
"bit_field",
|
||||||
"bitflags 2.6.0",
|
"bitflags 2.6.0",
|
||||||
|
@ -42,7 +42,7 @@ xarray = { git = "https://github.com/asterinas/xarray", version = "0.1.0" }
|
|||||||
[target.x86_64-unknown-none.dependencies]
|
[target.x86_64-unknown-none.dependencies]
|
||||||
x86_64 = "0.14.13"
|
x86_64 = "0.14.13"
|
||||||
x86 = "0.52.0"
|
x86 = "0.52.0"
|
||||||
acpi = "5.1.0"
|
acpi = "=5.2.0" # This upstream often bump minor versions with API changes
|
||||||
multiboot2 = "0.23.0"
|
multiboot2 = "0.23.0"
|
||||||
iced-x86 = { version = "1.21.0", default-features = false, features = [
|
iced-x86 = { version = "1.21.0", default-features = false, features = [
|
||||||
"no_std",
|
"no_std",
|
||||||
|
@ -22,23 +22,13 @@ pub(crate) fn init_cvm_guest() {
|
|||||||
// Unimplemented, no-op
|
// Unimplemented, no-op
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn init_on_bsp() {
|
pub(crate) unsafe fn late_init_on_bsp() {
|
||||||
// SAFETY: this function is only called once on BSP.
|
// SAFETY: this function is only called once on BSP.
|
||||||
unsafe {
|
unsafe {
|
||||||
trap::init(true);
|
trap::init(true);
|
||||||
}
|
}
|
||||||
irq::init();
|
irq::init();
|
||||||
|
|
||||||
// SAFETY: they are only called once on BSP and ACPI has been initialized.
|
|
||||||
unsafe {
|
|
||||||
crate::cpu::init_num_cpus();
|
|
||||||
crate::cpu::set_this_cpu_id(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
// SAFETY: no CPU local objects have been accessed by this far. And
|
|
||||||
// we are on the BSP.
|
|
||||||
unsafe { crate::cpu::local::init_on_bsp() };
|
|
||||||
|
|
||||||
crate::boot::smp::boot_all_aps();
|
crate::boot::smp::boot_all_aps();
|
||||||
|
|
||||||
timer::init();
|
timer::init();
|
||||||
|
@ -27,11 +27,9 @@
|
|||||||
//! This sequence does not need to be strictly followed, and there may be
|
//! This sequence does not need to be strictly followed, and there may be
|
||||||
//! different considerations in different systems.
|
//! different considerations in different systems.
|
||||||
|
|
||||||
use acpi::platform::PlatformInfo;
|
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
arch::x86::kernel::{
|
arch::x86::kernel::{
|
||||||
acpi::ACPI_TABLES,
|
acpi::get_acpi_tables,
|
||||||
apic::{
|
apic::{
|
||||||
self, ApicId, DeliveryMode, DeliveryStatus, DestinationMode, DestinationShorthand, Icr,
|
self, ApicId, DeliveryMode, DeliveryStatus, DestinationMode, DestinationShorthand, Icr,
|
||||||
Level, TriggerMode,
|
Level, TriggerMode,
|
||||||
@ -44,14 +42,20 @@ use crate::{
|
|||||||
///
|
///
|
||||||
/// This function needs to be called after the OS initializes the ACPI table.
|
/// This function needs to be called after the OS initializes the ACPI table.
|
||||||
pub(crate) fn get_num_processors() -> Option<u32> {
|
pub(crate) fn get_num_processors() -> Option<u32> {
|
||||||
if !ACPI_TABLES.is_completed() {
|
let acpi_tables = get_acpi_tables()?;
|
||||||
return None;
|
let mut local_apic_counts = 0;
|
||||||
}
|
acpi_tables
|
||||||
let processor_info = PlatformInfo::new(&*ACPI_TABLES.get().unwrap().lock())
|
.find_table::<acpi::madt::Madt>()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.processor_info
|
.get()
|
||||||
.unwrap();
|
.entries()
|
||||||
Some(processor_info.application_processors.len() as u32 + 1)
|
.for_each(|entry| {
|
||||||
|
if let acpi::madt::MadtEntry::LocalApic(_) = entry {
|
||||||
|
local_apic_counts += 1;
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
Some(local_apic_counts)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Brings up all application processors.
|
/// Brings up all application processors.
|
||||||
|
@ -13,7 +13,7 @@ use acpi::fadt::Fadt;
|
|||||||
use x86_64::instructions::port::{ReadOnlyAccess, WriteOnlyAccess};
|
use x86_64::instructions::port::{ReadOnlyAccess, WriteOnlyAccess};
|
||||||
|
|
||||||
use super::io_port::IoPort;
|
use super::io_port::IoPort;
|
||||||
use crate::arch::x86::kernel::acpi::ACPI_TABLES;
|
use crate::arch::x86::kernel::acpi::get_acpi_tables;
|
||||||
|
|
||||||
/// CMOS address I/O port
|
/// CMOS address I/O port
|
||||||
pub static CMOS_ADDRESS: IoPort<u8, WriteOnlyAccess> = unsafe { IoPort::new(0x70) };
|
pub static CMOS_ADDRESS: IoPort<u8, WriteOnlyAccess> = unsafe { IoPort::new(0x70) };
|
||||||
@ -23,10 +23,8 @@ pub static CMOS_DATA: IoPort<u8, ReadOnlyAccess> = unsafe { IoPort::new(0x71) };
|
|||||||
|
|
||||||
/// Gets the century register location. This function is used in RTC(Real Time Clock) module initialization.
|
/// Gets the century register location. This function is used in RTC(Real Time Clock) module initialization.
|
||||||
pub fn century_register() -> Option<u8> {
|
pub fn century_register() -> Option<u8> {
|
||||||
if !ACPI_TABLES.is_completed() {
|
let acpi_tables = get_acpi_tables()?;
|
||||||
return None;
|
match acpi_tables.find_table::<Fadt>() {
|
||||||
}
|
|
||||||
match ACPI_TABLES.get().unwrap().lock().find_table::<Fadt>() {
|
|
||||||
Ok(a) => Some(a.century),
|
Ok(a) => Some(a.century),
|
||||||
Err(er) => None,
|
Err(er) => None,
|
||||||
}
|
}
|
||||||
|
@ -74,9 +74,9 @@ unsafe impl AcpiTable for DmarHeader {
|
|||||||
impl Dmar {
|
impl Dmar {
|
||||||
/// Creates a instance from ACPI table.
|
/// Creates a instance from ACPI table.
|
||||||
pub fn new() -> Option<Self> {
|
pub fn new() -> Option<Self> {
|
||||||
let acpi_table_lock = super::ACPI_TABLES.get()?.lock();
|
let acpi_table = super::get_acpi_tables()?;
|
||||||
|
|
||||||
let dmar_mapping = acpi_table_lock.find_table::<DmarHeader>().ok()?;
|
let dmar_mapping = acpi_table.find_table::<DmarHeader>().ok()?;
|
||||||
|
|
||||||
let header = *dmar_mapping;
|
let header = *dmar_mapping;
|
||||||
// SAFETY: `find_table` returns a region of memory that belongs to the ACPI table. This
|
// SAFETY: `find_table` returns a region of memory that belongs to the ACPI table. This
|
||||||
|
@ -5,19 +5,15 @@ pub mod remapping;
|
|||||||
|
|
||||||
use core::ptr::NonNull;
|
use core::ptr::NonNull;
|
||||||
|
|
||||||
use acpi::{rsdp::Rsdp, AcpiHandler, AcpiTables};
|
use acpi::{platform::PlatformInfo, rsdp::Rsdp, AcpiHandler, AcpiTables};
|
||||||
use log::{info, warn};
|
use log::{info, warn};
|
||||||
use spin::Once;
|
use spin::Once;
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
boot::{self, BootloaderAcpiArg},
|
boot::{self, BootloaderAcpiArg},
|
||||||
mm::paddr_to_vaddr,
|
mm::paddr_to_vaddr,
|
||||||
sync::SpinLock,
|
|
||||||
};
|
};
|
||||||
|
|
||||||
/// RSDP information, key is the signature, value is the virtual address of the signature
|
|
||||||
pub static ACPI_TABLES: Once<SpinLock<AcpiTables<AcpiMemoryHandler>>> = Once::new();
|
|
||||||
|
|
||||||
#[derive(Debug, Clone)]
|
#[derive(Debug, Clone)]
|
||||||
pub struct AcpiMemoryHandler {}
|
pub struct AcpiMemoryHandler {}
|
||||||
|
|
||||||
@ -42,7 +38,7 @@ impl AcpiHandler for AcpiMemoryHandler {
|
|||||||
fn unmap_physical_region<T>(_region: &acpi::PhysicalMapping<Self, T>) {}
|
fn unmap_physical_region<T>(_region: &acpi::PhysicalMapping<Self, T>) {}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn init() {
|
pub(crate) fn get_acpi_tables() -> Option<AcpiTables<AcpiMemoryHandler>> {
|
||||||
let acpi_tables = match boot::EARLY_INFO.get().unwrap().acpi_arg {
|
let acpi_tables = match boot::EARLY_INFO.get().unwrap().acpi_arg {
|
||||||
BootloaderAcpiArg::Rsdp(addr) => unsafe {
|
BootloaderAcpiArg::Rsdp(addr) => unsafe {
|
||||||
AcpiTables::from_rsdp(AcpiMemoryHandler {}, addr).unwrap()
|
AcpiTables::from_rsdp(AcpiMemoryHandler {}, addr).unwrap()
|
||||||
@ -62,16 +58,39 @@ pub fn init() {
|
|||||||
},
|
},
|
||||||
Err(_) => {
|
Err(_) => {
|
||||||
warn!("ACPI info not found!");
|
warn!("ACPI info not found!");
|
||||||
return;
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
|
Some(acpi_tables)
|
||||||
|
}
|
||||||
|
|
||||||
|
static PLATFORM_INFO: Once<PlatformInfo<'static, alloc::alloc::Global>> = Once::new();
|
||||||
|
|
||||||
|
/// Initializes the platform information by parsing ACPI tables in to the heap.
|
||||||
|
///
|
||||||
|
/// Must be called after the heap is initialized.
|
||||||
|
pub(crate) fn init() {
|
||||||
|
let Some(acpi_tables) = get_acpi_tables() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
|
||||||
for header in acpi_tables.headers() {
|
for header in acpi_tables.headers() {
|
||||||
info!("ACPI found signature:{:?}", header.signature);
|
info!("ACPI found signature:{:?}", header.signature);
|
||||||
}
|
}
|
||||||
ACPI_TABLES.call_once(|| SpinLock::new(acpi_tables));
|
|
||||||
|
|
||||||
info!("acpi init complete");
|
let platform_info = PlatformInfo::new(&acpi_tables).unwrap();
|
||||||
|
PLATFORM_INFO.call_once(|| platform_info);
|
||||||
|
|
||||||
|
info!("ACPI initialization complete");
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gets the platform information.
|
||||||
|
///
|
||||||
|
/// Must be called after [`init()`]. Otherwise, there may not be any platform
|
||||||
|
/// information even if the system has ACPI tables.
|
||||||
|
pub(crate) fn get_platform_info() -> Option<&'static PlatformInfo<'static, alloc::alloc::Global>> {
|
||||||
|
PLATFORM_INFO.get()
|
||||||
}
|
}
|
||||||
|
@ -5,7 +5,6 @@
|
|||||||
use alloc::{vec, vec::Vec};
|
use alloc::{vec, vec::Vec};
|
||||||
use core::ptr::NonNull;
|
use core::ptr::NonNull;
|
||||||
|
|
||||||
use acpi::PlatformInfo;
|
|
||||||
use bit_field::BitField;
|
use bit_field::BitField;
|
||||||
use cfg_if::cfg_if;
|
use cfg_if::cfg_if;
|
||||||
use log::info;
|
use log::info;
|
||||||
@ -16,7 +15,7 @@ use volatile::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
arch::{iommu::has_interrupt_remapping, x86::kernel::acpi::ACPI_TABLES},
|
arch::{iommu::has_interrupt_remapping, x86::kernel::acpi::get_platform_info},
|
||||||
mm::paddr_to_vaddr,
|
mm::paddr_to_vaddr,
|
||||||
sync::SpinLock,
|
sync::SpinLock,
|
||||||
trap::IrqLine,
|
trap::IrqLine,
|
||||||
@ -180,7 +179,7 @@ impl IoApicAccess {
|
|||||||
pub static IO_APIC: Once<Vec<SpinLock<IoApic>>> = Once::new();
|
pub static IO_APIC: Once<Vec<SpinLock<IoApic>>> = Once::new();
|
||||||
|
|
||||||
pub fn init() {
|
pub fn init() {
|
||||||
if !ACPI_TABLES.is_completed() {
|
let Some(platform_info) = get_platform_info() else {
|
||||||
IO_APIC.call_once(|| {
|
IO_APIC.call_once(|| {
|
||||||
// FIXME: Is it possible to have an address that is not the default 0xFEC0_0000?
|
// FIXME: Is it possible to have an address that is not the default 0xFEC0_0000?
|
||||||
// Need to find a way to determine if it is a valid address or not.
|
// Need to find a way to determine if it is a valid address or not.
|
||||||
@ -211,10 +210,8 @@ pub fn init() {
|
|||||||
vec![SpinLock::new(IoApic::new(io_apic, 0))]
|
vec![SpinLock::new(IoApic::new(io_apic, 0))]
|
||||||
});
|
});
|
||||||
return;
|
return;
|
||||||
}
|
};
|
||||||
let table = ACPI_TABLES.get().unwrap().lock();
|
match &platform_info.interrupt_model {
|
||||||
let platform_info = PlatformInfo::new(&*table).unwrap();
|
|
||||||
match platform_info.interrupt_model {
|
|
||||||
acpi::InterruptModel::Unknown => panic!("not found APIC in ACPI Table"),
|
acpi::InterruptModel::Unknown => panic!("not found APIC in ACPI Table"),
|
||||||
acpi::InterruptModel::Apic(apic) => {
|
acpi::InterruptModel::Apic(apic) => {
|
||||||
let mut vec = Vec::new();
|
let mut vec = Vec::new();
|
||||||
|
@ -63,19 +63,21 @@ pub(crate) fn init_cvm_guest() {
|
|||||||
|
|
||||||
static CPU_FEATURES: Once<FeatureInfo> = Once::new();
|
static CPU_FEATURES: Once<FeatureInfo> = Once::new();
|
||||||
|
|
||||||
pub(crate) fn init_on_bsp() {
|
/// Architecture-specific initialization on the bootstrapping processor.
|
||||||
|
///
|
||||||
|
/// It should be called when the heap and frame allocators are available.
|
||||||
|
///
|
||||||
|
/// # Safety
|
||||||
|
///
|
||||||
|
/// This function must be called only once on the bootstrapping processor.
|
||||||
|
pub(crate) unsafe fn late_init_on_bsp() {
|
||||||
// SAFETY: this function is only called once on BSP.
|
// SAFETY: this function is only called once on BSP.
|
||||||
unsafe {
|
unsafe {
|
||||||
crate::arch::trap::init(true);
|
crate::arch::trap::init(true);
|
||||||
}
|
}
|
||||||
irq::init();
|
irq::init();
|
||||||
kernel::acpi::init();
|
|
||||||
|
|
||||||
// SAFETY: they are only called once on BSP and ACPI has been initialized.
|
kernel::acpi::init();
|
||||||
unsafe {
|
|
||||||
crate::cpu::init_num_cpus();
|
|
||||||
crate::cpu::set_this_cpu_id(0);
|
|
||||||
}
|
|
||||||
|
|
||||||
match kernel::apic::init() {
|
match kernel::apic::init() {
|
||||||
Ok(_) => {
|
Ok(_) => {
|
||||||
@ -88,12 +90,6 @@ pub(crate) fn init_on_bsp() {
|
|||||||
}
|
}
|
||||||
serial::callback_init();
|
serial::callback_init();
|
||||||
|
|
||||||
// SAFETY: no CPU local objects have been accessed by this far. And
|
|
||||||
// we are on the BSP.
|
|
||||||
unsafe { crate::cpu::local::init_on_bsp() };
|
|
||||||
|
|
||||||
crate::sync::init();
|
|
||||||
|
|
||||||
crate::boot::smp::boot_all_aps();
|
crate::boot::smp::boot_all_aps();
|
||||||
|
|
||||||
timer::init();
|
timer::init();
|
||||||
|
@ -11,7 +11,7 @@ use volatile::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use crate::{
|
use crate::{
|
||||||
arch::x86::kernel::{acpi::ACPI_TABLES, apic::ioapic},
|
arch::x86::kernel::{acpi::get_acpi_tables, apic::ioapic},
|
||||||
mm::paddr_to_vaddr,
|
mm::paddr_to_vaddr,
|
||||||
trap::IrqLine,
|
trap::IrqLine,
|
||||||
};
|
};
|
||||||
@ -135,11 +135,9 @@ impl Hpet {
|
|||||||
/// HPET init, need to init IOAPIC before init this function
|
/// HPET init, need to init IOAPIC before init this function
|
||||||
#[expect(dead_code)]
|
#[expect(dead_code)]
|
||||||
pub fn init() -> Result<(), AcpiError> {
|
pub fn init() -> Result<(), AcpiError> {
|
||||||
let hpet_info = {
|
let tables = get_acpi_tables().unwrap();
|
||||||
let lock = ACPI_TABLES.get().unwrap().lock();
|
|
||||||
HpetInfo::new(&*lock)?
|
|
||||||
};
|
|
||||||
|
|
||||||
|
let hpet_info = HpetInfo::new(&tables)?;
|
||||||
assert_ne!(hpet_info.base_address, 0, "HPET address should not be zero");
|
assert_ne!(hpet_info.base_address, 0, "HPET address should not be zero");
|
||||||
|
|
||||||
let base = NonNull::new(paddr_to_vaddr(hpet_info.base_address) as *mut u8).unwrap();
|
let base = NonNull::new(paddr_to_vaddr(hpet_info.base_address) as *mut u8).unwrap();
|
||||||
|
@ -141,24 +141,15 @@ impl<T: 'static + Sync> CpuLocal<T> {
|
|||||||
/// Panics if the CPU ID is out of range.
|
/// Panics if the CPU ID is out of range.
|
||||||
pub fn get_on_cpu(&'static self, cpu_id: CpuId) -> &'static T {
|
pub fn get_on_cpu(&'static self, cpu_id: CpuId) -> &'static T {
|
||||||
super::has_init::assert_true();
|
super::has_init::assert_true();
|
||||||
|
|
||||||
let cpu_id = cpu_id.as_usize();
|
|
||||||
|
|
||||||
// If on the BSP, just use the statically linked storage.
|
// If on the BSP, just use the statically linked storage.
|
||||||
if cpu_id == 0 {
|
if cpu_id.as_usize() == 0 {
|
||||||
return &self.0;
|
return &self.0;
|
||||||
}
|
}
|
||||||
|
|
||||||
// SAFETY: Here we use `Once::get_unchecked` to make getting the CPU-
|
// SAFETY: Here we use `Once::get_unchecked` to make getting the CPU-
|
||||||
// local base faster. The storages must be initialized here so it is
|
// local base faster. The storages must be initialized here (since this
|
||||||
// safe to do so.
|
// is not the BSP) so it is safe to do so.
|
||||||
let base = unsafe {
|
let base = unsafe { super::CPU_LOCAL_STORAGES.get_unchecked().get(cpu_id) };
|
||||||
super::CPU_LOCAL_STORAGES
|
|
||||||
.get_unchecked()
|
|
||||||
.get(cpu_id - 1)
|
|
||||||
.unwrap()
|
|
||||||
.start_paddr()
|
|
||||||
};
|
|
||||||
let base = crate::mm::paddr_to_vaddr(base);
|
let base = crate::mm::paddr_to_vaddr(base);
|
||||||
|
|
||||||
let offset = self.get_offset();
|
let offset = self.get_offset();
|
||||||
|
@ -34,16 +34,17 @@ mod cpu_local;
|
|||||||
|
|
||||||
pub(crate) mod single_instr;
|
pub(crate) mod single_instr;
|
||||||
|
|
||||||
use alloc::vec::Vec;
|
use core::alloc::Layout;
|
||||||
|
|
||||||
use align_ext::AlignExt;
|
use align_ext::AlignExt;
|
||||||
pub use cell::CpuLocalCell;
|
pub use cell::CpuLocalCell;
|
||||||
pub use cpu_local::{CpuLocal, CpuLocalDerefGuard};
|
pub use cpu_local::{CpuLocal, CpuLocalDerefGuard};
|
||||||
use spin::Once;
|
use spin::Once;
|
||||||
|
|
||||||
|
use super::CpuId;
|
||||||
use crate::{
|
use crate::{
|
||||||
arch,
|
arch,
|
||||||
mm::{frame::Segment, kspace::KernelMeta, paddr_to_vaddr, FrameAllocOptions, PAGE_SIZE},
|
mm::{frame::allocator, paddr_to_vaddr, Paddr, PAGE_SIZE},
|
||||||
};
|
};
|
||||||
|
|
||||||
// These symbols are provided by the linker script.
|
// These symbols are provided by the linker script.
|
||||||
@ -52,30 +53,81 @@ extern "C" {
|
|||||||
fn __cpu_local_end();
|
fn __cpu_local_end();
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Sets the base address of the CPU-local storage for the bootstrap processor.
|
/// The BSP initializes the CPU-local areas for APs.
|
||||||
///
|
static CPU_LOCAL_STORAGES: Once<CpuLocalStoragePointers> = Once::new();
|
||||||
/// It should be called early to let [`crate::task::disable_preempt`] work,
|
|
||||||
/// which needs to update a CPU-local preemption info. Otherwise it may
|
|
||||||
/// panic when calling [`crate::task::disable_preempt`]. It is needed since
|
|
||||||
/// heap allocations need to disable preemption, which would happen in the
|
|
||||||
/// very early stage of the kernel.
|
|
||||||
///
|
|
||||||
/// # Safety
|
|
||||||
///
|
|
||||||
/// It should be called only once and only on the BSP.
|
|
||||||
pub(crate) unsafe fn early_init_bsp_local_base() {
|
|
||||||
let start_base_va = __cpu_local_start as usize as u64;
|
|
||||||
|
|
||||||
// SAFETY: The base to be set is the start of the `.cpu_local` section,
|
struct CpuLocalStoragePointers(&'static mut [Paddr]);
|
||||||
// where accessing the CPU-local objects have defined behaviors.
|
|
||||||
unsafe {
|
impl CpuLocalStoragePointers {
|
||||||
arch::cpu::local::set_base(start_base_va);
|
/// Allocates frames for storing CPU-local data for each AP.
|
||||||
|
///
|
||||||
|
/// # Safety
|
||||||
|
///
|
||||||
|
/// The caller must ensure that the `num_cpus` matches the number of all
|
||||||
|
/// CPUs that will access CPU local storage.
|
||||||
|
///
|
||||||
|
/// The BSP's CPU-local storage should not be touched before calling
|
||||||
|
/// this method, since the CPU-local storage for APs is copied from the
|
||||||
|
/// BSP's CPU-local storage.
|
||||||
|
unsafe fn allocate(num_cpus: usize) -> Self {
|
||||||
|
let num_aps = num_cpus - 1; // BSP does not need allocated storage.
|
||||||
|
assert!(num_aps > 0, "No APs to allocate CPU-local storage");
|
||||||
|
|
||||||
|
// Allocate a region to store the pointers to the CPU-local storage segments.
|
||||||
|
let size = (core::mem::size_of::<Paddr>() * num_aps).align_up(PAGE_SIZE);
|
||||||
|
let addr =
|
||||||
|
allocator::early_alloc(Layout::from_size_align(size, PAGE_SIZE).unwrap()).unwrap();
|
||||||
|
let ptr = paddr_to_vaddr(addr) as *mut Paddr;
|
||||||
|
// SAFETY: The memory is allocated and the pointer is valid.
|
||||||
|
unsafe {
|
||||||
|
core::ptr::write_bytes(ptr as *mut u8, 0, size);
|
||||||
|
}
|
||||||
|
// SAFETY: The memory would not be deallocated. And it is valid.
|
||||||
|
let res = Self(unsafe { core::slice::from_raw_parts_mut(ptr, num_aps) });
|
||||||
|
|
||||||
|
// Allocate the CPU-local storage segments for APs.
|
||||||
|
let bsp_base_va = __cpu_local_start as usize;
|
||||||
|
let bsp_end_va = __cpu_local_end as usize;
|
||||||
|
for id in 0..num_aps {
|
||||||
|
let ap_pages = {
|
||||||
|
let nbytes = (bsp_end_va - bsp_base_va).align_up(PAGE_SIZE);
|
||||||
|
allocator::early_alloc(Layout::from_size_align(nbytes, PAGE_SIZE).unwrap()).unwrap()
|
||||||
|
};
|
||||||
|
let ap_pages_ptr = paddr_to_vaddr(ap_pages) as *mut u8;
|
||||||
|
|
||||||
|
// SAFETY: The BSP has not initialized the CPU-local area, so the objects in
|
||||||
|
// in the `.cpu_local` section can be bitwise bulk copied to the AP's local
|
||||||
|
// storage. The destination memory is allocated so it is valid to write to.
|
||||||
|
unsafe {
|
||||||
|
core::ptr::copy_nonoverlapping(
|
||||||
|
bsp_base_va as *const u8,
|
||||||
|
ap_pages_ptr,
|
||||||
|
bsp_end_va - bsp_base_va,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
res.0[id] = ap_pages;
|
||||||
|
}
|
||||||
|
|
||||||
|
res
|
||||||
|
}
|
||||||
|
|
||||||
|
fn get(&self, cpu_id: CpuId) -> Paddr {
|
||||||
|
let offset = cpu_id
|
||||||
|
.as_usize()
|
||||||
|
.checked_sub(1)
|
||||||
|
.expect("The BSP does not have allocated CPU-local storage");
|
||||||
|
|
||||||
|
let paddr = self.0[offset];
|
||||||
|
assert!(
|
||||||
|
paddr != 0,
|
||||||
|
"The CPU-local storage for CPU {} is not allocated",
|
||||||
|
cpu_id.as_usize()
|
||||||
|
);
|
||||||
|
paddr
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// The BSP initializes the CPU-local areas for APs.
|
|
||||||
static CPU_LOCAL_STORAGES: Once<Vec<Segment<KernelMeta>>> = Once::new();
|
|
||||||
|
|
||||||
/// Initializes the CPU local data for the bootstrap processor (BSP).
|
/// Initializes the CPU local data for the bootstrap processor (BSP).
|
||||||
///
|
///
|
||||||
/// # Safety
|
/// # Safety
|
||||||
@ -86,39 +138,17 @@ static CPU_LOCAL_STORAGES: Once<Vec<Segment<KernelMeta>>> = Once::new();
|
|||||||
/// this function being called, otherwise copying non-constant values
|
/// this function being called, otherwise copying non-constant values
|
||||||
/// will result in pretty bad undefined behavior.
|
/// will result in pretty bad undefined behavior.
|
||||||
pub unsafe fn init_on_bsp() {
|
pub unsafe fn init_on_bsp() {
|
||||||
let bsp_base_va = __cpu_local_start as usize;
|
|
||||||
let bsp_end_va = __cpu_local_end as usize;
|
|
||||||
|
|
||||||
let num_cpus = super::num_cpus();
|
let num_cpus = super::num_cpus();
|
||||||
|
|
||||||
let mut cpu_local_storages = Vec::with_capacity(num_cpus - 1);
|
if num_cpus > 1 {
|
||||||
for _ in 1..num_cpus {
|
// SAFETY: The number of CPUs is correct. And other conditions are
|
||||||
let ap_pages = {
|
// the caller's safety conditions.
|
||||||
let nbytes = (bsp_end_va - bsp_base_va).align_up(PAGE_SIZE);
|
let cpu_local_storages = unsafe { CpuLocalStoragePointers::allocate(num_cpus) };
|
||||||
FrameAllocOptions::new()
|
|
||||||
.zeroed(false)
|
|
||||||
.alloc_segment_with(nbytes / PAGE_SIZE, |_| KernelMeta)
|
|
||||||
.unwrap()
|
|
||||||
};
|
|
||||||
let ap_pages_ptr = paddr_to_vaddr(ap_pages.start_paddr()) as *mut u8;
|
|
||||||
|
|
||||||
// SAFETY: The BSP has not initialized the CPU-local area, so the objects in
|
CPU_LOCAL_STORAGES.call_once(|| cpu_local_storages);
|
||||||
// in the `.cpu_local` section can be bitwise bulk copied to the AP's local
|
|
||||||
// storage. The destination memory is allocated so it is valid to write to.
|
|
||||||
unsafe {
|
|
||||||
core::ptr::copy_nonoverlapping(
|
|
||||||
bsp_base_va as *const u8,
|
|
||||||
ap_pages_ptr,
|
|
||||||
bsp_end_va - bsp_base_va,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
cpu_local_storages.push(ap_pages);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
CPU_LOCAL_STORAGES.call_once(|| cpu_local_storages);
|
arch::cpu::local::set_base(__cpu_local_start as usize as u64);
|
||||||
|
|
||||||
arch::cpu::local::set_base(bsp_base_va as u64);
|
|
||||||
|
|
||||||
has_init::set_true();
|
has_init::set_true();
|
||||||
}
|
}
|
||||||
@ -132,17 +162,14 @@ pub unsafe fn init_on_ap(cpu_id: u32) {
|
|||||||
let ap_pages = CPU_LOCAL_STORAGES
|
let ap_pages = CPU_LOCAL_STORAGES
|
||||||
.get()
|
.get()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.get(cpu_id as usize - 1)
|
.get(CpuId::try_from(cpu_id as usize).unwrap());
|
||||||
.unwrap();
|
|
||||||
|
|
||||||
let ap_pages_ptr = paddr_to_vaddr(ap_pages.start_paddr()) as *mut u32;
|
let ap_pages_ptr = paddr_to_vaddr(ap_pages) as *mut u32;
|
||||||
|
|
||||||
// SAFETY: the memory will be dedicated to the AP. And we are on the AP.
|
// SAFETY: the memory will be dedicated to the AP. And we are on the AP.
|
||||||
unsafe {
|
unsafe {
|
||||||
arch::cpu::local::set_base(ap_pages_ptr as u64);
|
arch::cpu::local::set_base(ap_pages_ptr as u64);
|
||||||
}
|
}
|
||||||
|
|
||||||
crate::task::reset_preempt_info();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
mod has_init {
|
mod has_init {
|
||||||
|
@ -47,7 +47,7 @@ mod util;
|
|||||||
|
|
||||||
use core::sync::atomic::{AtomicBool, Ordering};
|
use core::sync::atomic::{AtomicBool, Ordering};
|
||||||
|
|
||||||
pub use ostd_macros::{main, panic_handler};
|
pub use ostd_macros::{global_frame_allocator, main, panic_handler};
|
||||||
pub use ostd_pod::Pod;
|
pub use ostd_pod::Pod;
|
||||||
|
|
||||||
pub use self::{error::Error, prelude::Result};
|
pub use self::{error::Error, prelude::Result};
|
||||||
@ -74,19 +74,33 @@ unsafe fn init() {
|
|||||||
|
|
||||||
logger::init();
|
logger::init();
|
||||||
|
|
||||||
// SAFETY: This function is called only once and only on the BSP.
|
// SAFETY: They are only called once on BSP and ACPI has been initialized.
|
||||||
unsafe { cpu::local::early_init_bsp_local_base() };
|
// No CPU local objects have been accessed by this far.
|
||||||
|
unsafe {
|
||||||
|
cpu::init_num_cpus();
|
||||||
|
cpu::local::init_on_bsp();
|
||||||
|
cpu::set_this_cpu_id(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
// SAFETY: We are on the BSP and APs are not yet started.
|
||||||
|
let meta_pages = unsafe { mm::frame::meta::init() };
|
||||||
|
// The frame allocator should be initialized immediately after the metadata
|
||||||
|
// is initialized. Otherwise the boot page table can't allocate frames.
|
||||||
|
// SAFETY: This function is called only once.
|
||||||
|
unsafe { mm::frame::allocator::init() };
|
||||||
|
|
||||||
|
mm::kspace::init_kernel_page_table(meta_pages);
|
||||||
|
|
||||||
// SAFETY: This function is called only once and only on the BSP.
|
// SAFETY: This function is called only once and only on the BSP.
|
||||||
unsafe { mm::heap_allocator::init() };
|
unsafe { mm::heap_allocator::init() };
|
||||||
|
|
||||||
|
crate::sync::init();
|
||||||
|
|
||||||
boot::init_after_heap();
|
boot::init_after_heap();
|
||||||
|
|
||||||
mm::frame::allocator::init();
|
|
||||||
mm::kspace::init_kernel_page_table(mm::init_page_meta());
|
|
||||||
mm::dma::init();
|
mm::dma::init();
|
||||||
|
|
||||||
arch::init_on_bsp();
|
unsafe { arch::late_init_on_bsp() };
|
||||||
|
|
||||||
smp::init();
|
smp::init();
|
||||||
|
|
||||||
|
@ -442,7 +442,12 @@ impl_frame_meta_for!(MetaPageMeta);
|
|||||||
/// Initializes the metadata of all physical frames.
|
/// Initializes the metadata of all physical frames.
|
||||||
///
|
///
|
||||||
/// The function returns a list of `Frame`s containing the metadata.
|
/// The function returns a list of `Frame`s containing the metadata.
|
||||||
pub(crate) fn init() -> Segment<MetaPageMeta> {
|
///
|
||||||
|
/// # Safety
|
||||||
|
///
|
||||||
|
/// This function should be called only once and only on the BSP,
|
||||||
|
/// before any APs are started.
|
||||||
|
pub(crate) unsafe fn init() -> Segment<MetaPageMeta> {
|
||||||
let max_paddr = {
|
let max_paddr = {
|
||||||
let regions = &crate::boot::EARLY_INFO.get().unwrap().memory_regions;
|
let regions = &crate::boot::EARLY_INFO.get().unwrap().memory_regions;
|
||||||
regions.iter().map(|r| r.base() + r.len()).max().unwrap()
|
regions.iter().map(|r| r.base() + r.len()).max().unwrap()
|
||||||
@ -518,6 +523,14 @@ fn alloc_meta_frames(tot_nr_frames: usize) -> (usize, Paddr) {
|
|||||||
(nr_meta_pages, start_paddr)
|
(nr_meta_pages, start_paddr)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns whether the global frame allocator is initialized.
|
||||||
|
pub(in crate::mm) fn is_initialized() -> bool {
|
||||||
|
// `init` sets it somewhere in the middle. But due to the safety
|
||||||
|
// requirement of the `init` function, we can assume that there
|
||||||
|
// is no race condition.
|
||||||
|
super::MAX_PADDR.load(Ordering::Relaxed) != 0
|
||||||
|
}
|
||||||
|
|
||||||
/// Adds a temporary linear mapping for the metadata frames.
|
/// Adds a temporary linear mapping for the metadata frames.
|
||||||
///
|
///
|
||||||
/// We only assume boot page table to contain 4G linear mapping. Thus if the
|
/// We only assume boot page table to contain 4G linear mapping. Thus if the
|
||||||
|
@ -39,8 +39,7 @@ pub use self::{
|
|||||||
vm_space::VmSpace,
|
vm_space::VmSpace,
|
||||||
};
|
};
|
||||||
pub(crate) use self::{
|
pub(crate) use self::{
|
||||||
frame::meta::init as init_page_meta, kspace::paddr_to_vaddr, page_prop::PrivilegedPageFlags,
|
kspace::paddr_to_vaddr, page_prop::PrivilegedPageFlags, page_table::PageTable,
|
||||||
page_table::PageTable,
|
|
||||||
};
|
};
|
||||||
use crate::arch::mm::PagingConsts;
|
use crate::arch::mm::PagingConsts;
|
||||||
|
|
||||||
|
@ -18,7 +18,6 @@ use core::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
use kernel_stack::KernelStack;
|
use kernel_stack::KernelStack;
|
||||||
pub(crate) use preempt::cpu_local::reset_preempt_info;
|
|
||||||
use processor::current_task;
|
use processor::current_task;
|
||||||
use utils::ForceSync;
|
use utils::ForceSync;
|
||||||
|
|
||||||
|
@ -57,19 +57,5 @@ cpu_local_cell! {
|
|||||||
static PREEMPT_INFO: u32 = NEED_PREEMPT_MASK;
|
static PREEMPT_INFO: u32 = NEED_PREEMPT_MASK;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Resets the preempt info to the initial state.
|
|
||||||
///
|
|
||||||
/// # Safety
|
|
||||||
///
|
|
||||||
/// This function is only useful for the initialization of application
|
|
||||||
/// processors' CPU-local storage. Because that the BSP should access the CPU-
|
|
||||||
/// local storage (`PREEMPT_INFO`) (when doing heap allocation) before we can
|
|
||||||
/// initialize the CPU-local storage for APs, the value of the AP's
|
|
||||||
/// `PREEMPT_INFO` would be that of the BSP's. Therefore, we need to reset the
|
|
||||||
/// `PREEMPT_INFO` to the initial state on APs' initialization.
|
|
||||||
pub(crate) unsafe fn reset_preempt_info() {
|
|
||||||
PREEMPT_INFO.store(NEED_PREEMPT_MASK);
|
|
||||||
}
|
|
||||||
|
|
||||||
const NEED_PREEMPT_MASK: u32 = 1 << 31;
|
const NEED_PREEMPT_MASK: u32 = 1 << 31;
|
||||||
const GUARD_COUNT_MASK: u32 = (1 << 31) - 1;
|
const GUARD_COUNT_MASK: u32 = (1 << 31) - 1;
|
||||||
|
Loading…
x
Reference in New Issue
Block a user