mirror of
https://github.com/DragonOS-Community/DragonOS.git
synced 2025-06-09 11:16:47 +00:00
* feat(kprobe): Add basic kprobe support for x86_64 * feat: add ebpf support (#912) - 实现bpf()一部分命令,包括几种基本map,相关的helper函数 - 实现部分perf相关的数据结构 - 暂时为文件实现简单mmap - 实现一个使用kprobe统计syscall 调用次数的ebpf程序 对eBPF支持程度(基本): - 简单的eBPF程序(没有指定特殊的Map) - 使用内核已经实现的Map的eBPF程序 - 可以和kprobe配合使用 - 内核Map相关的接口定义已经实现,添加新的Map较为简单 不支持的功能: - 区分不同的eBPF程序类型(Network/Cgroup)并限定可调用的helper函数集 - 与内核其它跟踪机制配合(tracepoint) - 其它helper和Map todo - [ ] 修改mmap,需要讨论,因为这个和块缓存层相关 - [x] 添加文档 - [x] 修复可能的错误 - [x] 增加rbpf版本信息 * feat: add /sys/devices/system/cpu/possible file * feat: add /sys/devices/system/cpu/online
720 lines
24 KiB
Rust
720 lines
24 KiB
Rust
use core::{
|
||
alloc::Layout,
|
||
cmp::{max, min},
|
||
intrinsics::unlikely,
|
||
panic,
|
||
};
|
||
|
||
use alloc::sync::Arc;
|
||
|
||
use crate::{
|
||
arch::{mm::PageMapper, MMArch},
|
||
libs::align::align_down,
|
||
mm::{
|
||
page::{page_manager_lock_irqsave, EntryFlags},
|
||
ucontext::LockedVMA,
|
||
VirtAddr, VmFaultReason, VmFlags,
|
||
},
|
||
process::{ProcessManager, ProcessState},
|
||
};
|
||
|
||
use crate::mm::MemoryManagementArch;
|
||
|
||
use super::{
|
||
allocator::page_frame::FrameAllocator,
|
||
page::{page_reclaimer_lock_irqsave, Page, PageFlags},
|
||
};
|
||
|
||
bitflags! {
|
||
pub struct FaultFlags: u64{
|
||
const FAULT_FLAG_WRITE = 1 << 0;
|
||
const FAULT_FLAG_MKWRITE = 1 << 1;
|
||
const FAULT_FLAG_ALLOW_RETRY = 1 << 2;
|
||
const FAULT_FLAG_RETRY_NOWAIT = 1 << 3;
|
||
const FAULT_FLAG_KILLABLE = 1 << 4;
|
||
const FAULT_FLAG_TRIED = 1 << 5;
|
||
const FAULT_FLAG_USER = 1 << 6;
|
||
const FAULT_FLAG_REMOTE = 1 << 7;
|
||
const FAULT_FLAG_INSTRUCTION = 1 << 8;
|
||
const FAULT_FLAG_INTERRUPTIBLE =1 << 9;
|
||
const FAULT_FLAG_UNSHARE = 1 << 10;
|
||
const FAULT_FLAG_ORIG_PTE_VALID = 1 << 11;
|
||
const FAULT_FLAG_VMA_LOCK = 1 << 12;
|
||
}
|
||
}
|
||
|
||
/// # 缺页异常信息结构体
|
||
/// 包含了页面错误处理的相关信息,例如出错的地址、VMA等
|
||
#[derive(Debug)]
|
||
pub struct PageFaultMessage<'a> {
|
||
/// 产生缺页的VMA结构体
|
||
vma: Arc<LockedVMA>,
|
||
/// 缺页地址
|
||
address: VirtAddr,
|
||
/// 异常处理标志
|
||
flags: FaultFlags,
|
||
/// 页表映射器
|
||
mapper: &'a mut PageMapper,
|
||
/// 缺页的文件页在文件中的偏移量
|
||
file_pgoff: Option<usize>,
|
||
/// 缺页对应PageCache中的文件页
|
||
page: Option<Arc<Page>>,
|
||
/// 写时拷贝需要的页面
|
||
cow_page: Option<Arc<Page>>,
|
||
}
|
||
|
||
impl<'a> PageFaultMessage<'a> {
|
||
pub fn new(
|
||
vma: Arc<LockedVMA>,
|
||
address: VirtAddr,
|
||
flags: FaultFlags,
|
||
mapper: &'a mut PageMapper,
|
||
) -> Self {
|
||
let guard = vma.lock_irqsave();
|
||
let file_pgoff = guard.file_page_offset().map(|file_page_offset| {
|
||
((address - guard.region().start()) >> MMArch::PAGE_SHIFT) + file_page_offset
|
||
});
|
||
Self {
|
||
vma: vma.clone(),
|
||
address: VirtAddr::new(crate::libs::align::page_align_down(address.data())),
|
||
flags,
|
||
file_pgoff,
|
||
page: None,
|
||
mapper,
|
||
cow_page: None,
|
||
}
|
||
}
|
||
|
||
#[inline(always)]
|
||
#[allow(dead_code)]
|
||
pub fn vma(&self) -> Arc<LockedVMA> {
|
||
self.vma.clone()
|
||
}
|
||
|
||
#[inline(always)]
|
||
#[allow(dead_code)]
|
||
pub fn address(&self) -> VirtAddr {
|
||
self.address
|
||
}
|
||
|
||
#[inline(always)]
|
||
#[allow(dead_code)]
|
||
pub fn address_aligned_down(&self) -> VirtAddr {
|
||
VirtAddr::new(crate::libs::align::page_align_down(self.address.data()))
|
||
}
|
||
|
||
#[inline(always)]
|
||
#[allow(dead_code)]
|
||
pub fn flags(&self) -> FaultFlags {
|
||
self.flags
|
||
}
|
||
}
|
||
|
||
/// 缺页中断处理结构体
|
||
pub struct PageFaultHandler;
|
||
|
||
impl PageFaultHandler {
|
||
/// 处理缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn handle_mm_fault(mut pfm: PageFaultMessage) -> VmFaultReason {
|
||
let flags = pfm.flags();
|
||
let vma = pfm.vma();
|
||
let current_pcb = ProcessManager::current_pcb();
|
||
let mut guard = current_pcb.sched_info().inner_lock_write_irqsave();
|
||
guard.set_state(ProcessState::Runnable);
|
||
|
||
if !MMArch::vma_access_permitted(
|
||
vma.clone(),
|
||
flags.contains(FaultFlags::FAULT_FLAG_WRITE),
|
||
flags.contains(FaultFlags::FAULT_FLAG_INSTRUCTION),
|
||
flags.contains(FaultFlags::FAULT_FLAG_REMOTE),
|
||
) {
|
||
return VmFaultReason::VM_FAULT_SIGSEGV;
|
||
}
|
||
|
||
let guard = vma.lock_irqsave();
|
||
let vm_flags = *guard.vm_flags();
|
||
drop(guard);
|
||
if unlikely(vm_flags.contains(VmFlags::VM_HUGETLB)) {
|
||
//TODO: 添加handle_hugetlb_fault处理大页缺页异常
|
||
} else {
|
||
Self::handle_normal_fault(&mut pfm);
|
||
}
|
||
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
}
|
||
|
||
/// 处理普通页缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn handle_normal_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let address = pfm.address_aligned_down();
|
||
let vma = pfm.vma.clone();
|
||
let mapper = &mut pfm.mapper;
|
||
if mapper.get_entry(address, 3).is_none() {
|
||
mapper
|
||
.allocate_table(address, 2)
|
||
.expect("failed to allocate PUD table");
|
||
}
|
||
let page_flags = vma.lock_irqsave().flags();
|
||
|
||
for level in 2..=3 {
|
||
let level = MMArch::PAGE_LEVELS - level;
|
||
if mapper.get_entry(address, level).is_none() {
|
||
if vma.is_hugepage() {
|
||
if vma.is_anonymous() {
|
||
mapper.map_huge_page(address, page_flags);
|
||
}
|
||
} else if mapper.allocate_table(address, level - 1).is_none() {
|
||
return VmFaultReason::VM_FAULT_OOM;
|
||
}
|
||
}
|
||
}
|
||
|
||
Self::handle_pte_fault(pfm)
|
||
}
|
||
|
||
/// 处理页表项异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn handle_pte_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let address = pfm.address_aligned_down();
|
||
let flags = pfm.flags;
|
||
let vma = pfm.vma.clone();
|
||
let mut ret = VmFaultReason::VM_FAULT_COMPLETED;
|
||
let mapper = &pfm.mapper;
|
||
|
||
// pte存在
|
||
if let Some(mut entry) = mapper.get_entry(address, 0) {
|
||
if !entry.present() {
|
||
ret = Self::do_swap_page(pfm);
|
||
}
|
||
|
||
if entry.protnone() && vma.is_accessible() {
|
||
ret = Self::do_numa_page(pfm);
|
||
}
|
||
|
||
if flags.intersects(FaultFlags::FAULT_FLAG_WRITE | FaultFlags::FAULT_FLAG_UNSHARE) {
|
||
if !entry.write() {
|
||
ret = Self::do_wp_page(pfm);
|
||
} else {
|
||
entry.set_flags(EntryFlags::from_data(MMArch::ENTRY_FLAG_DIRTY));
|
||
}
|
||
}
|
||
} else if vma.is_anonymous() {
|
||
ret = Self::do_anonymous_page(pfm);
|
||
} else {
|
||
ret = Self::do_fault(pfm);
|
||
}
|
||
|
||
vma.lock_irqsave().set_mapped(true);
|
||
|
||
return ret;
|
||
}
|
||
|
||
/// 处理匿名映射页缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_anonymous_page(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let address = pfm.address_aligned_down();
|
||
let vma = pfm.vma.clone();
|
||
let guard = vma.lock_irqsave();
|
||
let mapper = &mut pfm.mapper;
|
||
|
||
if let Some(flush) = mapper.map(address, guard.flags()) {
|
||
flush.flush();
|
||
crate::debug::klog::mm::mm_debug_log(
|
||
klog_types::AllocatorLogType::LazyAlloc(klog_types::AllocLogItem::new(
|
||
Layout::from_size_align(MMArch::PAGE_SIZE, MMArch::PAGE_SIZE).unwrap(),
|
||
Some(address.data()),
|
||
Some(mapper.translate(address).unwrap().0.data()),
|
||
)),
|
||
klog_types::LogSource::Buddy,
|
||
);
|
||
let paddr = mapper.translate(address).unwrap().0;
|
||
let mut page_manager_guard = page_manager_lock_irqsave();
|
||
let page = page_manager_guard.get_unwrap(&paddr);
|
||
page.write_irqsave().insert_vma(vma.clone());
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
} else {
|
||
VmFaultReason::VM_FAULT_OOM
|
||
}
|
||
}
|
||
|
||
/// 处理文件映射页的缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
if !pfm.flags().contains(FaultFlags::FAULT_FLAG_WRITE) {
|
||
Self::do_read_fault(pfm)
|
||
} else if !pfm
|
||
.vma()
|
||
.lock_irqsave()
|
||
.vm_flags()
|
||
.contains(VmFlags::VM_SHARED)
|
||
{
|
||
Self::do_cow_fault(pfm)
|
||
} else {
|
||
Self::do_shared_fault(pfm)
|
||
}
|
||
}
|
||
|
||
/// 处理私有文件映射的写时复制
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_cow_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let mut ret = Self::filemap_fault(pfm);
|
||
|
||
if unlikely(ret.intersects(
|
||
VmFaultReason::VM_FAULT_ERROR
|
||
| VmFaultReason::VM_FAULT_NOPAGE
|
||
| VmFaultReason::VM_FAULT_RETRY
|
||
| VmFaultReason::VM_FAULT_DONE_COW,
|
||
)) {
|
||
return ret;
|
||
}
|
||
|
||
let cache_page = pfm.page.clone().unwrap();
|
||
let mapper = &mut pfm.mapper;
|
||
|
||
let cow_page_phys = mapper.allocator_mut().allocate_one();
|
||
if cow_page_phys.is_none() {
|
||
return VmFaultReason::VM_FAULT_OOM;
|
||
}
|
||
let cow_page_phys = cow_page_phys.unwrap();
|
||
|
||
let cow_page = Arc::new(Page::new(false, cow_page_phys));
|
||
pfm.cow_page = Some(cow_page.clone());
|
||
|
||
//复制PageCache内容到新的页内
|
||
let new_frame = MMArch::phys_2_virt(cow_page_phys).unwrap();
|
||
(new_frame.data() as *mut u8).copy_from_nonoverlapping(
|
||
MMArch::phys_2_virt(cache_page.read_irqsave().phys_address())
|
||
.unwrap()
|
||
.data() as *mut u8,
|
||
MMArch::PAGE_SIZE,
|
||
);
|
||
|
||
let mut page_manager_guard = page_manager_lock_irqsave();
|
||
|
||
// 新页加入页管理器中
|
||
page_manager_guard.insert(cow_page_phys, &cow_page);
|
||
cow_page.write_irqsave().set_page_cache_index(
|
||
cache_page.read_irqsave().page_cache(),
|
||
cache_page.read_irqsave().index(),
|
||
);
|
||
|
||
ret = ret.union(Self::finish_fault(pfm));
|
||
|
||
ret
|
||
}
|
||
|
||
/// 处理文件映射页的缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_read_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let fs = pfm.vma().lock_irqsave().vm_file().unwrap().inode().fs();
|
||
|
||
let mut ret = Self::do_fault_around(pfm);
|
||
if !ret.is_empty() {
|
||
return ret;
|
||
}
|
||
|
||
ret = fs.fault(pfm);
|
||
|
||
ret = ret.union(Self::finish_fault(pfm));
|
||
|
||
ret
|
||
}
|
||
|
||
/// 处理对共享文件映射区写入引起的缺页
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_shared_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let mut ret = Self::filemap_fault(pfm);
|
||
|
||
let cache_page = pfm.page.clone().expect("no cache_page in PageFaultMessage");
|
||
|
||
// 将pagecache页设为脏页,以便回收时能够回写
|
||
cache_page.write_irqsave().add_flags(PageFlags::PG_DIRTY);
|
||
ret = ret.union(Self::finish_fault(pfm));
|
||
|
||
ret
|
||
}
|
||
|
||
/// 处理被置换页面的缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
#[allow(unused_variables)]
|
||
pub unsafe fn do_swap_page(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
panic!(
|
||
"do_swap_page has not yet been implemented,
|
||
fault message: {:?},
|
||
pid: {}\n",
|
||
pfm,
|
||
crate::process::ProcessManager::current_pid().data()
|
||
);
|
||
// TODO https://code.dragonos.org.cn/xref/linux-6.6.21/mm/memory.c#do_swap_page
|
||
}
|
||
|
||
/// 处理NUMA的缺页异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
#[allow(unused_variables)]
|
||
pub unsafe fn do_numa_page(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
panic!(
|
||
"do_numa_page has not yet been implemented,
|
||
fault message: {:?},
|
||
pid: {}\n",
|
||
pfm,
|
||
crate::process::ProcessManager::current_pid().data()
|
||
);
|
||
// TODO https://code.dragonos.org.cn/xref/linux-6.6.21/mm/memory.c#do_numa_page
|
||
}
|
||
|
||
/// 处理写保护页面的写保护异常
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_wp_page(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let address = pfm.address_aligned_down();
|
||
let vma = pfm.vma.clone();
|
||
let mapper = &mut pfm.mapper;
|
||
|
||
let old_paddr = mapper.translate(address).unwrap().0;
|
||
let mut page_manager = page_manager_lock_irqsave();
|
||
let old_page = page_manager.get_unwrap(&old_paddr);
|
||
let map_count = old_page.read_irqsave().map_count();
|
||
drop(page_manager);
|
||
|
||
let mut entry = mapper.get_entry(address, 0).unwrap();
|
||
let new_flags = entry.flags().set_write(true).set_dirty(true);
|
||
|
||
if vma.lock().vm_flags().contains(VmFlags::VM_SHARED) {
|
||
// 共享映射,直接修改页表项保护位,标记为脏页
|
||
let table = mapper.get_table(address, 0).unwrap();
|
||
let i = table.index_of(address).unwrap();
|
||
entry.set_flags(new_flags);
|
||
table.set_entry(i, entry);
|
||
|
||
old_page.write_irqsave().add_flags(PageFlags::PG_DIRTY);
|
||
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
} else if vma.is_anonymous() {
|
||
// 私有匿名映射,根据引用计数判断是否拷贝页面
|
||
if map_count == 1 {
|
||
let table = mapper.get_table(address, 0).unwrap();
|
||
let i = table.index_of(address).unwrap();
|
||
entry.set_flags(new_flags);
|
||
table.set_entry(i, entry);
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
} else if let Some(flush) = mapper.map(address, new_flags) {
|
||
let mut page_manager_guard = page_manager_lock_irqsave();
|
||
let old_page = page_manager_guard.get_unwrap(&old_paddr);
|
||
old_page.write_irqsave().remove_vma(&vma);
|
||
// drop(page_manager_guard);
|
||
|
||
flush.flush();
|
||
let paddr = mapper.translate(address).unwrap().0;
|
||
// let mut page_manager_guard = page_manager_lock_irqsave();
|
||
let page = page_manager_guard.get_unwrap(&paddr);
|
||
page.write_irqsave().insert_vma(vma.clone());
|
||
|
||
(MMArch::phys_2_virt(paddr).unwrap().data() as *mut u8).copy_from_nonoverlapping(
|
||
MMArch::phys_2_virt(old_paddr).unwrap().data() as *mut u8,
|
||
MMArch::PAGE_SIZE,
|
||
);
|
||
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
} else {
|
||
VmFaultReason::VM_FAULT_OOM
|
||
}
|
||
} else {
|
||
// 私有文件映射,必须拷贝页面
|
||
if let Some(flush) = mapper.map(address, new_flags) {
|
||
let mut page_manager_guard = page_manager_lock_irqsave();
|
||
let old_page = page_manager_guard.get_unwrap(&old_paddr);
|
||
old_page.write_irqsave().remove_vma(&vma);
|
||
// drop(page_manager_guard);
|
||
|
||
flush.flush();
|
||
let paddr = mapper.translate(address).unwrap().0;
|
||
// let mut page_manager_guard = page_manager_lock_irqsave();
|
||
let page = page_manager_guard.get_unwrap(&paddr);
|
||
page.write_irqsave().insert_vma(vma.clone());
|
||
|
||
(MMArch::phys_2_virt(paddr).unwrap().data() as *mut u8).copy_from_nonoverlapping(
|
||
MMArch::phys_2_virt(old_paddr).unwrap().data() as *mut u8,
|
||
MMArch::PAGE_SIZE,
|
||
);
|
||
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
} else {
|
||
VmFaultReason::VM_FAULT_OOM
|
||
}
|
||
}
|
||
}
|
||
|
||
/// 缺页附近页预读
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn do_fault_around(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let vma = pfm.vma();
|
||
let address = pfm.address();
|
||
let mapper = &mut pfm.mapper;
|
||
|
||
if mapper.get_table(address, 0).is_none() {
|
||
mapper
|
||
.allocate_table(address, 0)
|
||
.expect("failed to allocate pte table");
|
||
}
|
||
let vma_guard = vma.lock_irqsave();
|
||
let vma_region = *vma_guard.region();
|
||
drop(vma_guard);
|
||
|
||
// 缺页在VMA中的偏移量
|
||
let vm_pgoff = (address - vma_region.start()) >> MMArch::PAGE_SHIFT;
|
||
|
||
// 缺页在PTE中的偏移量
|
||
let pte_pgoff = (address.data() >> MMArch::PAGE_SHIFT) & (1 << MMArch::PAGE_ENTRY_SHIFT);
|
||
|
||
// 缺页在文件中的偏移量
|
||
let file_pgoff = pfm.file_pgoff.expect("no file_pgoff");
|
||
|
||
let vma_pages_count = (vma_region.end() - vma_region.start()) >> MMArch::PAGE_SHIFT;
|
||
|
||
let fault_around_page_number = 16;
|
||
|
||
// 开始位置不能超出当前pte和vma头部
|
||
let from_pte = max(
|
||
align_down(pte_pgoff, fault_around_page_number),
|
||
pte_pgoff - min(vm_pgoff, pte_pgoff),
|
||
);
|
||
|
||
// pte结束位置不能超过:
|
||
// 1.最大预读上限(默认16)
|
||
// 2.最大pte(512)
|
||
// 3.vma结束位置(pte_pgoff + (vma_pages_count - vm_pgoff)计算出vma结束页号对当前pte开头的偏移)
|
||
let to_pte = min(
|
||
from_pte + fault_around_page_number,
|
||
min(
|
||
1 << MMArch::PAGE_SHIFT,
|
||
pte_pgoff + (vma_pages_count - vm_pgoff),
|
||
),
|
||
);
|
||
|
||
// 预先分配pte页表(如果不存在)
|
||
if mapper.get_table(address, 0).is_none() && mapper.allocate_table(address, 0).is_none() {
|
||
return VmFaultReason::VM_FAULT_OOM;
|
||
}
|
||
|
||
let fs = pfm.vma().lock_irqsave().vm_file().unwrap().inode().fs();
|
||
// from_pte - pte_pgoff得出预读起始pte相对缺失页的偏移,加上pfm.file_pgoff(缺失页在文件中的偏移)得出起始页在文件中的偏移,结束pte同理
|
||
fs.map_pages(
|
||
pfm,
|
||
file_pgoff + (from_pte - pte_pgoff),
|
||
file_pgoff + (to_pte - pte_pgoff),
|
||
);
|
||
|
||
VmFaultReason::empty()
|
||
}
|
||
|
||
/// 通用的VMA文件映射页面映射函数,将PageCache中的页面映射到进程空间
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn filemap_map_pages(
|
||
pfm: &mut PageFaultMessage,
|
||
|
||
start_pgoff: usize,
|
||
end_pgoff: usize,
|
||
) -> VmFaultReason {
|
||
let vma = pfm.vma();
|
||
let vma_guard = vma.lock_irqsave();
|
||
let file = vma_guard.vm_file().expect("no vm_file in vma");
|
||
let page_cache = file.inode().page_cache().unwrap();
|
||
let mapper = &mut pfm.mapper;
|
||
|
||
// 起始页地址
|
||
let addr = vma_guard.region().start
|
||
+ ((start_pgoff
|
||
- vma_guard
|
||
.file_page_offset()
|
||
.expect("file_page_offset is none"))
|
||
<< MMArch::PAGE_SHIFT);
|
||
|
||
for pgoff in start_pgoff..=end_pgoff {
|
||
if let Some(page) = page_cache.get_page(pgoff) {
|
||
let page_guard = page.read_irqsave();
|
||
if page_guard.flags().contains(PageFlags::PG_UPTODATE) {
|
||
let phys = page_guard.phys_address();
|
||
|
||
let address =
|
||
VirtAddr::new(addr.data() + ((pgoff - start_pgoff) << MMArch::PAGE_SHIFT));
|
||
mapper
|
||
.map_phys(address, phys, vma_guard.flags())
|
||
.unwrap()
|
||
.flush();
|
||
}
|
||
}
|
||
}
|
||
VmFaultReason::empty()
|
||
}
|
||
|
||
/// 通用的VMA文件映射错误处理函数
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn filemap_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let vma = pfm.vma();
|
||
let vma_guard = vma.lock_irqsave();
|
||
let file = vma_guard.vm_file().expect("no vm_file in vma");
|
||
let page_cache = file.inode().page_cache().unwrap();
|
||
let file_pgoff = pfm.file_pgoff.expect("no file_pgoff");
|
||
let mapper = &mut pfm.mapper;
|
||
let mut ret = VmFaultReason::empty();
|
||
|
||
if let Some(page) = page_cache.get_page(file_pgoff) {
|
||
// TODO 异步从磁盘中预读页面进PageCache
|
||
|
||
// 直接将PageCache中的页面作为要映射的页面
|
||
pfm.page = Some(page.clone());
|
||
} else {
|
||
// TODO 同步预读
|
||
//涉及磁盘IO,返回标志为VM_FAULT_MAJOR
|
||
ret = VmFaultReason::VM_FAULT_MAJOR;
|
||
// let mut buf: Vec<u8> = vec![0; MMArch::PAGE_SIZE];
|
||
|
||
let allocator = mapper.allocator_mut();
|
||
|
||
// 分配一个物理页面作为加入PageCache的新页
|
||
let new_cache_page = allocator.allocate_one().unwrap();
|
||
// (MMArch::phys_2_virt(new_cache_page).unwrap().data() as *mut u8)
|
||
// .copy_from_nonoverlapping(buf.as_mut_ptr(), MMArch::PAGE_SIZE);
|
||
file.pread(
|
||
file_pgoff * MMArch::PAGE_SIZE,
|
||
MMArch::PAGE_SIZE,
|
||
core::slice::from_raw_parts_mut(
|
||
MMArch::phys_2_virt(new_cache_page).unwrap().data() as *mut u8,
|
||
MMArch::PAGE_SIZE,
|
||
),
|
||
)
|
||
.expect("failed to read file to create pagecache page");
|
||
|
||
let page = Arc::new(Page::new(true, new_cache_page));
|
||
pfm.page = Some(page.clone());
|
||
|
||
page.write_irqsave().add_flags(PageFlags::PG_LRU);
|
||
page_manager_lock_irqsave().insert(new_cache_page, &page);
|
||
page_reclaimer_lock_irqsave().insert_page(new_cache_page, &page);
|
||
page_cache.add_page(file_pgoff, &page);
|
||
|
||
page.write_irqsave()
|
||
.set_page_cache_index(Some(page_cache), Some(file_pgoff));
|
||
}
|
||
ret
|
||
}
|
||
|
||
/// 将文件页映射到缺页地址
|
||
/// ## 参数
|
||
///
|
||
/// - `pfm`: 缺页异常信息
|
||
/// - `mapper`: 页表映射器
|
||
///
|
||
/// ## 返回值
|
||
/// - VmFaultReason: 页面错误处理信息标志
|
||
pub unsafe fn finish_fault(pfm: &mut PageFaultMessage) -> VmFaultReason {
|
||
let vma = pfm.vma();
|
||
let vma_guard = vma.lock_irqsave();
|
||
let flags = pfm.flags();
|
||
let cache_page = pfm.page.clone();
|
||
let cow_page = pfm.cow_page.clone();
|
||
let address = pfm.address();
|
||
let mapper = &mut pfm.mapper;
|
||
|
||
let page_to_map = if flags.contains(FaultFlags::FAULT_FLAG_WRITE)
|
||
&& !vma_guard.vm_flags().contains(VmFlags::VM_SHARED)
|
||
{
|
||
// 私有文件映射的写时复制
|
||
cow_page.expect("no cow_page in PageFaultMessage")
|
||
} else {
|
||
// 直接映射到PageCache
|
||
cache_page.expect("no cache_page in PageFaultMessage")
|
||
};
|
||
|
||
let page_phys = page_to_map.read_irqsave().phys_address();
|
||
|
||
mapper.map_phys(address, page_phys, vma_guard.flags());
|
||
page_to_map.write_irqsave().insert_vma(pfm.vma());
|
||
VmFaultReason::VM_FAULT_COMPLETED
|
||
}
|
||
}
|