DragonOS-Community/DragonOS
1
1
Fork 0
mirror of https://github.com/DragonOS-Community/DragonOS.git synced 2025-06-09 07:06:47 +00:00
Code Issues Releases Wiki Activity

new: 为ipi添加xapic支持

Browse Source
This commit is contained in:
fslongjin 2022-09-01 17:40:11 +08:00
parent 588603a10a
commit dffa51b1ef
7 changed files with 140 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
kernel/arch/x86_64/x86_64_ipi.c
View File

@ -1,7 +1,8 @@
#include "x86_64_ipi.h"
#include <driver/interrupt/apic/apic.h>
void ipi_send_IPI(uint32_t dest_mode, uint32_t deliver_status, uint32_t level, uint32_t trigger,
uint32_t vector, uint32_t deliver_mode, uint32_t dest_shorthand, bool apic_type, uint32_t destination)
uint32_t vector, uint32_t deliver_mode, uint32_t dest_shorthand, uint32_t destination)
{
struct INT_CMD_REG icr_entry;
icr_entry.dest_mode = dest_mode;
@ -18,7 +19,7 @@ void ipi_send_IPI(uint32_t dest_mode, uint32_t deliver_status, uint32_t level, u
// x2APIC下，ICR寄存器地址为0x830
// xAPIC下则为0xfee00300(31-0) 0xfee00310 (63-32)
if (apic_type) // x2APIC
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED) // x2APIC
{
icr_entry.destination.x2apic_destination = destination;
wrmsr(0x830, *(unsigned long *)&icr_entry); // 发送ipi
@ -29,17 +30,17 @@ void ipi_send_IPI(uint32_t dest_mode, uint32_t deliver_status, uint32_t level, u
icr_entry.destination.apic_destination.dest_field = destination & 0xff;
icr_entry.destination.apic_destination.res_4 = 0;
// 先向高32bit写数据，然后再向低32bit写数据，不能调转
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + 0x310) = (uint32_t)(((*(ul *)&icr_entry) >> 32) & 0xffff);
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + 0x300) = (uint32_t)((*(ul *)&icr_entry) & 0xffff);
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + 0x310) = (uint32_t)(((*(ul *)&icr_entry) >> 32) & 0xffffffff);
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + 0x300) = (uint32_t)((*(ul *)&icr_entry) & 0xffffffff);
}
}
int ipi_regiserIPI(uint64_t irq_num, void *arg,
void (*handler)(uint64_t irq_num, uint64_t param, struct pt_regs *regs),
uint64_t param, hardware_intr_controller *controller, char *irq_name)
uint64_t param, hardware_intr_controller *controller, char *irq_name)
{
irq_desc_t *p = &SMP_IPI_desc[irq_num-200];
p->controller = NULL; // 由于ipi不涉及到具体的硬件操作，因此不需要controller
irq_desc_t *p = &SMP_IPI_desc[irq_num - 200];
p->controller = NULL; // 由于ipi不涉及到具体的硬件操作，因此不需要controller
p->irq_name = irq_name;
p->parameter = param;
p->flags = 0;

3
kernel/arch/x86_64/x86_64_ipi.h
View File

@ -24,11 +24,10 @@
* @param vector 中断向量
* @param deliver_mode 投递模式
* @param dest_shorthand 投递目标速记值
* @param apic_type apic的类型 （0:xapic 1: x2apic）
* @param destination 投递目标
*/
void ipi_send_IPI(uint32_t dest_mode, uint32_t deliver_status, uint32_t level, uint32_t trigger,
uint32_t vector, uint32_t deliver_mode, uint32_t dest_shorthand, bool apic_type, uint32_t destination);
uint32_t vector, uint32_t deliver_mode, uint32_t dest_shorthand, uint32_t destination);
/**
* @brief ipi中断处理注册函数

209
kernel/driver/interrupt/apic/apic.c
View File

@ -15,23 +15,36 @@
// 导出定义在irq.c中的中段门表
extern void (*interrupt_table[24])(void);
bool flag_support_apic = false;
bool flag_support_x2apic = false;
uint local_apic_version;
uint local_apic_max_LVT_entries;
static bool flag_support_apic = false;
static bool flag_support_x2apic = false;
uint8_t __apic_enable_state = APIC_XAPIC_ENABLED;
static uint local_apic_version;
static uint local_apic_max_LVT_entries;
static struct acpi_Multiple_APIC_Description_Table_t *madt;
static struct acpi_IO_APIC_Structure_t *io_apic_ICS;
#define send_EOI() \
do \
{ \
__asm__ __volatile__("movq $0x00, %%rdx \n\t" \
"movq $0x00, %%rax \n\t" \
"movq $0x80b, %%rcx \n\t" \
"wrmsr \n\t" :: \
: "memory"); \
} while (0)
static void __local_apic_xapic_init();
static void __local_apic_x2apic_init();
static __always_inline void __send_eoi()
{
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
{
__asm__ __volatile__("movq $0x00, %%rdx \n\t"
"movq $0x00, %%rax \n\t"
"movq $0x80b, %%rcx \n\t"
"wrmsr \n\t" ::
: "memory");
}
else
{
io_mfence();
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_EOI) = 0;
io_mfence();
}
}
/**
* @brief 初始化io_apic
@ -110,64 +123,26 @@ void apic_init_ap_core_local_apic()
uint64_t ia32_apic_base = rdmsr(0x1b);
ia32_apic_base |= (1 << 11);
if (flag_support_x2apic) // 如果支持x2apic，则启用
{
ia32_apic_base |= (1 << 10);
wrmsr(0x1b, ia32_apic_base);
wrmsr(0x1b, ia32_apic_base);
}
ia32_apic_base = rdmsr(0x1b);
eax = ia32_apic_base & 0xffffffff;
// 检测是否成功启用xAPIC和x2APIC
if (eax & 0xc00)
if ((eax & 0xc00) == 0xc00)
kinfo("xAPIC & x2APIC enabled!");
else if ((eax & 0x800) == 0x800)
kinfo("Only xAPIC enabled!");
else
kerror("Both xAPIC and x2APIC are not enabled.");
// 设置SVR寄存器，开启local APIC、禁止EOI广播
// enable SVR[8]
__asm__ __volatile__("movq $0x80f, %%rcx \n\t"
"rdmsr \n\t"
"bts $8, %%rax \n\t"
// "bts $12, %%rax\n\t"
"wrmsr \n\t"
"movq $0x80f, %%rcx \n\t"
"rdmsr \n\t"
: "=a"(eax), "=d"(edx)
:
: "memory");
if (eax & 0x100)
printk_color(RED, YELLOW, "SVR[8] enabled\n");
if (edx & 0x1000)
printk_color(RED, YELLOW, "SVR[12] enabled\n");
// get local APIC ID
__asm__ __volatile__("movq $0x802, %%rcx \n\t"
"rdmsr \n\t"
: "=a"(eax), "=d"(edx)
:
: "memory");
printk_color(RED, YELLOW, "x2APIC ID:%#010x\n", eax);
// 由于尚未配置LVT对应的处理程序，因此先屏蔽所有的LVT
// mask all LVT
__asm__ __volatile__( //"movq $0x82f, %%rcx \n\t" //CMCI
//"wrmsr \n\t"
"movq $0x832, %%rcx \n\t" // Timer
"wrmsr \n\t"
"movq $0x833, %%rcx \n\t" // Thermal Monitor
"wrmsr \n\t"
"movq $0x834, %%rcx \n\t" // Performance Counter
"wrmsr \n\t"
"movq $0x835, %%rcx \n\t" // LINT0
"wrmsr \n\t"
"movq $0x836, %%rcx \n\t" // LINT1
"wrmsr \n\t"
"movq $0x837, %%rcx \n\t" // Error
"wrmsr \n\t"
:
: "a"(0x10000), "d"(0x00)
: "memory");
kdebug("All LVT Masked");
if (flag_support_x2apic) // 当前为x2APIC
__local_apic_x2apic_init();
else // 当前为xapic
__local_apic_xapic_init();
}
/**
@ -176,9 +151,10 @@ void apic_init_ap_core_local_apic()
*/
static void __local_apic_xapic_init()
{
__apic_enable_state = APIC_XAPIC_ENABLED;
// 设置svr的 apic软件使能位
uint64_t qword = *(uint64_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_SVR);
kdebug("svr=%#018lx", qword);
qword |= (1 << 8);
*(uint64_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_SVR) = qword;
qword = *(uint64_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_SVR);
@ -230,6 +206,7 @@ static void __local_apic_xapic_init()
*/
static void __local_apic_x2apic_init()
{
__apic_enable_state = APIC_X2APIC_ENABLED;
uint32_t eax, edx;
__asm__ __volatile__("movq $0x80f, %%rcx \n\t"
"rdmsr \n\t"
@ -262,23 +239,23 @@ static void __local_apic_x2apic_init()
kdebug("Integrated APIC.");
// 由于尚未配置LVT对应的处理程序，因此先屏蔽所有的LVT
__asm__ __volatile__(// "movq $0x82f, %%rcx \n\t" // CMCI
// "wrmsr \n\t"
"movq $0x832, %%rcx \n\t" // Timer
"wrmsr \n\t"
"movq $0x833, %%rcx \n\t" // Thermal Monitor
"wrmsr \n\t"
"movq $0x834, %%rcx \n\t" // Performance Counter
"wrmsr \n\t"
"movq $0x835, %%rcx \n\t" // LINT0
"wrmsr \n\t"
"movq $0x836, %%rcx \n\t" // LINT1
"wrmsr \n\t"
"movq $0x837, %%rcx \n\t" // Error
"wrmsr \n\t"
:
: "a"(0x10000), "d"(0x00)
: "memory");
__asm__ __volatile__( // "movq $0x82f, %%rcx \n\t" // CMCI
// "wrmsr \n\t"
"movq $0x832, %%rcx \n\t" // Timer
"wrmsr \n\t"
"movq $0x833, %%rcx \n\t" // Thermal Monitor
"wrmsr \n\t"
"movq $0x834, %%rcx \n\t" // Performance Counter
"wrmsr \n\t"
"movq $0x835, %%rcx \n\t" // LINT0
"wrmsr \n\t"
"movq $0x836, %%rcx \n\t" // LINT1
"wrmsr \n\t"
"movq $0x837, %%rcx \n\t" // Error
"wrmsr \n\t"
:
: "a"(0x10000), "d"(0x00)
: "memory");
kdebug("All LVT Masked");
}
@ -329,8 +306,10 @@ void apic_local_apic_init()
ia32_apic_base = rdmsr(0x1b);
ia32_apic_base |= (1 << 11);
if (flag_support_x2apic) // 如果支持x2apic，则启用
{
ia32_apic_base |= (1 << 10);
wrmsr(0x1b, ia32_apic_base);
wrmsr(0x1b, ia32_apic_base);
}
ia32_apic_base = rdmsr(0x1b);
eax = ia32_apic_base & 0xffffffff;
@ -375,21 +354,7 @@ void apic_init()
for (int i = 150; i < 160; ++i)
set_intr_gate(i, 0, local_apic_interrupt_table[i - 150]);
// 初始化主芯片
// io_out8(0x20, 0x11); // 初始化主芯片的icw1
// io_out8(0x21, 0x20); // 设置主芯片的中断向量号为0x20(0x20-0x27)
// io_out8(0x21, 0x04); // 设置int2端口级联从芯片
// io_out8(0x21, 0x01); // 设置为AEOI模式、FNM、无缓冲
// // 初始化从芯片
// io_out8(0xa0, 0x11);
// io_out8(0xa1, 0x28); // 设置从芯片的中断向量号为0x28(0x28-0x2f)
// io_out8(0xa1, 0x02); // 设置从芯片连接到主芯片的int2
// io_out8(0xa1, 0x01);
// 屏蔽类8259A芯片
// todo: 这里有bug：在真机或bochs上，无法屏蔽8259A，以至于sti()后，接收到来自与8259A相连的时钟的中断。然后造成错误。
io_out8(0x21, 0xff);
io_out8(0xa1, 0xff);
@ -445,26 +410,10 @@ void do_IRQ(struct pt_regs *rsp, ul number)
if (irq->controller != NULL && irq->controller->ack != NULL)
irq->controller->ack(number);
else
{
// 向EOI寄存器写入0x00表示结束中断
__asm__ __volatile__("movq $0x00, %%rdx \n\t"
"movq $0x00, %%rax \n\t"
"movq $0x80b, %%rcx \n\t"
"wrmsr \n\t" ::
: "memory");
}
}
else if (number == 0x80) // 系统调用
{
// ps: 当前已经将系统调用直接使用系统调用门实现，不走这里。。
do_syscall_int(rsp, 0);
__send_eoi();
}
else if (number >= 200)
{
// printk_color(RED, BLACK, "SMP IPI [ %d ]\n", number);
apic_local_apic_edge_ack(number);
{
@ -486,16 +435,14 @@ void do_IRQ(struct pt_regs *rsp, ul number)
if (irq->controller != NULL && irq->controller->ack != NULL)
irq->controller->ack(number);
else
{
// 向EOI寄存器写入0x00表示结束中断
send_EOI();
}
__send_eoi(); // 向EOI寄存器写入0x00表示结束中断
}
else
{
kwarn("do IRQ receive: %d", number);
// 忽略未知中断
return;
}
// kdebug("before softirq");
@ -600,17 +547,7 @@ void apic_ioapic_uninstall(ul irq_num)
void apic_ioapic_level_ack(ul irq_num) // 电平触发
{
// 向EOI寄存器写入0x00表示结束中断
/*io_mfence();
uint *eoi = (uint *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_EOI);
*eoi = 0x00;
io_mfence(); */
__asm__ __volatile__("movq $0x00, %%rdx \n\t"
"movq $0x00, %%rax \n\t"
"movq $0x80b, %%rcx \n\t"
"wrmsr \n\t" ::
: "memory");
__send_eoi();
*apic_ioapic_map.virtual_EOI_addr = irq_num;
}
@ -623,11 +560,7 @@ void apic_ioapic_edge_ack(ul irq_num) // 边沿触发
*eoi = 0x00;
*/
__asm__ __volatile__("movq $0x00, %%rdx \n\t"
"movq $0x00, %%rax \n\t"
"movq $0x80b, %%rcx \n\t"
"wrmsr \n\t" ::
: "memory");
__send_eoi();
}
/**
@ -639,11 +572,7 @@ void apic_ioapic_edge_ack(ul irq_num) // 边沿触发
void apic_local_apic_edge_ack(ul irq_num)
{
// 向EOI寄存器写入0x00表示结束中断
__asm__ __volatile__("movq $0x00, %%rdx \n\t"
"movq $0x00, %%rax \n\t"
"movq $0x80b, %%rcx \n\t"
"wrmsr \n\t" ::
: "memory");
__send_eoi();
}
/**

9
kernel/driver/interrupt/apic/apic.h
View File

@ -8,13 +8,20 @@
#pragma GCC push_options
#pragma GCC optimize("O0")
#define APIC_SUCCESS 0
#define APIC_E_NOTFOUND 1
#define APIC_IO_APIC_VIRT_BASE_ADDR SPECIAL_MEMOEY_MAPPING_VIRT_ADDR_BASE + IO_APIC_MAPPING_OFFSET
#define APIC_LOCAL_APIC_VIRT_BASE_ADDR SPECIAL_MEMOEY_MAPPING_VIRT_ADDR_BASE + LOCAL_APIC_MAPPING_OFFSET
// ======== local apic 寄存器地址偏移量表 =======
// 当前apic启用状态标志
extern uint8_t __apic_enable_state;
#define APIC_XAPIC_ENABLED 0
#define APIC_X2APIC_ENABLED 1
#define CURRENT_APIC_STATE (__apic_enable_state )
// ======== local apic 寄存器虚拟地址偏移量表 =======
// 0x00~0x10 Reserved.
#define LOCAL_APIC_OFFSET_Local_APIC_ID 0x20
#define LOCAL_APIC_OFFSET_Local_APIC_Version 0x30

69
kernel/driver/interrupt/apic/apic_timer.h
View File

@ -12,27 +12,32 @@ extern uint64_t apic_timer_ticks_result;
#pragma GCC push_options
#pragma GCC optimize("O0")
/**
* @brief 设置apic定时器的分频计数
*
* @param divider 分频除数
*/
#define apic_timer_set_div(divider) \
do \
{ \
wrmsr(0x83e, divider); \
} while (0)
static __always_inline void apic_timer_set_div(uint64_t divider)
{
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
wrmsr(0x83e, divider);
else
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_CLKDIV) = (uint32_t)divider;
}
/**
* @brief 设置apic定时器的初始计数值
*
* @param init_cnt 初始计数值
*/
#define apic_timer_set_init_cnt(init_cnt) \
do \
{ \
wrmsr(0x838, init_cnt); \
} while (0)
static __always_inline void apic_timer_set_init_cnt(uint32_t init_cnt)
{
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
wrmsr(0x838, init_cnt);
else
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_INITIAL_COUNT_REG) = (uint32_t)init_cnt;
}
/**
* @brief 设置apic定时器的lvt，并启动定时器
@ -41,28 +46,46 @@ extern uint64_t apic_timer_ticks_result;
* @param mask 是否屏蔽（1：屏蔽， 0：不屏蔽）
* @param mode 计时模式
*/
#define apic_timer_set_LVT(vector, mask, mode) \
do \
{ \
wrmsr(0x832, (mode << 17) | vector | (mask ? (APIC_LVT_INT_MASKED) : 0)); \
} while (0)
static __always_inline void apic_timer_set_LVT(uint32_t vector, uint32_t mask, uint32_t mode)
{
register uint32_t val = (mode << 17) | vector | (mask ? (APIC_LVT_INT_MASKED) : 0);
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
wrmsr(0x832, val);
else
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_LVT_TIMER) = (uint32_t)val;
}
#define apic_timer_write_LVT(value) \
do \
{ \
wrmsr(0x832, value); \
} while (0)
static __always_inline void apic_timer_write_LVT(uint32_t value)
{
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
wrmsr(0x832, value);
else
*(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_LVT_TIMER) = (uint32_t)value;
}
/**
* @brief 获取apic定时器的LVT的值
*
*/
#define apic_timer_get_LVT() (rdmsr(0x832))
static __always_inline uint32_t apic_timer_get_LVT()
{
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
return rdmsr(0x832);
else
return *(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_LVT_TIMER);
}
/**
* @brief 获取apic定时器当前计数值
*
*/
#define apic_timer_get_current() (rdmsr(0x839))
static __always_inline uint32_t apic_timer_get_current()
{
if (CURRENT_APIC_STATE == APIC_X2APIC_ENABLED)
return (uint32_t)rdmsr(0x839);
else
return *(uint32_t *)(APIC_LOCAL_APIC_VIRT_BASE_ADDR + LOCAL_APIC_OFFSET_Local_APIC_CURRENT_COUNT_REG);
}
/**
* @brief 停止apic定时器
@ -71,7 +94,7 @@ extern uint64_t apic_timer_ticks_result;
#define apic_timer_stop() \
do \
{ \
uint64_t val = apic_timer_get_LVT(); \
uint32_t val = apic_timer_get_LVT(); \
val |= APIC_LVT_INT_MASKED; \
apic_timer_write_LVT(val); \
} while (0)

2
kernel/smp/ipi.h
View File

@ -21,7 +21,7 @@
* @param destination 投递目标
*/
extern void ipi_send_IPI(uint32_t dest_mode, uint32_t deliver_status, uint32_t level, uint32_t trigger,
uint32_t vector, uint32_t deliver_mode, uint32_t dest_shorthand, bool apic_type, uint32_t destination);
uint32_t vector, uint32_t deliver_mode, uint32_t dest_shorthand, uint32_t destination);
/**
* @brief ipi中断处理注册函数

12
kernel/smp/smp.c
View File

@ -13,7 +13,7 @@
void ipi_0xc8_handler(uint64_t irq_num, uint64_t param, struct pt_regs *regs); // 由BSP转发的HPET中断处理函数
static spinlock_t multi_core_starting_lock; // 多核启动锁
static spinlock_t multi_core_starting_lock={1}; // 多核启动锁
static struct acpi_Processor_Local_APIC_Structure_t *proc_local_apic_structs[MAX_SUPPORTED_PROCESSOR_NUM];
static uint32_t total_processor_num = 0;
@ -43,14 +43,16 @@ void smp_init()
for (int i = 200; i < 210; ++i)
set_intr_gate(i, 0, SMP_interrupt_table[i - 200]);
memset((void *)SMP_IPI_desc, 0, sizeof(irq_desc_t) * SMP_IRQ_NUM);
io_mfence();
// 注册接收bsp处理器的hpet中断转发的处理函数
ipi_regiserIPI(0xc8, NULL, &ipi_0xc8_handler, NULL, NULL, "IPI 0xc8");
io_mfence();
ipi_send_IPI(DEST_PHYSICAL, IDLE, ICR_LEVEL_DE_ASSERT, EDGE_TRIGGER, 0x00, ICR_INIT, ICR_ALL_EXCLUDE_Self, true, 0x00);
ipi_send_IPI(DEST_PHYSICAL, IDLE, ICR_LEVEL_DE_ASSERT, EDGE_TRIGGER, 0x00, ICR_INIT, ICR_ALL_EXCLUDE_Self, 0x00);
kdebug("total_processor_num=%d", total_processor_num);
// total_processor_num = 3;
for (int i = 1; i < total_processor_num; ++i) // i从1开始，不初始化bsp
{
io_mfence();
@ -90,9 +92,9 @@ void smp_init()
cpu_core_info[current_starting_cpu].ist_stack_start, cpu_core_info[current_starting_cpu].ist_stack_start, cpu_core_info[current_starting_cpu].ist_stack_start, cpu_core_info[current_starting_cpu].ist_stack_start, cpu_core_info[current_starting_cpu].ist_stack_start, cpu_core_info[current_starting_cpu].ist_stack_start, cpu_core_info[current_starting_cpu].ist_stack_start);
io_mfence();
// 连续发送两次start-up IPI
ipi_send_IPI(DEST_PHYSICAL, IDLE, ICR_LEVEL_DE_ASSERT, EDGE_TRIGGER, 0x20, ICR_Start_up, ICR_No_Shorthand, true, proc_local_apic_structs[i]->local_apic_id);
ipi_send_IPI(DEST_PHYSICAL, IDLE, ICR_LEVEL_DE_ASSERT, EDGE_TRIGGER, 0x20, ICR_Start_up, ICR_No_Shorthand, proc_local_apic_structs[i]->local_apic_id);
io_mfence();
ipi_send_IPI(DEST_PHYSICAL, IDLE, ICR_LEVEL_DE_ASSERT, EDGE_TRIGGER, 0x20, ICR_Start_up, ICR_No_Shorthand, true, proc_local_apic_structs[i]->local_apic_id);
ipi_send_IPI(DEST_PHYSICAL, IDLE, ICR_LEVEL_DE_ASSERT, EDGE_TRIGGER, 0x20, ICR_Start_up, ICR_No_Shorthand, proc_local_apic_structs[i]->local_apic_id);
}
io_mfence();
while (num_cpu_started != total_processor_num)