From 3c62be3f5ddd721f9e7396174b2e806babc74afa Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 11:14:41 +0200 Subject: [PATCH 01/46] BAN: Implement make_signed and make_unsigned --- BAN/include/BAN/Traits.h | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/BAN/include/BAN/Traits.h b/BAN/include/BAN/Traits.h index c72704dd2f..5902978235 100644 --- a/BAN/include/BAN/Traits.h +++ b/BAN/include/BAN/Traits.h @@ -106,6 +106,36 @@ namespace BAN template inline constexpr bool is_unsigned_v = is_unsigned::value; template concept unsigned_integral = is_unsigned_v && is_integral_v; +#define __BAN_TRAITS_MAKE_UNSIGNED_CV(__type) \ + template<> struct make_unsigned<__type> { using type = unsigned __type; }; \ + template<> struct make_unsigned { using type = unsigned const __type; }; \ + template<> struct make_unsigned { using type = unsigned volatile __type; }; \ + template<> struct make_unsigned { using type = unsigned const volatile __type; }; + + template requires is_arithmetic_v struct make_unsigned { using type = T; }; + __BAN_TRAITS_MAKE_UNSIGNED_CV(char) + __BAN_TRAITS_MAKE_UNSIGNED_CV(short) + __BAN_TRAITS_MAKE_UNSIGNED_CV(int) + __BAN_TRAITS_MAKE_UNSIGNED_CV(long) + __BAN_TRAITS_MAKE_UNSIGNED_CV(long long) + template using make_unsigned_t = typename make_unsigned::type; +#undef __BAN_TRAITS_MAKE_UNSIGNED_CV + +#define __BAN_TRAITS_MAKE_SIGNED_CV(__type) \ + template<> struct make_signed { using type = __type; }; \ + template<> struct make_signed { using type = const __type; }; \ + template<> struct make_signed { using type = volatile __type; }; \ + template<> struct make_signed { using type = const volatile __type; }; + + template requires is_arithmetic_v struct make_signed { using type = T; }; + __BAN_TRAITS_MAKE_SIGNED_CV(char) + __BAN_TRAITS_MAKE_SIGNED_CV(short) + __BAN_TRAITS_MAKE_SIGNED_CV(int) + __BAN_TRAITS_MAKE_SIGNED_CV(long) + __BAN_TRAITS_MAKE_SIGNED_CV(long long) + template using make_signed_t = typename make_signed::type; +#undef __BAN_TRAITS_MAKE_SIGNED_CV + template struct less { constexpr bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; } }; template struct equal { constexpr bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; } }; template struct greater { constexpr bool operator()(const T& lhs, const T& rhs) const { return lhs > rhs; } }; From 9ecd156622b79a7dd0a3d72c2e2d1c191871d9ca Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 11:21:03 +0200 Subject: [PATCH 02/46] Kenrel: Ext2 fix signed-unsigned comparisons --- kernel/kernel/FS/Ext2/Inode.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kernel/kernel/FS/Ext2/Inode.cpp b/kernel/kernel/FS/Ext2/Inode.cpp index ac0baaad5f..03f9a0d9e0 100644 --- a/kernel/kernel/FS/Ext2/Inode.cpp +++ b/kernel/kernel/FS/Ext2/Inode.cpp @@ -108,10 +108,10 @@ namespace Kernel ASSERT(!mode().ifdir()); ASSERT(offset >= 0); - if (offset >= UINT32_MAX || buffer.size() >= UINT32_MAX || buffer.size() >= (size_t)(UINT32_MAX - offset)) + if (static_cast>(offset) >= UINT32_MAX || buffer.size() >= UINT32_MAX || buffer.size() >= (size_t)(UINT32_MAX - offset)) return BAN::Error::from_errno(EOVERFLOW); - if (offset >= m_inode.size) + if (static_cast>(offset) >= m_inode.size) return 0; uint32_t count = buffer.size(); @@ -152,7 +152,7 @@ namespace Kernel ASSERT(!mode().ifdir()); ASSERT(offset >= 0); - if (offset >= UINT32_MAX || buffer.size() >= UINT32_MAX || buffer.size() >= (size_t)(UINT32_MAX - offset)) + if (static_cast>(offset) >= UINT32_MAX || buffer.size() >= UINT32_MAX || buffer.size() >= (size_t)(UINT32_MAX - offset)) return BAN::Error::from_errno(EOVERFLOW); if (m_inode.size < offset + buffer.size()) @@ -304,7 +304,7 @@ done: ASSERT(mode().ifdir()); ASSERT(offset >= 0); - if (offset >= max_used_data_block_count()) + if (static_cast>(offset) >= max_used_data_block_count()) { list->entry_count = 0; return {}; From aa2e53c4f805d4a6e95a197b60b2eebd4fbfa0a3 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 11:21:30 +0200 Subject: [PATCH 03/46] Kernel: E1000 fix physical address on 32 bit target --- kernel/kernel/Networking/E1000/E1000.cpp | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kernel/kernel/Networking/E1000/E1000.cpp b/kernel/kernel/Networking/E1000/E1000.cpp index ec9d52663e..e06ee43a64 100644 --- a/kernel/kernel/Networking/E1000/E1000.cpp +++ b/kernel/kernel/Networking/E1000/E1000.cpp @@ -173,8 +173,9 @@ namespace Kernel rx_descriptors[i].status = 0; } - write32(REG_RDBAL0, m_rx_descriptor_region->paddr() & 0xFFFFFFFF); - write32(REG_RDBAH0, m_rx_descriptor_region->paddr() >> 32); + uint64_t paddr64 = m_rx_descriptor_region->paddr(); + write32(REG_RDBAL0, paddr64 & 0xFFFFFFFF); + write32(REG_RDBAH0, paddr64 >> 32); write32(REG_RDLEN0, E1000_RX_DESCRIPTOR_COUNT * sizeof(e1000_rx_desc)); write32(REG_RDH0, 0); write32(REG_RDT0, E1000_RX_DESCRIPTOR_COUNT - 1); @@ -206,8 +207,9 @@ namespace Kernel tx_descriptors[i].cmd = 0; } - write32(REG_TDBAL, m_tx_descriptor_region->paddr() & 0xFFFFFFFF); - write32(REG_TDBAH, m_tx_descriptor_region->paddr() >> 32); + uint64_t paddr64 = m_tx_descriptor_region->paddr(); + write32(REG_TDBAL, paddr64 & 0xFFFFFFFF); + write32(REG_TDBAH, paddr64 >> 32); write32(REG_TDLEN, E1000_TX_DESCRIPTOR_COUNT * sizeof(e1000_tx_desc)); write32(REG_TDH, 0); write32(REG_TDT, 0); From 7ce0370b6a660ae37f5d445c146963ffe6802f99 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 11:22:02 +0200 Subject: [PATCH 04/46] Kernel: Define KERNEL_OFFSET for i386 target --- kernel/include/kernel/Memory/Types.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/kernel/include/kernel/Memory/Types.h b/kernel/include/kernel/Memory/Types.h index beecfaa4c1..5c06c52e7b 100644 --- a/kernel/include/kernel/Memory/Types.h +++ b/kernel/include/kernel/Memory/Types.h @@ -4,12 +4,15 @@ #if ARCH(x86_64) #define KERNEL_OFFSET 0xFFFFFFFF80000000 - #define V2P(vaddr) (((vaddr_t)(vaddr)) - KERNEL_OFFSET) - #define P2V(paddr) (((paddr_t)(paddr)) + KERNEL_OFFSET) +#elif ARCH(i386) + #define KERNEL_OFFSET 0xC0000000 #else #error #endif +#define V2P(vaddr) (((vaddr_t)(vaddr)) - KERNEL_OFFSET) +#define P2V(paddr) (((paddr_t)(paddr)) + KERNEL_OFFSET) + #define PAGE_SIZE ((uintptr_t)4096) #define PAGE_SIZE_SHIFT 12 #define PAGE_ADDR_MASK (~(uintptr_t)0xFFF) From 26ed689d30c4dd9d2be48829d915f4b3dbd59fc9 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 12:47:34 +0200 Subject: [PATCH 05/46] Kernel: Remove old GDT, IDT and MMU code from i386 It will be easier to just rewrite them --- kernel/CMakeLists.txt | 3 - kernel/arch/i386/GDT.cpp | 147 --------------------- kernel/arch/i386/IDT.cpp | 270 --------------------------------------- kernel/arch/i386/MMU.cpp | 227 -------------------------------- 4 files changed, 647 deletions(-) delete mode 100644 kernel/arch/i386/GDT.cpp delete mode 100644 kernel/arch/i386/IDT.cpp delete mode 100644 kernel/arch/i386/MMU.cpp diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 441505f56c..14a2c771be 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -117,9 +117,6 @@ elseif("${BANAN_ARCH}" STREQUAL "i386") set(KERNEL_SOURCES ${KERNEL_SOURCES} arch/i386/boot.S - arch/i386/GDT.cpp - arch/i386/IDT.cpp - arch/i386/MMU.cpp arch/i386/SpinLock.S arch/i386/Thread.S ) diff --git a/kernel/arch/i386/GDT.cpp b/kernel/arch/i386/GDT.cpp deleted file mode 100644 index d5ada22b79..0000000000 --- a/kernel/arch/i386/GDT.cpp +++ /dev/null @@ -1,147 +0,0 @@ -#include -#include - -#include - -extern "C" uintptr_t g_boot_stack_top[0]; - -namespace Kernel::GDT -{ - - struct TaskStateSegment - { - uint16_t link; - uint16_t reserved1; - uint32_t esp0; - uint16_t ss0; - uint16_t reserved2; - uint32_t esp1; - uint16_t ss1; - uint16_t reserved3; - uint32_t esp2; - uint16_t ss2; - uint16_t reserved4; - uint32_t cr3; - uint32_t eip; - uint32_t eflags; - uint32_t eax; - uint32_t ecx; - uint32_t edx; - uint32_t ebx; - uint32_t esp; - uint32_t ebp; - uint32_t esi; - uint32_t edi; - uint16_t es; - uint16_t reserved5; - uint16_t cs; - uint16_t reserved6; - uint16_t ss; - uint16_t reserved7; - uint16_t ds; - uint16_t reserved8; - uint16_t fs; - uint16_t reserved9; - uint16_t gs; - uint16_t reserved10; - uint16_t ldtr; - uint16_t reserved11; - uint16_t reserved12; - uint16_t iopb; - uint32_t ssp; - } __attribute__((packed)); - - union SegmentDescriptor - { - struct - { - uint16_t limit1; - uint16_t base1; - uint8_t base2; - uint8_t access; - uint8_t limit2 : 4; - uint8_t flags : 4; - uint8_t base3; - } __attribute__((packed)); - - struct - { - uint32_t low; - uint32_t high; - } __attribute__((packed)); - - } __attribute__((packed)); - - struct GDTR - { - uint16_t size; - uint32_t address; - } __attribute__((packed)); - - static TaskStateSegment* s_tss = nullptr; - static SegmentDescriptor* s_gdt = nullptr; - static GDTR s_gdtr; - - static void write_entry(uint8_t offset, uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) - { - SegmentDescriptor& desc = *(SegmentDescriptor*)((uintptr_t)s_gdt + offset); - desc.base1 = base; - desc.base2 = base >> 16; - desc.base3 = base >> 24; - - desc.limit1 = limit; - desc.limit2 = limit >> 16; - - desc.access = access; - - desc.flags = flags; - } - - static void write_tss(uint8_t offset) - { - s_tss = new TaskStateSegment(); - ASSERT(s_tss); - - memset(s_tss, 0x00, sizeof(TaskStateSegment)); - s_tss->ss0 = 0x10; - s_tss->esp0 = (uintptr_t)g_boot_stack_top; - - write_entry(offset, (uint32_t)s_tss, sizeof(TaskStateSegment), 0x89, 0x0); - } - - void set_tss_stack(uintptr_t esp) - { - s_tss->esp0 = esp; - } - - static void flush_gdt() - { - asm volatile("lgdt %0" :: "m"(s_gdtr)); - } - - extern "C" void flush_tss(uint16_t offset) - { - asm volatile("ltr %0" :: "m"(offset)); - } - - void initialize() - { - constexpr uint32_t descriptor_count = 6; - s_gdt = new SegmentDescriptor[descriptor_count]; - ASSERT(s_gdt); - - s_gdtr.address = (uint64_t)s_gdt; - s_gdtr.size = descriptor_count * sizeof(SegmentDescriptor) - 1; - - write_entry(0x00, 0x00000000, 0x00000, 0x00, 0x0); // null - write_entry(0x08, 0x00000000, 0xFFFFF, 0x9A, 0xC); // kernel code - write_entry(0x10, 0x00000000, 0xFFFFF, 0x92, 0xC); // kernel data - write_entry(0x18, 0x00000000, 0xFFFFF, 0xFA, 0xC); // user code - write_entry(0x20, 0x00000000, 0xFFFFF, 0xF2, 0xC); // user data - write_tss(0x28); - - flush_gdt(); - flush_tss(0x28); - } - -} diff --git a/kernel/arch/i386/IDT.cpp b/kernel/arch/i386/IDT.cpp deleted file mode 100644 index 33626a567a..0000000000 --- a/kernel/arch/i386/IDT.cpp +++ /dev/null @@ -1,270 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#define INTERRUPT_HANDLER____(i, msg) \ - static void interrupt ## i () \ - { \ - uint32_t eax, ebx, ecx, edx; \ - uint32_t esp, ebp; \ - uint32_t cr0, cr2, cr3, cr4; \ - asm volatile("":"=a"(eax),"=b"(ebx),"=c"(ecx),"=d"(edx)); \ - asm volatile("movl %%esp, %%eax":"=a"(esp)); \ - asm volatile("movl %%ebp, %%eax":"=a"(ebp)); \ - asm volatile("movl %%cr0, %%eax":"=a"(cr0)); \ - asm volatile("movl %%cr2, %%eax":"=a"(cr2)); \ - asm volatile("movl %%cr3, %%eax":"=a"(cr3)); \ - asm volatile("movl %%cr4, %%eax":"=a"(cr4)); \ - Kernel::panic(msg "\r\nRegister dump\r\n" \ - "eax=0x{8H}, ebx=0x{8H}, ecx=0x{8H}, edx=0x{8H}\r\n" \ - "esp=0x{8H}, ebp=0x{8H}\r\n" \ - "CR0=0x{8H}, CR2=0x{8H}, CR3=0x{8H}, CR4=0x{8H}\r\n", \ - eax, ebx, ecx, edx, esp, ebp, cr0, cr2, cr3, cr4); \ - } - -#define INTERRUPT_HANDLER_ERR(i, msg) \ - static void interrupt ## i () \ - { \ - uint32_t eax, ebx, ecx, edx; \ - uint32_t esp, ebp; \ - uint32_t cr0, cr2, cr3, cr4; \ - uint32_t error_code; \ - asm volatile("":"=a"(eax),"=b"(ebx),"=c"(ecx),"=d"(edx)); \ - asm volatile("movl %%esp, %%eax":"=a"(esp)); \ - asm volatile("movl %%ebp, %%eax":"=a"(ebp)); \ - asm volatile("movl %%cr0, %%eax":"=a"(cr0)); \ - asm volatile("movl %%cr2, %%eax":"=a"(cr2)); \ - asm volatile("movl %%cr3, %%eax":"=a"(cr3)); \ - asm volatile("movl %%cr4, %%eax":"=a"(cr4)); \ - asm volatile("popl %%eax":"=a"(error_code)); \ - Kernel::panic(msg " (error code: 0x{8H})\r\n" \ - "Register dump\r\n" \ - "eax=0x{8H}, ebx=0x{8H}, ecx=0x{8H}, edx=0x{8H}\r\n" \ - "esp=0x{8H}, ebp=0x{8H}\r\n" \ - "CR0=0x{8H}, CR2=0x{8H}, CR3=0x{8H}, CR4=0x{8H}\r\n", \ - eax, ebx, ecx, edx, esp, ebp, cr0, cr2, cr3, cr4, error_code); \ - } - -#define REGISTER_HANDLER(i) register_interrupt_handler(i, interrupt ## i) - -namespace IDT -{ - - struct GateDescriptor - { - uint16_t offset1; - uint16_t selector; - uint8_t reserved : 5; - uint8_t zero1 : 3; - uint8_t type : 4; - uint8_t zero2 : 1; - uint8_t DPL : 2; - uint8_t present : 1; - uint16_t offset2; - } __attribute__((packed)); - - struct IDTR - { - uint16_t size; - void* offset; - } __attribute((packed)); - - static IDTR s_idtr; - static GateDescriptor* s_idt = nullptr; - - static void(*s_irq_handlers[16])() { nullptr }; - - INTERRUPT_HANDLER____(0x00, "Division Error") - INTERRUPT_HANDLER____(0x01, "Debug") - INTERRUPT_HANDLER____(0x02, "Non-maskable Interrupt") - INTERRUPT_HANDLER____(0x03, "Breakpoint") - INTERRUPT_HANDLER____(0x04, "Overflow") - INTERRUPT_HANDLER____(0x05, "Bound Range Exception") - INTERRUPT_HANDLER____(0x06, "Invalid Opcode") - INTERRUPT_HANDLER____(0x07, "Device Not Available") - INTERRUPT_HANDLER_ERR(0x08, "Double Fault") - INTERRUPT_HANDLER____(0x09, "Coprocessor Segment Overrun") - INTERRUPT_HANDLER_ERR(0x0A, "Invalid TSS") - INTERRUPT_HANDLER_ERR(0x0B, "Segment Not Present") - INTERRUPT_HANDLER_ERR(0x0C, "Stack-Segment Fault") - INTERRUPT_HANDLER_ERR(0x0D, "General Protection Fault") - INTERRUPT_HANDLER_ERR(0x0E, "Page Fault") - INTERRUPT_HANDLER____(0x0F, "Unknown Exception 0x0F") - INTERRUPT_HANDLER____(0x10, "x87 Floating-Point Exception") - INTERRUPT_HANDLER_ERR(0x11, "Alignment Check") - INTERRUPT_HANDLER____(0x12, "Machine Check") - INTERRUPT_HANDLER____(0x13, "SIMD Floating-Point Exception") - INTERRUPT_HANDLER____(0x14, "Virtualization Exception") - INTERRUPT_HANDLER_ERR(0x15, "Control Protection Exception") - INTERRUPT_HANDLER____(0x16, "Unknown Exception 0x16") - INTERRUPT_HANDLER____(0x17, "Unknown Exception 0x17") - INTERRUPT_HANDLER____(0x18, "Unknown Exception 0x18") - INTERRUPT_HANDLER____(0x19, "Unknown Exception 0x19") - INTERRUPT_HANDLER____(0x1A, "Unknown Exception 0x1A") - INTERRUPT_HANDLER____(0x1B, "Unknown Exception 0x1B") - INTERRUPT_HANDLER____(0x1C, "Hypervisor Injection Exception") - INTERRUPT_HANDLER_ERR(0x1D, "VMM Communication Exception") - INTERRUPT_HANDLER_ERR(0x1E, "Security Exception") - INTERRUPT_HANDLER____(0x1F, "Unkown Exception 0x1F") - - extern "C" void handle_irq() - { - uint8_t irq; - for (uint32_t i = 0; i < 16; i++) - { - if (InterruptController::get().is_in_service(i)) - { - irq = i; - goto found; - } - } - - dprintln("Spurious irq"); - return; -found: - if (s_irq_handlers[irq]) - s_irq_handlers[irq](); - else - dprintln("no handler for irq 0x{2H}\n", irq); - - // NOTE: Scheduler sends PIT eoi's - if (irq != PIT_IRQ) - InterruptController::get().eoi(irq); - - Kernel::Scheduler::get().reschedule_if_idling(); - } - - extern "C" void handle_irq_common(); - asm( - ".globl handle_irq_common;" - "handle_irq_common:" - "pusha;" - "pushw %ds;" - "pushw %es;" - "pushw %ss;" - "pushw %ss;" - "popw %ds;" - "popw %es;" - "call handle_irq;" - "popw %es;" - "popw %ds;" - "popa;" - "iret;" - ); - - extern "C" void syscall_asm(); - asm( - ".global syscall_asm;" - "syscall_asm:" - "pusha;" - "pushw %ds;" - "pushw %es;" - "pushw %ss;" - "pushw %ss;" - "popw %ds;" - "popw %es;" - "pushl %edx;" - "pushl %ecx;" - "pushl %ebx;" - "pushl %eax;" - "call cpp_syscall_handler;" - "addl $16, %esp;" - "popw %es;" - "popw %ds;" - - // NOTE: following instructions are same as in 'popa', except we skip eax - // since it holds the return value of the syscall. - "popl %edi;" - "popl %esi;" - "popl %ebp;" - "addl $4, %esp;" - "popl %ebx;" - "popl %edx;" - "popl %ecx;" - "addl $4, %esp;" - - "iret;" - ); - - static void flush_idt() - { - asm volatile("lidt %0"::"m"(s_idtr)); - } - - static void register_interrupt_handler(uint8_t index, void(*f)()) - { - GateDescriptor& descriptor = s_idt[index]; - descriptor.offset1 = (uint32_t)f & 0xFFFF; - descriptor.selector = 0x08; - descriptor.type = 0xE; - descriptor.DPL = 0; - descriptor.present = 1; - descriptor.offset2 = (uint32_t)f >> 16; - } - - void register_irq_handler(uint8_t irq, void(*f)()) - { - s_irq_handlers[irq] = f; - register_interrupt_handler(IRQ_VECTOR_BASE + irq, handle_irq_common); - flush_idt(); - } - - void register_syscall_handler(uint8_t offset, void(*handler)()) - { - register_interrupt_handler(offset, handler); - s_idt[offset].DPL = 3; - } - - void initialize() - { - constexpr size_t idt_size = 0x100 * sizeof(GateDescriptor); - - s_idt = (GateDescriptor*)kmalloc(idt_size); - ASSERT(s_idt); - memset(s_idt, 0x00, idt_size); - - s_idtr.offset = s_idt; - s_idtr.size = idt_size - 1; - - REGISTER_HANDLER(0x00); - REGISTER_HANDLER(0x01); - REGISTER_HANDLER(0x02); - REGISTER_HANDLER(0x03); - REGISTER_HANDLER(0x04); - REGISTER_HANDLER(0x05); - REGISTER_HANDLER(0x06); - REGISTER_HANDLER(0x07); - REGISTER_HANDLER(0x08); - REGISTER_HANDLER(0x09); - REGISTER_HANDLER(0x0A); - REGISTER_HANDLER(0x0B); - REGISTER_HANDLER(0x0C); - REGISTER_HANDLER(0x0D); - REGISTER_HANDLER(0x0E); - REGISTER_HANDLER(0x0F); - REGISTER_HANDLER(0x10); - REGISTER_HANDLER(0x11); - REGISTER_HANDLER(0x12); - REGISTER_HANDLER(0x13); - REGISTER_HANDLER(0x14); - REGISTER_HANDLER(0x15); - REGISTER_HANDLER(0x16); - REGISTER_HANDLER(0x17); - REGISTER_HANDLER(0x18); - REGISTER_HANDLER(0x19); - REGISTER_HANDLER(0x1A); - REGISTER_HANDLER(0x1B); - REGISTER_HANDLER(0x1C); - REGISTER_HANDLER(0x1D); - REGISTER_HANDLER(0x1E); - REGISTER_HANDLER(0x1F); - - register_syscall_handler(0x80, syscall_asm); - - flush_idt(); - } - -} diff --git a/kernel/arch/i386/MMU.cpp b/kernel/arch/i386/MMU.cpp deleted file mode 100644 index 4946a7eddf..0000000000 --- a/kernel/arch/i386/MMU.cpp +++ /dev/null @@ -1,227 +0,0 @@ -#include -#include -#include -#include - -#include - -#define MMU_DEBUG_PRINT 0 - -// bits 31-12 set -#define PAGE_MASK 0xfffff000 -#define FLAGS_MASK 0x00000fff - -namespace Kernel -{ - - static MMU* s_instance = nullptr; - - void MMU::initialize() - { - ASSERT(s_instance == nullptr); - s_instance = new MMU(); - ASSERT(s_instance); - s_instance->initialize_kernel(); - s_instance->load(); - } - - MMU& MMU::get() - { - ASSERT(s_instance); - return *s_instance; - } - - static uint64_t* allocate_page_aligned_page() - { - uint64_t* page = (uint64_t*)kmalloc(PAGE_SIZE, PAGE_SIZE); - ASSERT(page); - ASSERT(((uintptr_t)page % PAGE_SIZE) == 0); - memset(page, 0, PAGE_SIZE); - return page; - } - - void MMU::initialize_kernel() - { - m_highest_paging_struct = (uint64_t*)kmalloc(sizeof(uint64_t) * 4, 32); - ASSERT(m_highest_paging_struct); - ASSERT(((uintptr_t)m_highest_paging_struct % 32) == 0); - - // allocate all page directories - for (int i = 0; i < 4; i++) - { - uint64_t* page_directory = allocate_page_aligned_page(); - m_highest_paging_struct[i] = (uint64_t)page_directory | Flags::Present; - } - - // FIXME: We should just identity map until g_kernel_end - - // create and identity map first 6 MiB - uint64_t* page_directory1 = (uint64_t*)(m_highest_paging_struct[0] & PAGE_MASK); - for (uint64_t i = 0; i < 3; i++) - { - uint64_t* page_table = allocate_page_aligned_page(); - for (uint64_t j = 0; j < 512; j++) - page_table[j] = (i << 21) | (j << 12) | Flags::ReadWrite | Flags::Present; - - page_directory1[i] = (uint64_t)page_table | Flags::ReadWrite | Flags::Present; - } - - // dont map first page (0 -> 4 KiB) so that nullptr dereference - // causes page fault :) - uint64_t* page_table1 = (uint64_t*)(page_directory1[0] & PAGE_MASK); - page_table1[0] = 0; - } - - MMU::MMU() - { - if (s_instance == nullptr) - return; - - // Here we copy the s_instances paging structs since they are - // global for every process - - uint64_t* global_pdpt = s_instance->m_highest_paging_struct; - - uint64_t* pdpt = (uint64_t*)kmalloc(sizeof(uint64_t) * 4, 32); - ASSERT(pdpt); - - for (uint32_t pdpte = 0; pdpte < 4; pdpte++) - { - if (!(global_pdpt[pdpte] & Flags::Present)) - continue; - - uint64_t* global_pd = (uint64_t*)(global_pdpt[pdpte] & PAGE_MASK); - - uint64_t* pd = allocate_page_aligned_page(); - pdpt[pdpte] = (uint64_t)pd | (global_pdpt[pdpte] & ~PAGE_MASK); - - for (uint32_t pde = 0; pde < 512; pde++) - { - if (!(global_pd[pde] & Flags::Present)) - continue; - - uint64_t* global_pt = (uint64_t*)(global_pd[pde] & PAGE_MASK); - - uint64_t* pt = allocate_page_aligned_page(); - pd[pde] = (uint64_t)pt | (global_pd[pde] & ~PAGE_MASK); - - memcpy(pt, global_pt, PAGE_SIZE); - } - } - - m_highest_paging_struct = pdpt; - } - - MMU::~MMU() - { - uint64_t* pdpt = m_highest_paging_struct; - for (uint32_t pdpte = 0; pdpte < 512; pdpte++) - { - if (!(pdpt[pdpte] & Flags::Present)) - continue; - uint64_t* pd = (uint64_t*)(pdpt[pdpte] & PAGE_MASK); - for (uint32_t pde = 0; pde < 512; pde++) - { - if (!(pd[pde] & Flags::Present)) - continue; - kfree((void*)(pd[pde] & PAGE_MASK)); - } - kfree(pd); - } - kfree(pdpt); - } - - void MMU::load() - { - asm volatile("movl %0, %%cr3" :: "r"(m_highest_paging_struct)); - } - - void MMU::map_page_at(paddr_t paddr, vaddr_t vaddr, uint8_t flags) - { - #if MMU_DEBUG_PRINT - dprintln("AllocatePage(0x{8H})", address); - #endif - ASSERT(flags & Flags::Present); - - ASSERT(!(paddr & ~PAGE_MASK)); - ASSERT(!(vaddr & ~PAGE_MASK)); - - uint32_t pdpte = (vaddr & 0xC0000000) >> 30; - uint32_t pde = (vaddr & 0x3FE00000) >> 21; - uint32_t pte = (vaddr & 0x001FF000) >> 12; - - uint64_t* page_directory = (uint64_t*)(m_highest_paging_struct[pdpte] & PAGE_MASK); - if (!(page_directory[pde] & Flags::Present)) - { - uint64_t* page_table = allocate_page_aligned_page(); - page_directory[pde] = (uint64_t)page_table; - } - page_directory[pde] |= flags; - - uint64_t* page_table = (uint64_t*)(page_directory[pde] & PAGE_MASK); - page_table[pte] = paddr | flags; - } - - void MMU::identity_map_page(paddr_t address, uint8_t flags) - { - address &= PAGE_MASK; - map_page_at(address, address, flags); - } - - void MMU::identity_map_range(paddr_t address, ptrdiff_t size, uint8_t flags) - { - paddr_t s_page = address & PAGE_MASK; - paddr_t e_page = (address + size - 1) & PAGE_MASK; - for (paddr_t page = s_page; page <= e_page; page += PAGE_SIZE) - identity_map_page(page, flags); - } - - void MMU::unmap_page(vaddr_t address) - { - #if MMU_DEBUG_PRINT - dprintln("UnAllocatePage(0x{8H})", address & PAGE_MASK); - #endif - - uint32_t pdpte = (address & 0xC0000000) >> 30; - uint32_t pde = (address & 0x3FE00000) >> 21; - uint32_t pte = (address & 0x001FF000) >> 12; - - uint64_t* page_directory = (uint64_t*)(m_highest_paging_struct[pdpte] & PAGE_MASK); - if (!(page_directory[pde] & Flags::Present)) - return; - - uint64_t* page_table = (uint64_t*)(page_directory[pde] & PAGE_MASK); - if (!(page_table[pte] & Flags::Present)) - return; - - page_table[pte] = 0; - - // TODO: Unallocate the page table if this was the only allocated page - } - - void MMU::unmap_range(vaddr_t address, ptrdiff_t size) - { - uintptr_t s_page = address & PAGE_MASK; - uintptr_t e_page = (address + size - 1) & PAGE_MASK; - for (uintptr_t page = s_page; page <= e_page; page += PAGE_SIZE) - unmap_page(page); - } - - uint8_t MMU::get_page_flags(vaddr_t address) const - { - uint32_t pdpte = (address & 0xC0000000) >> 30; - uint32_t pde = (address & 0x3FE00000) >> 21; - uint32_t pte = (address & 0x001FF000) >> 12; - - uint64_t* page_directory = (uint64_t*)(m_highest_paging_struct[pdpte] & PAGE_MASK); - if (!(page_directory[pde] & Flags::Present)) - return 0; - - uint64_t* page_table = (uint64_t*)(page_directory[pde] & PAGE_MASK); - if (!(page_table[pte] & Flags::Present)) - return 0; - - return page_table[pte] & FLAGS_MASK; - } - -} From 45cea14165bac11f68fb4742e44a2662404656a5 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 12:48:54 +0200 Subject: [PATCH 06/46] Kernel: Move sys_fork trampolines to kernel/arch/ directory --- kernel/CMakeLists.txt | 3 ++- kernel/arch/i386/Syscall.S | 18 ++++++++++++++++++ kernel/{kernel => arch/x86_64}/Syscall.S | 0 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 kernel/arch/i386/Syscall.S rename kernel/{kernel => arch/x86_64}/Syscall.S (100%) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 14a2c771be..a7438da8bd 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -83,7 +83,6 @@ set(KERNEL_SOURCES kernel/Storage/Partition.cpp kernel/Storage/StorageDevice.cpp kernel/Syscall.cpp - kernel/Syscall.S kernel/Terminal/FramebufferTerminal.cpp kernel/Terminal/Serial.cpp kernel/Terminal/TTY.cpp @@ -111,6 +110,7 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") arch/x86_64/interrupts.S arch/x86_64/PageTable.cpp arch/x86_64/Signal.S + arch/x86_64/Syscall.S arch/x86_64/Thread.S ) elseif("${BANAN_ARCH}" STREQUAL "i386") @@ -118,6 +118,7 @@ elseif("${BANAN_ARCH}" STREQUAL "i386") ${KERNEL_SOURCES} arch/i386/boot.S arch/i386/SpinLock.S + arch/i386/Syscall.S arch/i386/Thread.S ) else() diff --git a/kernel/arch/i386/Syscall.S b/kernel/arch/i386/Syscall.S new file mode 100644 index 0000000000..617549c45d --- /dev/null +++ b/kernel/arch/i386/Syscall.S @@ -0,0 +1,18 @@ +.global sys_fork_trampoline +sys_fork_trampoline: + subl $4, %esp + pushl %ebx + pushl %ebp + call read_rip + testl %eax, %eax + je .done + subl $8, %esp + pushl %eax + pushl %esp + call sys_fork + addl $16, %esp +.done: + popl %ebp + popl %ebx + addl $4, %esp + ret diff --git a/kernel/kernel/Syscall.S b/kernel/arch/x86_64/Syscall.S similarity index 100% rename from kernel/kernel/Syscall.S rename to kernel/arch/x86_64/Syscall.S From d92078525630fcb3f83352febf3f56811db82f4c Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 12:58:38 +0200 Subject: [PATCH 07/46] Kernel: RDRAND on i386 is called twice with 32 bit register --- kernel/kernel/Random.cpp | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/kernel/kernel/Random.cpp b/kernel/kernel/Random.cpp index 1990635d1b..447019ed3d 100644 --- a/kernel/kernel/Random.cpp +++ b/kernel/kernel/Random.cpp @@ -20,7 +20,19 @@ namespace Kernel if (ecx & CPUID::ECX_RDRND) { - asm volatile("rdrand %0" : "=a"(s_rand_seed)); +#if ARCH(x86_64) + asm volatile("rdrand %0" : "=r"(s_rand_seed)); +#elif ARCH(i386) + uint32_t lo, hi; + asm volatile( + "rdrand %[lo];" + "rdrand %[hi];" + : [lo]"=r"(lo), [hi]"=r"(hi) + ); + s_rand_seed = ((uint64_t)hi << 32) | lo; +#else + #error +#endif dprintln("RNG seeded by RDRAND"); } else From c0dff5e2039719278e01c1ca277c4c5d4e195e0d Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 12:59:35 +0200 Subject: [PATCH 08/46] Kernel: Scheduler/Thread add inline assembly for i386 --- kernel/kernel/Scheduler.cpp | 16 ++++++++++++++++ kernel/kernel/Thread.cpp | 21 +++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 2b892ae2d3..8df08cf889 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -19,7 +19,13 @@ namespace Kernel ALWAYS_INLINE static void load_temp_stack() { +#if ARCH(x86_64) asm volatile("movq %0, %%rsp" :: "rm"(Processor::current_stack_top())); +#elif ARCH(i386) + asm volatile("movl %0, %%esp" :: "rm"(Processor::current_stack_top())); +#else + #error +#endif } BAN::ErrorOr Scheduler::initialize() @@ -209,11 +215,21 @@ namespace Kernel #if __enable_sse if (current != Thread::sse_thread()) { +#if ARCH(x86_64) asm volatile( "movq %cr0, %rax;" "orq $(1 << 3), %rax;" "movq %rax, %cr0" ); +#elif ARCH(i386) + asm volatile( + "movl %cr0, %eax;" + "orl $(1 << 3), %eax;" + "movl %eax, %cr0" + ); +#else + #error +#endif } #endif diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 7e44db5ea9..3552088b2d 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -99,16 +99,33 @@ namespace Kernel : m_tid(tid), m_process(process) { #if __enable_sse + #if ARCH(x86_64) uintptr_t cr0; asm volatile( "movq %%cr0, %%rax;" - "movq %%rax, %%rbx;" + "movq %%rax, %[cr0];" "andq $~(1 << 3), %%rax;" "movq %%rax, %%cr0;" - : "=b"(cr0) + : [cr0]"=r"(cr0) + :: "rax" ); save_sse(); asm volatile("movq %0, %%cr0" :: "r"(cr0)); + #elif ARCH(i386) + uintptr_t cr0; + asm volatile( + "movl %%cr0, %%eax;" + "movl %%eax, %[cr0];" + "andl $~(1 << 3), %%eax;" + "movl %%eax, %%cr0;" + : [cr0]"=r"(cr0) + :: "eax" + ); + save_sse(); + asm volatile("movl %0, %%cr0" :: "r"(cr0)); + #else + #error + #endif #endif } From 2352c8604846d32c69963e5ff5e07e6097913854 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 13:00:57 +0200 Subject: [PATCH 09/46] Kernel: i386 has 14 indirect blocks in TmpInode instead of 2 This allows keeping size of TmpInodeInfo as 128 --- kernel/include/kernel/FS/TmpFS/Definitions.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kernel/include/kernel/FS/TmpFS/Definitions.h b/kernel/include/kernel/FS/TmpFS/Definitions.h index 92543775db..bfbe33d3d1 100644 --- a/kernel/include/kernel/FS/TmpFS/Definitions.h +++ b/kernel/include/kernel/FS/TmpFS/Definitions.h @@ -21,12 +21,23 @@ namespace Kernel size_t size { 0 }; blkcnt_t blocks { 0 }; +#if ARCH(x86_64) // 2x direct blocks // 1x singly indirect // 1x doubly indirect // 1x triply indirect BAN::Array block; static constexpr size_t direct_block_count = 2; +#elif ARCH(i386) + // 14x direct blocks + // 1x singly indirect + // 1x doubly indirect + // 1x triply indirect + BAN::Array block; + static constexpr size_t direct_block_count = 14; +#else + #error +#endif static constexpr size_t max_size = direct_block_count * PAGE_SIZE + (PAGE_SIZE / sizeof(paddr_t)) * PAGE_SIZE + From 0424082e7b0eb533d8c01bf6b96f35ca7367ba66 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 13:32:07 +0200 Subject: [PATCH 10/46] Kernel: Only compile lai for x86_64 targets I will be dropping lai entirely soon. Once I get to writing AML interpreter. --- kernel/CMakeLists.txt | 17 ++++++++--------- kernel/kernel/ACPI.cpp | 4 ++++ kernel/kernel/InterruptController.cpp | 2 ++ kernel/kernel/Process.cpp | 22 ++++++++++------------ 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index a7438da8bd..6cd5e41fd8 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -113,6 +113,13 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") arch/x86_64/Syscall.S arch/x86_64/Thread.S ) + file(GLOB_RECURSE LAI_SOURCES + lai/*.c + ) + set(LAI_SOURCES + ${LAI_SOURCES} + kernel/lai_host.cpp + ) elseif("${BANAN_ARCH}" STREQUAL "i386") set(KERNEL_SOURCES ${KERNEL_SOURCES} @@ -125,14 +132,6 @@ else() message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}") endif() -file(GLOB_RECURSE LAI_SOURCES - lai/*.c -) -set(LAI_SOURCES - ${LAI_SOURCES} - kernel/lai_host.cpp -) - set(BAN_SOURCES ../BAN/BAN/Assert.cpp ../BAN/BAN/New.cpp @@ -177,7 +176,7 @@ if(ENABLE_KERNEL_UBSAN) endif() if("${BANAN_ARCH}" STREQUAL "x86_64") - target_compile_options(kernel PUBLIC -mcmodel=kernel -mno-red-zone -mno-mmx) + target_compile_options(kernel PUBLIC -mcmodel=kernel -mno-red-zone) target_link_options(kernel PUBLIC LINKER:-z,max-page-size=4096) target_link_options(kernel PUBLIC LINKER:-T,${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/linker.ld) elseif("${BANAN_ARCH}" STREQUAL "i386") diff --git a/kernel/kernel/ACPI.cpp b/kernel/kernel/ACPI.cpp index fabd8e38a6..2d8ccf33a3 100644 --- a/kernel/kernel/ACPI.cpp +++ b/kernel/kernel/ACPI.cpp @@ -31,7 +31,9 @@ namespace Kernel if (s_instance == nullptr) return BAN::Error::from_errno(ENOMEM); TRY(s_instance->initialize_impl()); +#if ARCH(x86_64) lai_create_namespace(); +#endif return {}; } @@ -93,7 +95,9 @@ namespace Kernel const RSDP* rsdp = locate_rsdp(); if (rsdp == nullptr) return BAN::Error::from_error_code(ErrorCode::ACPI_NoRootSDT); +#if ARCH(x86_64) lai_set_acpi_revision(rsdp->revision); +#endif uint32_t root_entry_count = 0; diff --git a/kernel/kernel/InterruptController.cpp b/kernel/kernel/InterruptController.cpp index 7989e4c1ed..6cd043925f 100644 --- a/kernel/kernel/InterruptController.cpp +++ b/kernel/kernel/InterruptController.cpp @@ -42,7 +42,9 @@ namespace Kernel void InterruptController::enter_acpi_mode() { +#if ARCH(x86_64) if (lai_enable_acpi(m_using_apic ? 1 : 0) != 0) +#endif dwarnln("could not enter acpi mode"); } diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 73a85bdba5..9bb7431bf7 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -1187,7 +1187,9 @@ namespace Kernel [[noreturn]] static void reset_system() { +#if ARCH(x86_64) lai_acpi_reset(); +#endif // acpi reset did not work @@ -1206,21 +1208,17 @@ namespace Kernel DevFileSystem::get().initiate_sync(true); - lai_api_error_t error; - switch (command) - { - case POWEROFF_REBOOT: - reset_system(); - break; - case POWEROFF_SHUTDOWN: - error = lai_enter_sleep(5); - break; - default: - ASSERT_NOT_REACHED(); - } + if (command == POWEROFF_REBOOT) + reset_system(); +#if ARCH(x86_64) + auto error = lai_enter_sleep(5); // If we reach here, there was an error dprintln("{}", lai_api_error_to_string(error)); +#else + dprintln("poweroff available only on x86_64"); +#endif + return BAN::Error::from_errno(EUNKNOWN); } From 7f029b271305f2ed5ee01015cb300b794cda0919 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 13:33:13 +0200 Subject: [PATCH 11/46] Kernel: Allow Processor compilation for i386 targets This is achieved by rewriting some inline assembly and changing ProcessorID to be 32 bit value. For some reason if processor id is 8 bits gcc runs out of 8 bit registers on i386. --- kernel/include/kernel/Processor.h | 26 +++++++++++++------------- kernel/kernel/Processor.cpp | 11 ++++++----- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index fe039349a2..f559f308d0 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -16,10 +16,10 @@ namespace Kernel Enabled, }; - using ProcessorID = uint8_t; - constexpr ProcessorID PROCESSOR_NONE = 0xFF; + using ProcessorID = uint32_t; + constexpr ProcessorID PROCESSOR_NONE = 0xFFFFFFFF; -#if ARCH(x86_64) +#if ARCH(x86_64) || ARCH(i386) class Processor { BAN_NON_COPYABLE(Processor); @@ -75,16 +75,16 @@ namespace Kernel template static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8) { -#define __ASM_INPUT(operation) operation " %%gs:%a[offset], %[result]" : [result]"=r"(result) : [offset]"ir"(offset) +#define __ASM_INPUT(operation) asm volatile(operation " %%gs:%a[offset], %[result]" : [result]"=r"(result) : [offset]"ir"(offset)) T result; if constexpr(sizeof(T) == 8) - asm volatile(__ASM_INPUT("movq")); + __ASM_INPUT("movq"); if constexpr(sizeof(T) == 4) - asm volatile(__ASM_INPUT("movl")); + __ASM_INPUT("movl"); if constexpr(sizeof(T) == 2) - asm volatile(__ASM_INPUT("movw")); + __ASM_INPUT("movw"); if constexpr(sizeof(T) == 1) - asm volatile(__ASM_INPUT("movb")); + __ASM_INPUT("movb"); return result; #undef __ASM_INPUT } @@ -92,15 +92,15 @@ namespace Kernel template static void write_gs_sized(uintptr_t offset, T value) requires(sizeof(T) <= 8) { -#define __ASM_INPUT(operation) operation " %[value], %%gs:%a[offset]" :: [value]"r"(value), [offset]"ir"(offset) : "memory" +#define __ASM_INPUT(operation) asm volatile(operation " %[value], %%gs:%a[offset]" :: [value]"r"(value), [offset]"ir"(offset) : "memory") if constexpr(sizeof(T) == 8) - asm volatile(__ASM_INPUT("movq")); + __ASM_INPUT("movq"); if constexpr(sizeof(T) == 4) - asm volatile(__ASM_INPUT("movl")); + __ASM_INPUT("movl"); if constexpr(sizeof(T) == 2) - asm volatile(__ASM_INPUT("movw")); + __ASM_INPUT("movw"); if constexpr(sizeof(T) == 1) - asm volatile(__ASM_INPUT("movb")); + __ASM_INPUT("movb"); #undef __ASM_INPUT } diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index f7805f9f83..a9b6acd819 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -13,14 +13,13 @@ namespace Kernel static ProcessorID read_processor_id() { - uint8_t id; + uint32_t id; asm volatile( "movl $1, %%eax;" "cpuid;" "shrl $24, %%ebx;" - "movb %%bl, %0;" - : "=rm"(id) - :: "eax", "ebx", "ecx", "edx" + : "=b"(id) + :: "eax", "ecx", "edx" ); return id; } @@ -55,7 +54,9 @@ namespace Kernel // set gs base to pointer to this processor uint64_t ptr = reinterpret_cast(&processor); - asm volatile("wrmsr" :: "d"(ptr >> 32), "a"(ptr), "c"(MSR_IA32_GS_BASE)); + uint32_t ptr_hi = ptr >> 32; + uint32_t ptr_lo = ptr & 0xFFFFFFFF; + asm volatile("wrmsr" :: "d"(ptr_hi), "a"(ptr_lo), "c"(MSR_IA32_GS_BASE)); ASSERT(processor.m_gdt); processor.gdt().load(); From b5aae34d8621db8088244e72114f7ef24742e8b6 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 13:34:26 +0200 Subject: [PATCH 12/46] Kernel: Specify template paramenters where they cannot be deduced --- kernel/kernel/Device/FramebufferDevice.cpp | 2 +- kernel/kernel/Storage/NVMe/Namespace.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kernel/kernel/Device/FramebufferDevice.cpp b/kernel/kernel/Device/FramebufferDevice.cpp index f2a0fb5aa9..f6ffe36693 100644 --- a/kernel/kernel/Device/FramebufferDevice.cpp +++ b/kernel/kernel/Device/FramebufferDevice.cpp @@ -240,7 +240,7 @@ namespace Kernel return BAN::Error::from_errno(ENOMEM); auto region = BAN::UniqPtr::adopt(region_ptr); - TRY(region->initialize({ m_vaddr, m_vaddr + BAN::Math::div_round_up(m_size, PAGE_SIZE) * PAGE_SIZE })); + TRY(region->initialize({ m_vaddr, m_vaddr + BAN::Math::div_round_up(m_size, PAGE_SIZE) * PAGE_SIZE })); return BAN::UniqPtr(BAN::move(region)); } diff --git a/kernel/kernel/Storage/NVMe/Namespace.cpp b/kernel/kernel/Storage/NVMe/Namespace.cpp index ca494ed12c..866b656652 100644 --- a/kernel/kernel/Storage/NVMe/Namespace.cpp +++ b/kernel/kernel/Storage/NVMe/Namespace.cpp @@ -63,7 +63,7 @@ namespace Kernel for (uint64_t i = 0; i < sector_count;) { - uint16_t count = BAN::Math::min(sector_count - i, m_dma_region->size() / m_block_size); + uint16_t count = BAN::Math::min(sector_count - i, m_dma_region->size() / m_block_size); NVMe::SubmissionQueueEntry sqe {}; sqe.opc = NVMe::OPC_IO_READ; @@ -90,7 +90,7 @@ namespace Kernel for (uint64_t i = 0; i < sector_count;) { - uint16_t count = BAN::Math::min(sector_count - i, m_dma_region->size() / m_block_size); + uint16_t count = BAN::Math::min(sector_count - i, m_dma_region->size() / m_block_size); memcpy(reinterpret_cast(m_dma_region->vaddr()), buffer.data() + i * m_block_size, count * m_block_size); From 3e4d410646865a0e1266e09ab5b1a8247c10f95b Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 13:34:57 +0200 Subject: [PATCH 13/46] Kernel: Fix AHCI device physical address writing on i386 target --- kernel/kernel/Storage/ATA/AHCI/Device.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/kernel/Storage/ATA/AHCI/Device.cpp b/kernel/kernel/Storage/ATA/AHCI/Device.cpp index 52f6b706f0..c671f6337d 100644 --- a/kernel/kernel/Storage/ATA/AHCI/Device.cpp +++ b/kernel/kernel/Storage/ATA/AHCI/Device.cpp @@ -70,11 +70,11 @@ namespace Kernel stop_cmd(m_port); - paddr_t fis_paddr = m_dma_region->paddr(); + uint64_t fis_paddr = m_dma_region->paddr(); m_port->fb = fis_paddr & 0xFFFFFFFF; m_port->fbu = fis_paddr >> 32; - paddr_t command_list_paddr = fis_paddr + sizeof(ReceivedFIS); + uint64_t command_list_paddr = fis_paddr + sizeof(ReceivedFIS); m_port->clb = command_list_paddr & 0xFFFFFFFF; m_port->clbu = command_list_paddr >> 32; @@ -236,8 +236,9 @@ namespace Kernel volatile auto& command_table = *reinterpret_cast(m_dma_region->paddr_to_vaddr(command_header.ctba)); memset(const_cast(&command_table), 0x00, sizeof(HBACommandTable)); - command_table.prdt_entry[0].dba = m_data_dma_region->paddr() & 0xFFFFFFFF; - command_table.prdt_entry[0].dbau = m_data_dma_region->paddr() >> 32; + uint64_t data_dma_paddr64 = m_data_dma_region->paddr(); + command_table.prdt_entry[0].dba = data_dma_paddr64 & 0xFFFFFFFF; + command_table.prdt_entry[0].dbau = data_dma_paddr64 >> 32; command_table.prdt_entry[0].dbc = sector_count * sector_size() - 1; command_table.prdt_entry[0].i = 1; From fe17958b9f5c8b0f8c5534e94c3ff1ddf6e4c725 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 14:48:33 +0200 Subject: [PATCH 14/46] Kernel: Rename rsp->sp and rip->ip This makes more sense if we support i386 --- kernel/arch/i386/Syscall.S | 2 +- kernel/arch/i386/Thread.S | 12 ++-- kernel/arch/x86_64/IDT.cpp | 20 +++---- kernel/arch/x86_64/Syscall.S | 2 +- kernel/arch/x86_64/Thread.S | 12 ++-- kernel/include/kernel/Arch.h | 6 +- kernel/include/kernel/InterruptStack.h | 10 ++-- kernel/include/kernel/Thread.h | 26 ++++---- kernel/kernel/Debug.cpp | 18 +++--- kernel/kernel/Process.cpp | 4 +- kernel/kernel/Scheduler.cpp | 18 +++--- kernel/kernel/Syscall.cpp | 8 +-- kernel/kernel/Thread.cpp | 82 +++++++++++++------------- 13 files changed, 108 insertions(+), 112 deletions(-) diff --git a/kernel/arch/i386/Syscall.S b/kernel/arch/i386/Syscall.S index 617549c45d..f4a40759b7 100644 --- a/kernel/arch/i386/Syscall.S +++ b/kernel/arch/i386/Syscall.S @@ -3,7 +3,7 @@ sys_fork_trampoline: subl $4, %esp pushl %ebx pushl %ebp - call read_rip + call read_ip testl %eax, %eax je .done subl $8, %esp diff --git a/kernel/arch/i386/Thread.S b/kernel/arch/i386/Thread.S index 0c83770b4e..30e8811e8f 100644 --- a/kernel/arch/i386/Thread.S +++ b/kernel/arch/i386/Thread.S @@ -1,6 +1,6 @@ -# uint32_t read_rip() -.global read_rip -read_rip: +# uint32_t read_ip() +.global read_ip +read_ip: popl %eax jmp *%eax @@ -9,7 +9,7 @@ exit_thread_trampoline: pushl (%esp) ret -# void start_thread(uint32_t esp, uint32_t eip) +# void start_thread(uint32_t sp, uint32_t ip) .global start_thread start_thread: movl 8(%esp), %ecx @@ -19,7 +19,7 @@ start_thread: sti jmp *%ecx -# void continue_thread(uint32_t rsp, uint32_t rip) +# void continue_thread(uint32_t sp, uint32_t ip) .global continue_thread continue_thread: movl 8(%esp), %ecx @@ -27,7 +27,7 @@ continue_thread: movl $0, %eax jmp *%ecx -# void thread_jump_userspace(uint32_t rsp, uint32_t rip) +# void thread_jump_userspace(uint32_t sp, uint32_t ip) .global thread_jump_userspace thread_jump_userspace: movl $0x23, %eax diff --git a/kernel/arch/x86_64/IDT.cpp b/kernel/arch/x86_64/IDT.cpp index ceadf7458c..0cf1531654 100644 --- a/kernel/arch/x86_64/IDT.cpp +++ b/kernel/arch/x86_64/IDT.cpp @@ -161,24 +161,24 @@ namespace Kernel if (tid) { - Thread::current().set_return_rsp(interrupt_stack.rsp); - Thread::current().set_return_rip(interrupt_stack.rip); + Thread::current().set_return_sp(interrupt_stack.sp); + Thread::current().set_return_ip(interrupt_stack.ip); if (isr == ISR::PageFault) { // Check if stack is OOB auto& stack = Thread::current().stack(); auto& istack = Thread::current().interrupt_stack(); - if (stack.vaddr() < interrupt_stack.rsp && interrupt_stack.rsp <= stack.vaddr() + stack.size()) + if (stack.vaddr() < interrupt_stack.sp && interrupt_stack.sp <= stack.vaddr() + stack.size()) ; // using normal stack - else if (istack.vaddr() < interrupt_stack.rsp && interrupt_stack.rsp <= istack.vaddr() + istack.size()) + else if (istack.vaddr() < interrupt_stack.sp && interrupt_stack.sp <= istack.vaddr() + istack.size()) ; // using interrupt stack else { derrorln("Stack pointer out of bounds!"); - derrorln("rip {H}", interrupt_stack.rip); + derrorln("rip {H}", interrupt_stack.ip); derrorln("rsp {H}, stack {H}->{H}, istack {H}->{H}", - interrupt_stack.rsp, + interrupt_stack.sp, stack.vaddr(), stack.vaddr() + stack.size(), istack.vaddr(), istack.vaddr() + istack.size() ); @@ -225,9 +225,9 @@ namespace Kernel #endif } - if (PageTable::current().get_page_flags(interrupt_stack.rip & PAGE_ADDR_MASK) & PageTable::Flags::Present) + if (PageTable::current().get_page_flags(interrupt_stack.ip & PAGE_ADDR_MASK) & PageTable::Flags::Present) { - auto* machine_code = (const uint8_t*)interrupt_stack.rip; + auto* machine_code = (const uint8_t*)interrupt_stack.ip; dwarnln("While executing: {2H}{2H}{2H}{2H}{2H}{2H}{2H}{2H}", machine_code[0], machine_code[1], @@ -308,8 +308,8 @@ done: if (Scheduler::current_tid()) { - Thread::current().set_return_rsp(interrupt_stack.rsp); - Thread::current().set_return_rip(interrupt_stack.rip); + Thread::current().set_return_sp(interrupt_stack.sp); + Thread::current().set_return_ip(interrupt_stack.ip); } if (!InterruptController::get().is_in_service(irq)) diff --git a/kernel/arch/x86_64/Syscall.S b/kernel/arch/x86_64/Syscall.S index 4dc203c06e..45c1656086 100644 --- a/kernel/arch/x86_64/Syscall.S +++ b/kernel/arch/x86_64/Syscall.S @@ -6,7 +6,7 @@ sys_fork_trampoline: pushq %r13 pushq %r14 pushq %r15 - call read_rip + call read_ip testq %rax, %rax je .done movq %rax, %rsi diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index e5c55ff6ee..62973ada37 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -1,6 +1,6 @@ -# uint64_t read_rip() -.global read_rip -read_rip: +# uint64_t read_() +.global read_ip +read_ip: popq %rax jmp *%rax @@ -8,7 +8,7 @@ exit_thread_trampoline: movq 8(%rsp), %rdi ret -# void start_thread(uint64_t rsp, uint64_t rip) +# void start_thread(uint64_t sp, uint64_t ip) .global start_thread start_thread: movq %rdi, %rsp @@ -18,14 +18,14 @@ start_thread: sti jmp *%rsi -# void continue_thread(uint64_t rsp, uint64_t rip) +# void continue_thread(uint64_t sp, uint64_t ip) .global continue_thread continue_thread: movq %rdi, %rsp movq $0, %rax jmp *%rsi -# void thread_userspace_trampoline(uint64_t rsp, uint64_t rip, int argc, char** argv, char** envp) +# void thread_userspace_trampoline(uint64_t sp, uint64_t ip, int argc, char** argv, char** envp) .global thread_userspace_trampoline thread_userspace_trampoline: pushq $0x23 diff --git a/kernel/include/kernel/Arch.h b/kernel/include/kernel/Arch.h index 068ebdd577..ee3b2e7add 100644 --- a/kernel/include/kernel/Arch.h +++ b/kernel/include/kernel/Arch.h @@ -21,8 +21,4 @@ #include -#ifdef __cplusplus -extern "C" uintptr_t read_rip(); -#else -extern uintptr_t read_rip(); -#endif +extern "C" uintptr_t read_ip(); diff --git a/kernel/include/kernel/InterruptStack.h b/kernel/include/kernel/InterruptStack.h index 2ed66d2ef2..6a112874e4 100644 --- a/kernel/include/kernel/InterruptStack.h +++ b/kernel/include/kernel/InterruptStack.h @@ -7,11 +7,11 @@ namespace Kernel struct InterruptStack { - uint64_t rip; - uint64_t cs; - uint64_t flags; - uint64_t rsp; - uint64_t ss; + uintptr_t ip; + uintptr_t cs; + uintptr_t flags; + uintptr_t sp; + uintptr_t ss; }; } diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index 7b0e8e98bb..bae255407e 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -33,7 +33,7 @@ namespace Kernel static BAN::ErrorOr create_userspace(Process*); ~Thread(); - BAN::ErrorOr clone(Process*, uintptr_t rsp, uintptr_t rip); + BAN::ErrorOr clone(Process*, uintptr_t sp, uintptr_t ip); void setup_exec(); void setup_process_cleanup(); @@ -52,17 +52,17 @@ namespace Kernel BAN::ErrorOr block_or_eintr_or_timeout(Semaphore& semaphore, uint64_t timeout_ms, bool etimedout); BAN::ErrorOr block_or_eintr_or_waketime(Semaphore& semaphore, uint64_t wake_time_ms, bool etimedout); - void set_return_rsp(uintptr_t& rsp) { m_return_rsp = &rsp; } - void set_return_rip(uintptr_t& rip) { m_return_rip = &rip; } - uintptr_t return_rsp() { ASSERT(m_return_rsp); return *m_return_rsp; } - uintptr_t return_rip() { ASSERT(m_return_rip); return *m_return_rip; } + void set_return_sp(uintptr_t& sp) { m_return_sp = &sp; } + void set_return_ip(uintptr_t& ip) { m_return_ip = &ip; } + uintptr_t return_sp() { ASSERT(m_return_sp); return *m_return_sp; } + uintptr_t return_ip() { ASSERT(m_return_ip); return *m_return_ip; } pid_t tid() const { return m_tid; } - void set_rsp(uintptr_t rsp) { m_rsp = rsp; validate_stack(); } - void set_rip(uintptr_t rip) { m_rip = rip; } - uintptr_t rsp() const { return m_rsp; } - uintptr_t rip() const { return m_rip; } + void set_sp(uintptr_t sp) { m_sp = sp; validate_stack(); } + void set_ip(uintptr_t ip) { m_ip = ip; } + uintptr_t sp() const { return m_sp; } + uintptr_t ip() const { return m_ip; } void set_started() { ASSERT(m_state == State::NotStarted); m_state = State::Executing; } State state() const { return m_state; } @@ -104,15 +104,15 @@ namespace Kernel static constexpr size_t m_interrupt_stack_size = PAGE_SIZE * 2; BAN::UniqPtr m_interrupt_stack; BAN::UniqPtr m_stack; - uintptr_t m_rip { 0 }; - uintptr_t m_rsp { 0 }; + uintptr_t m_ip { 0 }; + uintptr_t m_sp { 0 }; const pid_t m_tid { 0 }; State m_state { State::NotStarted }; Process* m_process { nullptr }; bool m_is_userspace { false }; - uintptr_t* m_return_rsp { nullptr }; - uintptr_t* m_return_rip { nullptr }; + uintptr_t* m_return_sp { nullptr }; + uintptr_t* m_return_ip { nullptr }; uint64_t m_signal_pending_mask { 0 }; uint64_t m_signal_block_mask { 0 }; diff --git a/kernel/kernel/Debug.cpp b/kernel/kernel/Debug.cpp index e096059438..bbf4b68316 100644 --- a/kernel/kernel/Debug.cpp +++ b/kernel/kernel/Debug.cpp @@ -21,8 +21,8 @@ namespace Debug struct stackframe { - stackframe* rbp; - uintptr_t rip; + stackframe* bp; + uintptr_t ip; }; SpinLockGuard _(s_debug_lock); @@ -33,8 +33,8 @@ namespace Debug dprintln("Could not get frame address"); return; } - uintptr_t first_rip = frame->rip; - uintptr_t last_rip = 0; + uintptr_t first_ip = frame->ip; + uintptr_t last_ip = 0; bool first = true; BAN::Formatter::print(Debug::putchar, "\e[36mStack trace:\r\n"); @@ -46,21 +46,21 @@ namespace Debug break; } - BAN::Formatter::print(Debug::putchar, " {}\r\n", (void*)frame->rip); + BAN::Formatter::print(Debug::putchar, " {}\r\n", (void*)frame->ip); - if (!first && frame->rip == first_rip) + if (!first && frame->ip == first_ip) { derrorln("looping kernel panic :("); break; } - else if (!first && frame->rip == last_rip) + else if (!first && frame->ip == last_ip) { derrorln("repeating stack trace"); break; } - last_rip = frame->rip; - frame = frame->rbp; + last_ip = frame->ip; + frame = frame->bp; first = false; } BAN::Formatter::print(Debug::putchar, "\e[m"); diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 9bb7431bf7..721759bebb 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -390,7 +390,7 @@ namespace Kernel return TRY(LibELF::LoadableELF::load_from_inode(page_table, file.inode)); } - BAN::ErrorOr Process::sys_fork(uintptr_t rsp, uintptr_t rip) + BAN::ErrorOr Process::sys_fork(uintptr_t sp, uintptr_t ip) { auto page_table = BAN::UniqPtr::adopt(TRY(PageTable::create_userspace())); @@ -423,7 +423,7 @@ namespace Kernel ASSERT(this == &Process::current()); // FIXME: this should be able to fail - Thread* thread = MUST(Thread::current().clone(forked, rsp, rip)); + Thread* thread = MUST(Thread::current().clone(forked, sp, ip)); forked->add_thread(thread); forked->register_to_scheduler(); diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 8df08cf889..4a64eae682 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -11,8 +11,8 @@ namespace Kernel { - extern "C" [[noreturn]] void start_thread(uintptr_t rsp, uintptr_t rip); - extern "C" [[noreturn]] void continue_thread(uintptr_t rsp, uintptr_t rip); + extern "C" [[noreturn]] void start_thread(uintptr_t sp, uintptr_t ip); + extern "C" [[noreturn]] void continue_thread(uintptr_t sp, uintptr_t ip); static Scheduler* s_instance = nullptr; static BAN::Atomic s_started { false }; @@ -144,18 +144,18 @@ namespace Kernel { ASSERT(m_lock.current_processor_has_lock()); - uintptr_t rsp, rip; + uintptr_t sp, ip; push_callee_saved(); - if (!(rip = read_rip())) + if (!(ip = read_ip())) { pop_callee_saved(); return true; } - read_rsp(rsp); + read_rsp(sp); Thread& current = current_thread(); - current.set_rip(rip); - current.set_rsp(rsp); + current.set_ip(ip); + current.set_sp(sp); load_temp_stack(); @@ -261,12 +261,12 @@ namespace Kernel case Thread::State::NotStarted: current->set_started(); m_lock.unlock(InterruptState::Disabled); - start_thread(current->rsp(), current->rip()); + start_thread(current->sp(), current->ip()); case Thread::State::Executing: m_lock.unlock(InterruptState::Disabled); while (current->can_add_signal_to_execute()) current->handle_signal(); - continue_thread(current->rsp(), current->rip()); + continue_thread(current->sp(), current->ip()); case Thread::State::Terminated: ASSERT_NOT_REACHED(); } diff --git a/kernel/kernel/Syscall.cpp b/kernel/kernel/Syscall.cpp index ff167e1b7f..c941bb4db8 100644 --- a/kernel/kernel/Syscall.cpp +++ b/kernel/kernel/Syscall.cpp @@ -10,9 +10,9 @@ namespace Kernel { - extern "C" long sys_fork(uintptr_t rsp, uintptr_t rip) + extern "C" long sys_fork(uintptr_t sp, uintptr_t ip) { - auto ret = Process::current().sys_fork(rsp, rip); + auto ret = Process::current().sys_fork(sp, ip); if (ret.is_error()) return -ret.error().get_error_code(); return ret.value(); @@ -32,8 +32,8 @@ namespace Kernel { ASSERT((interrupt_stack.cs & 0b11) == 0b11); - Thread::current().set_return_rsp(interrupt_stack.rsp); - Thread::current().set_return_rip(interrupt_stack.rip); + Thread::current().set_return_sp(interrupt_stack.sp); + Thread::current().set_return_ip(interrupt_stack.ip); asm volatile("sti"); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 3552088b2d..4f61e15487 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -12,8 +12,8 @@ namespace Kernel { - extern "C" void thread_userspace_trampoline(uint64_t rsp, uint64_t rip, int argc, char** argv, char** envp); - extern "C" uintptr_t read_rip(); + extern "C" void thread_userspace_trampoline(uint64_t sp, uint64_t ip, int argc, char** argv, char** envp); + extern "C" uintptr_t read_ip(); extern "C" void signal_trampoline(); @@ -46,14 +46,14 @@ namespace Kernel PageTable::Flags::ReadWrite | PageTable::Flags::Present, true )); - thread->m_rsp = thread->stack_base() + thread->stack_size(); - thread->m_rip = (uintptr_t)entry; + thread->m_sp = thread->stack_base() + thread->stack_size(); + thread->m_ip = (uintptr_t)entry; // Initialize stack for returning - write_to_stack(thread->m_rsp, nullptr); // alignment - write_to_stack(thread->m_rsp, thread); - write_to_stack(thread->m_rsp, &Thread::on_exit); - write_to_stack(thread->m_rsp, data); + write_to_stack(thread->m_sp, nullptr); // alignment + write_to_stack(thread->m_sp, thread); + write_to_stack(thread->m_sp, &Thread::on_exit); + write_to_stack(thread->m_sp, data); thread_deleter.disable(); @@ -144,7 +144,7 @@ namespace Kernel { } - BAN::ErrorOr Thread::clone(Process* new_process, uintptr_t rsp, uintptr_t rip) + BAN::ErrorOr Thread::clone(Process* new_process, uintptr_t sp, uintptr_t ip) { ASSERT(m_is_userspace); ASSERT(m_state == State::Executing); @@ -161,8 +161,8 @@ namespace Kernel thread->m_state = State::Executing; - thread->m_rip = rip; - thread->m_rsp = rsp; + thread->m_ip = ip; + thread->m_sp = sp; thread_deleter.disable(); @@ -177,24 +177,24 @@ namespace Kernel [](void*) { const auto& info = Process::current().userspace_info(); - thread_userspace_trampoline(Thread::current().rsp(), info.entry, info.argc, info.argv, info.envp); + thread_userspace_trampoline(Thread::current().sp(), info.entry, info.argc, info.argv, info.envp); ASSERT_NOT_REACHED(); } ); - m_rsp = stack_base() + stack_size(); - m_rip = (uintptr_t)entry_trampoline; + m_sp = stack_base() + stack_size(); + m_ip = (uintptr_t)entry_trampoline; // Signal mask is inherited // Setup stack for returning - ASSERT(m_rsp % PAGE_SIZE == 0); - PageTable::with_fast_page(process().page_table().physical_address_of(m_rsp - PAGE_SIZE), [&] { - uintptr_t rsp = PageTable::fast_page() + PAGE_SIZE; - write_to_stack(rsp, nullptr); // alignment - write_to_stack(rsp, this); - write_to_stack(rsp, &Thread::on_exit); - write_to_stack(rsp, nullptr); - m_rsp -= 4 * sizeof(uintptr_t); + ASSERT(m_sp % PAGE_SIZE == 0); + PageTable::with_fast_page(process().page_table().physical_address_of(m_sp - PAGE_SIZE), [&] { + uintptr_t sp = PageTable::fast_page() + PAGE_SIZE; + write_to_stack(sp, nullptr); // alignment + write_to_stack(sp, this); + write_to_stack(sp, &Thread::on_exit); + write_to_stack(sp, nullptr); + m_sp -= 4 * sizeof(uintptr_t); }); } @@ -210,20 +210,20 @@ namespace Kernel ASSERT_NOT_REACHED(); } ); - m_rsp = stack_base() + stack_size(); - m_rip = (uintptr_t)entry; + m_sp = stack_base() + stack_size(); + m_ip = (uintptr_t)entry; m_signal_pending_mask = 0; m_signal_block_mask = ~0ull; - ASSERT(m_rsp % PAGE_SIZE == 0); - PageTable::with_fast_page(process().page_table().physical_address_of(m_rsp - PAGE_SIZE), [&] { - uintptr_t rsp = PageTable::fast_page() + PAGE_SIZE; - write_to_stack(rsp, nullptr); // alignment - write_to_stack(rsp, this); - write_to_stack(rsp, &Thread::on_exit); - write_to_stack(rsp, m_process); - m_rsp -= 4 * sizeof(uintptr_t); + ASSERT(m_sp % PAGE_SIZE == 0); + PageTable::with_fast_page(process().page_table().physical_address_of(m_sp - PAGE_SIZE), [&] { + uintptr_t sp = PageTable::fast_page() + PAGE_SIZE; + write_to_stack(sp, nullptr); // alignment + write_to_stack(sp, this); + write_to_stack(sp, &Thread::on_exit); + write_to_stack(sp, m_process); + m_sp -= 4 * sizeof(uintptr_t); }); } @@ -250,7 +250,7 @@ namespace Kernel if (!is_userspace() || m_state != State::Executing) return false; auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); - return interrupt_stack.rip == (uintptr_t)signal_trampoline; + return interrupt_stack.ip == (uintptr_t)signal_trampoline; } void Thread::handle_signal(int signal) @@ -290,11 +290,11 @@ namespace Kernel else if (signal_handler != (vaddr_t)SIG_DFL) { // call userspace signal handlers - interrupt_stack.rsp -= 128; // skip possible red-zone - write_to_stack(interrupt_stack.rsp, interrupt_stack.rip); - write_to_stack(interrupt_stack.rsp, signal); - write_to_stack(interrupt_stack.rsp, signal_handler); - interrupt_stack.rip = (uintptr_t)signal_trampoline; + interrupt_stack.sp -= 128; // skip possible red-zone + write_to_stack(interrupt_stack.sp, interrupt_stack.ip); + write_to_stack(interrupt_stack.sp, signal); + write_to_stack(interrupt_stack.sp, signal_handler); + interrupt_stack.ip = (uintptr_t)signal_trampoline; } else { @@ -392,11 +392,11 @@ namespace Kernel void Thread::validate_stack() const { - if (stack_base() <= m_rsp && m_rsp <= stack_base() + stack_size()) + if (stack_base() <= m_sp && m_sp <= stack_base() + stack_size()) return; - if (interrupt_stack_base() <= m_rsp && m_rsp <= interrupt_stack_base() + interrupt_stack_size()) + if (interrupt_stack_base() <= m_sp && m_sp <= interrupt_stack_base() + interrupt_stack_size()) return; - Kernel::panic("rsp {8H}, stack {8H}->{8H}, interrupt_stack {8H}->{8H}", m_rsp, + Kernel::panic("sp {8H}, stack {8H}->{8H}, interrupt_stack {8H}->{8H}", m_sp, stack_base(), stack_base() + stack_size(), interrupt_stack_base(), interrupt_stack_base() + interrupt_stack_size() ); From 1ab2722850867339c3d26ba48d7112d8d9c35834 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 15:13:33 +0200 Subject: [PATCH 15/46] Kernel: Add PageTable stub to progress linking --- kernel/CMakeLists.txt | 1 + kernel/arch/i386/PageTable.cpp | 144 +++++++++++++++++++++++++++++++++ 2 files changed, 145 insertions(+) create mode 100644 kernel/arch/i386/PageTable.cpp diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 6cd5e41fd8..bb58d1b6c9 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -124,6 +124,7 @@ elseif("${BANAN_ARCH}" STREQUAL "i386") set(KERNEL_SOURCES ${KERNEL_SOURCES} arch/i386/boot.S + arch/i386/PageTable.cpp arch/i386/SpinLock.S arch/i386/Syscall.S arch/i386/Thread.S diff --git a/kernel/arch/i386/PageTable.cpp b/kernel/arch/i386/PageTable.cpp new file mode 100644 index 0000000000..fe28366c3a --- /dev/null +++ b/kernel/arch/i386/PageTable.cpp @@ -0,0 +1,144 @@ +#include +#include + +namespace Kernel +{ + + RecursiveSpinLock PageTable::s_fast_page_lock; + + void PageTable::initialize() + { + ASSERT_NOT_REACHED(); + } + + PageTable& PageTable::kernel() + { + ASSERT_NOT_REACHED(); + } + + bool PageTable::is_valid_pointer(uintptr_t) + { + ASSERT_NOT_REACHED(); + } + + BAN::ErrorOr PageTable::create_userspace() + { + ASSERT_NOT_REACHED(); + } + + PageTable::~PageTable() + { + ASSERT_NOT_REACHED(); + } + + void PageTable::unmap_page(vaddr_t) + { + ASSERT_NOT_REACHED(); + } + + void PageTable::unmap_range(vaddr_t, size_t) + { + ASSERT_NOT_REACHED(); + } + + void PageTable::map_range_at(paddr_t, vaddr_t, size_t, flags_t) + { + ASSERT_NOT_REACHED(); + } + + void PageTable::map_page_at(paddr_t, vaddr_t, flags_t) + { + ASSERT_NOT_REACHED(); + } + + paddr_t PageTable::physical_address_of(vaddr_t) const + { + ASSERT_NOT_REACHED(); + } + + PageTable::flags_t PageTable::get_page_flags(vaddr_t) const + { + ASSERT_NOT_REACHED(); + } + + bool PageTable::is_page_free(vaddr_t) const + { + ASSERT_NOT_REACHED(); + } + + bool PageTable::is_range_free(vaddr_t, size_t) const + { + ASSERT_NOT_REACHED(); + } + + bool PageTable::reserve_page(vaddr_t, bool) + { + ASSERT_NOT_REACHED(); + } + + bool PageTable::reserve_range(vaddr_t, size_t, bool) + { + ASSERT_NOT_REACHED(); + } + + vaddr_t PageTable::reserve_free_page(vaddr_t, vaddr_t) + { + ASSERT_NOT_REACHED(); + } + + vaddr_t PageTable::reserve_free_contiguous_pages(size_t, vaddr_t, vaddr_t) + { + ASSERT_NOT_REACHED(); + } + + void PageTable::load() + { + ASSERT_NOT_REACHED(); + } + + void PageTable::initial_load() + { + ASSERT_NOT_REACHED(); + } + + void PageTable::debug_dump() + { + ASSERT_NOT_REACHED(); + } + + uint64_t PageTable::get_page_data(vaddr_t) const + { + ASSERT_NOT_REACHED(); + } + + void PageTable::initialize_kernel() + { + ASSERT_NOT_REACHED(); + } + + void PageTable::map_kernel_memory() + { + ASSERT_NOT_REACHED(); + } + + void PageTable::prepare_fast_page() + { + ASSERT_NOT_REACHED(); + } + + void PageTable::invalidate(vaddr_t) + { + ASSERT_NOT_REACHED(); + } + + void PageTable::map_fast_page(paddr_t) + { + ASSERT_NOT_REACHED(); + } + + void PageTable::unmap_fast_page() + { + ASSERT_NOT_REACHED(); + } + +} From 0d92719433beac95897611686819b854f005401b Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 15:16:37 +0200 Subject: [PATCH 16/46] Kernel: Remove old i386 spinlock code --- kernel/CMakeLists.txt | 1 - kernel/arch/i386/SpinLock.S | 19 ------------------- 2 files changed, 20 deletions(-) delete mode 100644 kernel/arch/i386/SpinLock.S diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index bb58d1b6c9..2bbe58780b 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -125,7 +125,6 @@ elseif("${BANAN_ARCH}" STREQUAL "i386") ${KERNEL_SOURCES} arch/i386/boot.S arch/i386/PageTable.cpp - arch/i386/SpinLock.S arch/i386/Syscall.S arch/i386/Thread.S ) diff --git a/kernel/arch/i386/SpinLock.S b/kernel/arch/i386/SpinLock.S deleted file mode 100644 index 1b139d6a8f..0000000000 --- a/kernel/arch/i386/SpinLock.S +++ /dev/null @@ -1,19 +0,0 @@ -.global spinlock_lock_asm -spinlock_lock_asm: - movl 4(%esp), %eax - lock; btsl $0, (%eax) - jnc .done -.retry: - pause - testl $1, (%eax) - jne .retry - lock; btsl $0, (%eax) - jc .retry -.done: - ret - -.global spinlock_unlock_asm -spinlock_unlock_asm: - movl 4(%esp), %eax - movl $0, (%eax) - ret From 26585bb1d9cf66c05deef714aaba90c8a2bd3293 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 22 Mar 2024 15:33:39 +0200 Subject: [PATCH 17/46] Kernel: Implement signal trampoline for i386 --- kernel/CMakeLists.txt | 1 + kernel/arch/i386/Signal.S | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 kernel/arch/i386/Signal.S diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 2bbe58780b..deab4d0ecd 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -125,6 +125,7 @@ elseif("${BANAN_ARCH}" STREQUAL "i386") ${KERNEL_SOURCES} arch/i386/boot.S arch/i386/PageTable.cpp + arch/i386/Signal.S arch/i386/Syscall.S arch/i386/Thread.S ) diff --git a/kernel/arch/i386/Signal.S b/kernel/arch/i386/Signal.S new file mode 100644 index 0000000000..41a3b834a7 --- /dev/null +++ b/kernel/arch/i386/Signal.S @@ -0,0 +1,29 @@ +.section .userspace, "aw" + +// stack contains +// return address +// signal number +// signal handler + +.global signal_trampoline +signal_trampoline: + pushl %ebp + movl %esp, %ebp + subl $8, %esp + + pusha + + movl 40(%esp), %edi + movl 36(%esp), %eax + + subl $12, %esp + pushl %edi + call *%eax + addl $16, %esp + + popa + + leave + addl $8, %esp + + ret From 2dd0bfdece34254a938cab8e410b26b3c201a5ea Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 00:07:24 +0200 Subject: [PATCH 18/46] Kernel: Make i386 thread tramplines crash --- kernel/arch/i386/Thread.S | 39 ++++++------------------------------- kernel/arch/x86_64/Thread.S | 2 +- 2 files changed, 7 insertions(+), 34 deletions(-) diff --git a/kernel/arch/i386/Thread.S b/kernel/arch/i386/Thread.S index 30e8811e8f..d0b6a04f90 100644 --- a/kernel/arch/i386/Thread.S +++ b/kernel/arch/i386/Thread.S @@ -4,44 +4,17 @@ read_ip: popl %eax jmp *%eax -exit_thread_trampoline: - addl $4, %esp - pushl (%esp) - ret - # void start_thread(uint32_t sp, uint32_t ip) .global start_thread start_thread: - movl 8(%esp), %ecx - movl 4(%esp), %esp - movl $0, %ebp - pushl $exit_thread_trampoline - sti - jmp *%ecx + ud2 # void continue_thread(uint32_t sp, uint32_t ip) .global continue_thread continue_thread: - movl 8(%esp), %ecx - movl 4(%esp), %esp - movl $0, %eax - jmp *%ecx + ud2 -# void thread_jump_userspace(uint32_t sp, uint32_t ip) -.global thread_jump_userspace -thread_jump_userspace: - movl $0x23, %eax - movw %ax, %ds - movw %ax, %es - movw %ax, %fs - movw %ax, %gs - - movl 8(%esp), %ecx - movl 4(%esp), %esp - - pushl $0x23 - pushl %esp - pushfl - pushl $0x1B - pushl %ecx - iret +# void thread_userspace_trampoline(uint32_t sp, uint32_t ip, int argc, char** argv, char** envp) +.global thread_userspace_trampoline +thread_userspace_trampoline: + ud2 diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index 62973ada37..de1e59ab52 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -1,4 +1,4 @@ -# uint64_t read_() +# uint64_t read_ip() .global read_ip read_ip: popq %rax From 097d9a6479e1f6eb6b6ef71432bdec94e4c0f357 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 00:08:11 +0200 Subject: [PATCH 19/46] Kernel: Implement dummy IDT and GDT for i386 --- kernel/CMakeLists.txt | 2 ++ kernel/arch/i386/GDT.cpp | 21 +++++++++++++++++++++ kernel/arch/i386/IDT.cpp | 31 +++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 kernel/arch/i386/GDT.cpp create mode 100644 kernel/arch/i386/IDT.cpp diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index deab4d0ecd..2fb59ab17d 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -124,6 +124,8 @@ elseif("${BANAN_ARCH}" STREQUAL "i386") set(KERNEL_SOURCES ${KERNEL_SOURCES} arch/i386/boot.S + arch/i386/GDT.cpp + arch/i386/IDT.cpp arch/i386/PageTable.cpp arch/i386/Signal.S arch/i386/Syscall.S diff --git a/kernel/arch/i386/GDT.cpp b/kernel/arch/i386/GDT.cpp new file mode 100644 index 0000000000..f76d47f025 --- /dev/null +++ b/kernel/arch/i386/GDT.cpp @@ -0,0 +1,21 @@ +#include + +namespace Kernel +{ + + GDT* GDT::create() + { + ASSERT_NOT_REACHED(); + } + + void GDT::write_entry(uint8_t, uint32_t, uint32_t, uint8_t, uint8_t) + { + ASSERT_NOT_REACHED(); + } + + void GDT::write_tss() + { + ASSERT_NOT_REACHED(); + } + +} diff --git a/kernel/arch/i386/IDT.cpp b/kernel/arch/i386/IDT.cpp new file mode 100644 index 0000000000..b11241fda7 --- /dev/null +++ b/kernel/arch/i386/IDT.cpp @@ -0,0 +1,31 @@ +#include + +namespace Kernel +{ + + IDT* IDT::create() + { + ASSERT_NOT_REACHED(); + } + + [[noreturn]] void IDT::force_triple_fault() + { + ASSERT_NOT_REACHED(); + } + + void IDT::register_irq_handler(uint8_t, Interruptable*) + { + ASSERT_NOT_REACHED(); + } + + void IDT::register_interrupt_handler(uint8_t, void (*)()) + { + ASSERT_NOT_REACHED(); + } + + void IDT::register_syscall_handler(uint8_t, void (*)()) + { + ASSERT_NOT_REACHED(); + } + +} From fc7e96fa6646e1c855df911e96d7baed3be32e24 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 00:09:05 +0200 Subject: [PATCH 20/46] Kernel: Rewrite i386 boot code + linker script --- kernel/arch/i386/boot.S | 288 ++++++++++++++++++++++++------------- kernel/arch/i386/linker.ld | 31 +++- 2 files changed, 211 insertions(+), 108 deletions(-) diff --git a/kernel/arch/i386/boot.S b/kernel/arch/i386/boot.S index da553e33c9..ea69fff792 100644 --- a/kernel/arch/i386/boot.S +++ b/kernel/arch/i386/boot.S @@ -1,61 +1,98 @@ -# Declare constants for the multiboot header -.set ALIGN, 1<<0 # align loaded modules on page boundaries -.set MEMINFO, 1<<1 # provide memory map -.set VIDEOINFO, 1<<2 # provide video info -.set MB_FLAGS, ALIGN | MEMINFO | VIDEOINFO # this is the Multiboot 'flag' field -.set MB_MAGIC, 0x1BADB002 # 'magic number' lets bootloader find the header -.set MB_CHECKSUM, -(MB_MAGIC + MB_FLAGS) #checksum of above, to prove we are multiboot +.set PG_PRESENT, 1<<0 +.set PG_READ_WRITE, 1<<1 +.set PG_PAGE_SIZE, 1<<7 -# Multiboot header +.set FB_WIDTH, 800 +.set FB_HEIGHT, 600 +.set FB_BPP, 32 + +#define KERNEL_OFFSET 0xC0000000 +#define V2P(vaddr) ((vaddr) - KERNEL_OFFSET) + +.code32 + +# multiboot2 header .section .multiboot, "aw" - .align 4 - .long MB_MAGIC - .long MB_FLAGS - .long MB_CHECKSUM - .skip 20 - + .align 8 +multiboot2_start: + .long 0xE85250D6 .long 0 - .long 800 - .long 600 - .long 32 + .long multiboot2_end - multiboot2_start + .long -(0xE85250D6 + (multiboot2_end - multiboot2_start)) + + # framebuffer tag + .align 8 + .short 5 + .short 0 + .long 20 + .long FB_WIDTH + .long FB_HEIGHT + .long FB_BPP + + # legacy start + .align 8 + .short 3 + .short 0 + .long 12 + .long V2P(_start) + + .align 8 + .short 0 + .short 0 + .long 8 +multiboot2_end: + +.section .bananboot, "aw" + .align 8 +bananboot_start: + .long 0xBABAB007 + .long -(0xBABAB007 + FB_WIDTH + FB_HEIGHT + FB_BPP) + .long FB_WIDTH + .long FB_HEIGHT + .long FB_BPP +bananboot_end: .section .bss, "aw", @nobits -# Create stack -.global g_boot_stack_bottom -g_boot_stack_bottom: - .skip 16384 -.global g_boot_stack_top -g_boot_stack_top: + boot_stack_bottom: + .skip 4096 * 4 + boot_stack_top: + + .global g_kernel_cmdline + g_kernel_cmdline: + .skip 4096 + + bootloader_magic: + .skip 8 + bootloader_info: + .skip 8 + +.section .data -# 0 MiB -> 1 MiB: bootloader stuff -# 1 MiB -> : kernel -.align 32 -boot_page_directory_pointer_table: - .skip 4 * 8 .align 4096 -boot_page_directory1: - .skip 512 * 8 - -.global g_kernel_cmdline -g_kernel_cmdline: - .skip 4096 - -.global g_multiboot_info -g_multiboot_info: - .skip 4 -.global g_multiboot_magic -g_multiboot_magic: - .skip 4 - -.section .text +boot_pml4: +boot_pdpt_lo: +boot_pdpt_hi: +boot_pd: boot_gdt: - .quad 0x0000000000000000 # null + .quad 0x0000000000000000 # null descriptor .quad 0x00CF9A000000FFFF # kernel code .quad 0x00CF92000000FFFF # kernel data boot_gdtr: .short . - boot_gdt - 1 - .long boot_gdt + .long V2P(boot_gdt) + +.global g_ap_startup_done +g_ap_startup_done: + .byte 0 +.global g_ap_running_count +g_ap_running_count: + .byte 0 +.global g_ap_stack_loaded +g_ap_stack_loaded: + .byte 0 + +.section .text has_cpuid: pushfl @@ -69,42 +106,13 @@ has_cpuid: testl $0x00200000, %eax ret -has_pae: - movl $0, %eax - cpuid - testl $(1 << 6), %edx - ret - -has_sse: - movl $1, %eax - cpuid - testl $(1 << 25), %edx - ret - check_requirements: call has_cpuid jz .exit - call has_pae - jz .exit - call has_sse - jz .exit ret .exit: jmp system_halt -copy_kernel_commandline: - pushl %esi - pushl %edi - movl g_multiboot_info, %esi - addl $16, %esi - movl (%esi), %esi - movl $1024, %ecx - movl $g_kernel_cmdline, %edi - rep movsl - popl %edi - popl %esi - ret - enable_sse: movl %cr0, %eax andw $0xFFFB, %ax @@ -116,19 +124,13 @@ enable_sse: ret initialize_paging: - # identity map first 6 MiB - movl $(0x00000000 + 0x83), boot_page_directory1 + 0 - movl $(0x00200000 + 0x83), boot_page_directory1 + 8 - movl $(0x00400000 + 0x83), boot_page_directory1 + 16 - movl $(boot_page_directory1 + 0x01), boot_page_directory_pointer_table - # enable PAE movl %cr4, %ecx orl $0x20, %ecx movl %ecx, %cr4 # set address of paging structures - movl $boot_page_directory_pointer_table, %ecx + movl $V2P(boot_pml4), %ecx movl %ecx, %cr3 # enable paging @@ -138,41 +140,52 @@ initialize_paging: ret -initialize_gdt: - lgdt boot_gdtr - - # flush gdt - movw $0x10, %ax - movw %ax, %ds - movw %ax, %es - movw %ax, %fs - movw %ax, %gs - movw %ax, %ss - jmp $0x08, $flush -flush: - ret - .global _start .type _start, @function _start: - # Initialize stack and multiboot info - movl $g_boot_stack_top, %esp - movl %eax, g_multiboot_magic - movl %ebx, g_multiboot_info + cli; cld + + # Initialize stack and multiboot info + movl %eax, V2P(bootloader_magic) + movl %ebx, V2P(bootloader_info) + + movl $V2P(boot_stack_top), %esp - call copy_kernel_commandline call check_requirements call enable_sse call initialize_paging - call initialize_gdt + # flush gdt + lgdt V2P(boot_gdtr) + ljmpl $0x08, $V2P(gdt_flush) + +gdt_flush: + movw $0x10, %ax + movw %ax, %ds + movw %ax, %ss + movw %ax, %es + + # move stack pointer to higher half + movl %esp, %esp + addl $KERNEL_OFFSET, %esp + + # jump to higher half + leal higher_half, %ecx + jmp *%ecx + +higher_half: + # call global constuctors call _init # call to the kernel itself (clear ebp for stacktrace) xorl %ebp, %ebp + + movl V2P(bootloader_magic), %edi + movl V2P(bootloader_info), %esi call kernel_main + # call global destructors call _fini system_halt: @@ -180,3 +193,76 @@ system_halt: cli 1: hlt jmp 1b + + +.section .ap_init, "ax" + +.code16 +.global ap_trampoline +ap_trampoline: + jmp 1f +.align 8 +ap_stack_ptr: + .skip 4 +1: + cli; cld + ljmpl $0x00, $ap_cs_clear + +ap_cs_clear: + xorw %ax, %ax + movw %ax, %ds + + # load ap gdt and enter protected mode + lgdt ap_gdtr + movl %cr0, %eax + orb $1, %al + movl %eax, %cr0 + ljmpl $0x08, $ap_protected_mode + +.code32 +ap_protected_mode: + movw $0x10, %ax + movw %ax, %ds + movw %ax, %ss + movw %ax, %es + + movl ap_stack_ptr, %esp + movb $1, V2P(g_ap_stack_loaded) + + call V2P(enable_sse) + + call V2P(initialize_paging) + + # load boot gdt and enter long mode + lgdt V2P(boot_gdtr) + ljmpl $0x08, $ap_flush_gdt + +ap_flush_gdt: + # move stack pointer to higher half + movl %esp, %esp + addl $KERNEL_OFFSET, %esp + + # jump to higher half + leal ap_higher_half, %ecx + jmp *%ecx + +ap_higher_half: + # clear rbp for stacktrace + xorl %ebp, %ebp + +1: pause + cmpb $0, g_ap_startup_done + jz 1b + + lock incb g_ap_running_count + + call ap_main + jmp system_halt + +ap_gdt: + .quad 0x0000000000000000 # null descriptor + .quad 0x00CF9A000000FFFF # 32 bit code + .quad 0x00CF92000000FFFF # 32 bit data +ap_gdtr: + .short . - ap_gdt - 1 + .long ap_gdt diff --git a/kernel/arch/i386/linker.ld b/kernel/arch/i386/linker.ld index 8a5fd7a9c9..a8e042d997 100644 --- a/kernel/arch/i386/linker.ld +++ b/kernel/arch/i386/linker.ld @@ -1,28 +1,45 @@ ENTRY (_start) +KERNEL_OFFSET = 0xC0000000; + SECTIONS { - . = 0x00100000; + . = 0xF000; + .ap_init ALIGN(4K) : AT(ADDR(.ap_init)) + { + g_ap_init_addr = .; + *(.ap_init) + } + + . = 0x00100000 + KERNEL_OFFSET; g_kernel_start = .; - .text BLOCK(4K) : ALIGN(4K) + .text ALIGN(4K) : AT(ADDR(.text) - KERNEL_OFFSET) { + g_kernel_execute_start = .; *(.multiboot) - *(.text) + *(.bananboot) + *(.text.*) } - .rodata BLOCK(4K) : ALIGN(4K) + .userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET) + { + g_userspace_start = .; + *(.userspace) + g_userspace_end = .; + g_kernel_execute_end = .; + } + .rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET) { *(.rodata.*) } - .data BLOCK(4K) : ALIGN(4K) + .data ALIGN(4K) : AT(ADDR(.data) - KERNEL_OFFSET) { *(.data) } - .bss BLOCK(4K) : ALIGN(4K) + .bss ALIGN(4K) : AT(ADDR(.bss) - KERNEL_OFFSET) { *(COMMON) *(.bss) } - g_kernel_end = .; } From a9db4dd9a3295520dcdcc458e91c6a8d7a8d2183 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 01:45:43 +0200 Subject: [PATCH 21/46] Kernel: NVMe Queue max simultaneous commands is dependent on arch This allows mask to be atomic on 32 bit architectures --- kernel/include/kernel/Storage/NVMe/Queue.h | 12 +++++++----- kernel/kernel/Storage/NVMe/Queue.cpp | 18 +++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/kernel/include/kernel/Storage/NVMe/Queue.h b/kernel/include/kernel/Storage/NVMe/Queue.h index f174ed157f..8cbe97d0ad 100644 --- a/kernel/include/kernel/Storage/NVMe/Queue.h +++ b/kernel/include/kernel/Storage/NVMe/Queue.h @@ -31,11 +31,13 @@ namespace Kernel uint32_t m_cq_head { 0 }; uint16_t m_cq_valid_phase { 1 }; - Semaphore m_semaphore; - SpinLock m_lock; - BAN::Atomic m_used_mask { 0 }; - BAN::Atomic m_done_mask { 0 }; - volatile uint16_t m_status_codes[64] { }; + Semaphore m_semaphore; + SpinLock m_lock; + BAN::Atomic m_used_mask { 0 }; + BAN::Atomic m_done_mask { 0 }; + volatile uint16_t m_status_codes[64] { }; + + static constexpr size_t m_mask_bits = sizeof(size_t) * 8; }; } diff --git a/kernel/kernel/Storage/NVMe/Queue.cpp b/kernel/kernel/Storage/NVMe/Queue.cpp index 68b72b7a4f..63b48d0187 100644 --- a/kernel/kernel/Storage/NVMe/Queue.cpp +++ b/kernel/kernel/Storage/NVMe/Queue.cpp @@ -15,8 +15,8 @@ namespace Kernel , m_doorbell(db) , m_qdepth(qdepth) { - for (uint32_t i = qdepth; i < 64; i++) - m_used_mask |= (uint64_t)1 << i; + for (uint32_t i = qdepth; i < m_mask_bits; i++) + m_used_mask |= (size_t)1 << i; set_irq(irq); enable_interrupt(); } @@ -29,8 +29,8 @@ namespace Kernel { uint16_t sts = cq_ptr[m_cq_head].sts >> 1; uint16_t cid = cq_ptr[m_cq_head].cid; - uint64_t cid_mask = (uint64_t)1 << cid; - ASSERT(cid < 64); + size_t cid_mask = (size_t)1 << cid; + ASSERT(cid < m_mask_bits); ASSERT((m_done_mask & cid_mask) == 0); @@ -50,7 +50,7 @@ namespace Kernel uint16_t NVMeQueue::submit_command(NVMe::SubmissionQueueEntry& sqe) { uint16_t cid = reserve_cid(); - uint64_t cid_mask = (uint64_t)1 << cid; + size_t cid_mask = (size_t)1 << cid; { SpinLockGuard _(m_lock); @@ -98,13 +98,13 @@ namespace Kernel } uint16_t cid = 0; - for (; cid < 64; cid++) - if ((m_used_mask & ((uint64_t)1 << cid)) == 0) + for (; cid < m_mask_bits; cid++) + if ((m_used_mask & ((size_t)1 << cid)) == 0) break; - ASSERT(cid < 64); + ASSERT(cid < m_mask_bits); ASSERT(cid < m_qdepth); - m_used_mask |= (uint64_t)1 << cid; + m_used_mask |= (size_t)1 << cid; m_lock.unlock(state); return cid; From fbef90f7cba9787f87acca1b6c1bc801a48825f2 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 02:27:38 +0200 Subject: [PATCH 22/46] Kernel/LibC: Write cxx abi with proper locking --- kernel/icxxabi.cpp | 134 ++++++++++----------------------------------- libc/icxxabi.cpp | 40 ++++++++------ 2 files changed, 50 insertions(+), 124 deletions(-) diff --git a/kernel/icxxabi.cpp b/kernel/icxxabi.cpp index 175318e121..a1f334db9f 100644 --- a/kernel/icxxabi.cpp +++ b/kernel/icxxabi.cpp @@ -1,115 +1,41 @@ -#include #include #define ATEXIT_MAX_FUNCS 128 -#ifdef __cplusplus -extern "C" { -#endif - -typedef unsigned uarch_t; - struct atexit_func_entry_t { - /* - * Each member is at least 4 bytes large. Such that each entry is 12bytes. - * 128 * 12 = 1.5KB exact. - **/ - void (*destructor_func)(void *); - void *obj_ptr; - void *dso_handle; + void(*func)(void*); + void* arg; + void* dso_handle; }; -atexit_func_entry_t __atexit_funcs[ATEXIT_MAX_FUNCS]; -uarch_t __atexit_func_count = 0; +static atexit_func_entry_t __atexit_funcs[ATEXIT_MAX_FUNCS]; +static size_t __atexit_func_count = 0; -int __cxa_atexit(void (*f)(void *), void *objptr, void *dso) +extern "C" int __cxa_atexit(void(*func)(void*), void* arg, void* dso_handle) { - if (__atexit_func_count >= ATEXIT_MAX_FUNCS) {return -1;}; - __atexit_funcs[__atexit_func_count].destructor_func = f; - __atexit_funcs[__atexit_func_count].obj_ptr = objptr; - __atexit_funcs[__atexit_func_count].dso_handle = dso; - __atexit_func_count++; - return 0; /*I would prefer if functions returned 1 on success, but the ABI says...*/ + if (__atexit_func_count >= ATEXIT_MAX_FUNCS) + return -1; + auto& atexit_func = __atexit_funcs[__atexit_func_count++]; + atexit_func.func = func; + atexit_func.arg = arg; + atexit_func.dso_handle = dso_handle; + return 0; }; -void __cxa_finalize(void *f) +extern "C" void __cxa_finalize(void* f) { - uarch_t i = __atexit_func_count; - if (!f) + for (size_t i = __atexit_func_count; i > 0; i--) { - /* - * According to the Itanium C++ ABI, if __cxa_finalize is called without a - * function ptr, then it means that we should destroy EVERYTHING MUAHAHAHA!! - * - * TODO: - * Note well, however, that deleting a function from here that contains a __dso_handle - * means that one link to a shared object file has been terminated. In other words, - * We should monitor this list (optional, of course), since it tells us how many links to - * an object file exist at runtime in a particular application. This can be used to tell - * when a shared object is no longer in use. It is one of many methods, however. - **/ - //You may insert a prinf() here to tell you whether or not the function gets called. Testing - //is CRITICAL! - while (i--) + auto& atexit_func = __atexit_funcs[i - 1]; + if (atexit_func.func == nullptr) + continue; + if (f == nullptr || f == atexit_func.func) { - if (__atexit_funcs[i].destructor_func) - { - /* ^^^ That if statement is a safeguard... - * To make sure we don't call any entries that have already been called and unset at runtime. - * Those will contain a value of 0, and calling a function with value 0 - * will cause undefined behaviour. Remember that linear address 0, - * in a non-virtual address space (physical) contains the IVT and BDA. - * - * In a virtual environment, the kernel will receive a page fault, and then probably - * map in some trash, or a blank page, or something stupid like that. - * This will result in the processor executing trash, and...we don't want that. - **/ - (*__atexit_funcs[i].destructor_func)(__atexit_funcs[i].obj_ptr); - }; - }; - return; - }; - - while (i--) - { - /* - * The ABI states that multiple calls to the __cxa_finalize(destructor_func_ptr) function - * should not destroy objects multiple times. Only one call is needed to eliminate multiple - * entries with the same address. - * - * FIXME: - * This presents the obvious problem: all destructors must be stored in the order they - * were placed in the list. I.e: the last initialized object's destructor must be first - * in the list of destructors to be called. But removing a destructor from the list at runtime - * creates holes in the table with unfilled entries. - * Remember that the insertion algorithm in __cxa_atexit simply inserts the next destructor - * at the end of the table. So, we have holes with our current algorithm - * This function should be modified to move all the destructors above the one currently - * being called and removed one place down in the list, so as to cover up the hole. - * Otherwise, whenever a destructor is called and removed, an entire space in the table is wasted. - **/ - if (__atexit_funcs[i].destructor_func == f) - { - /* - * Note that in the next line, not every destructor function is a class destructor. - * It is perfectly legal to register a non class destructor function as a simple cleanup - * function to be called on program termination, in which case, it would not NEED an - * object This pointer. A smart programmer may even take advantage of this and register - * a C function in the table with the address of some structure containing data about - * what to clean up on exit. - * In the case of a function that takes no arguments, it will simply be ignore within the - * function itself. No worries. - **/ - (*__atexit_funcs[i].destructor_func)(__atexit_funcs[i].obj_ptr); - __atexit_funcs[i].destructor_func = 0; - - /* - * Notice that we didn't decrement __atexit_func_count: this is because this algorithm - * requires patching to deal with the FIXME outlined above. - **/ - }; - }; + atexit_func.func(atexit_func.arg); + atexit_func.func = nullptr; + } + } }; namespace __cxxabiv1 @@ -118,23 +44,19 @@ namespace __cxxabiv1 int __cxa_guard_acquire (__guard* g) { - auto& atomic = *reinterpret_cast*>(g); - return atomic == 0; + uint8_t* byte = reinterpret_cast(g); + uint8_t zero = 0; + return __atomic_compare_exchange_n(byte, &zero, 1, false, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE); } void __cxa_guard_release (__guard* g) { - auto& atomic = *reinterpret_cast*>(g); - atomic = 1; + uint8_t* byte = reinterpret_cast(g); + __atomic_store_n(byte, 0, __ATOMIC_RELEASE); } void __cxa_guard_abort (__guard*) { Kernel::panic("__cxa_guard_abort"); - __builtin_unreachable(); } } - -#ifdef __cplusplus -}; -#endif diff --git a/libc/icxxabi.cpp b/libc/icxxabi.cpp index 5484137a77..8a91d1cacc 100644 --- a/libc/icxxabi.cpp +++ b/libc/icxxabi.cpp @@ -1,37 +1,41 @@ -#include +#include +#include +#include #define ATEXIT_MAX_FUNCS 128 struct atexit_func_entry_t { - void (*destructor)(void*); - void* data; + void(*func)(void*); + void* arg; void* dso_handle; }; static atexit_func_entry_t __atexit_funcs[ATEXIT_MAX_FUNCS]; -static int __atexit_func_count = 0; +static size_t __atexit_func_count = 0; -int __cxa_atexit(void (*func)(void*), void* data, void* dso_handle) +extern "C" int __cxa_atexit(void(*func)(void*), void* arg, void* dso_handle) { if (__atexit_func_count >= ATEXIT_MAX_FUNCS) - return -1;; - __atexit_funcs[__atexit_func_count].destructor = func; - __atexit_funcs[__atexit_func_count].data = data; - __atexit_funcs[__atexit_func_count].dso_handle = dso_handle; - __atexit_func_count++; + return -1; + auto& atexit_func = __atexit_funcs[__atexit_func_count++]; + atexit_func.func = func; + atexit_func.arg = arg; + atexit_func.dso_handle = dso_handle; return 0; }; -void __cxa_finalize(void* func) +extern "C" void __cxa_finalize(void* f) { - for (int i = __atexit_func_count - 1; i >= 0; i--) + for (size_t i = __atexit_func_count; i > 0; i--) { - if (func && func != __atexit_funcs[i].destructor) + auto& atexit_func = __atexit_funcs[i - 1]; + if (atexit_func.func == nullptr) continue; - if (__atexit_funcs[i].destructor == nullptr) - continue; - __atexit_funcs[i].destructor(__atexit_funcs[i].data); - __atexit_funcs[i].destructor = nullptr; + if (f == nullptr || f == atexit_func.func) + { + atexit_func.func(atexit_func.arg); + atexit_func.func = nullptr; + } } -} +}; From 93975fdc456cb1cce58e1dfbf618b7636359cb0b Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 02:46:51 +0200 Subject: [PATCH 23/46] Kernel: Process signal mask is now 2 32 bit values This allows signal mask to be atomic on 32 bit target --- kernel/include/kernel/Process.h | 30 +++++++++++++++++++++++++++++- kernel/include/kernel/Thread.h | 1 + kernel/kernel/Process.cpp | 4 ++-- kernel/kernel/Thread.cpp | 12 +++++++++--- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index 1927a23e27..056cc93c11 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -199,6 +199,33 @@ namespace Kernel BAN::ErrorOr validate_string_access(const char*); BAN::ErrorOr validate_pointer_access(const void*, size_t); + uint64_t signal_pending_mask() const + { + return ((uint64_t)m_signal_pending_mask[1].load() << 32) | m_signal_pending_mask[0].load(); + } + + void add_pending_signal(uint8_t signal) + { + ASSERT(signal >= _SIGMIN); + ASSERT(signal <= _SIGMAX); + ASSERT(signal < 64); + if (signal < 32) + m_signal_pending_mask[0] |= (uint32_t)1 << signal; + else + m_signal_pending_mask[1] |= (uint32_t)1 << (signal - 32); + } + + void remove_pending_signal(uint8_t signal) + { + ASSERT(signal >= _SIGMIN); + ASSERT(signal <= _SIGMAX); + ASSERT(signal < 64); + if (signal < 32) + m_signal_pending_mask[0] &= ~((uint32_t)1 << signal); + else + m_signal_pending_mask[1] &= ~((uint32_t)1 << (signal - 32)); + } + private: struct ExitStatus { @@ -226,7 +253,8 @@ namespace Kernel BAN::Vector m_threads; BAN::Atomic m_signal_handlers[_SIGMAX + 1] { }; - BAN::Atomic m_signal_pending_mask { 0 }; + // This is 2 32 bit values to allow atomicity on 32 targets + BAN::Atomic m_signal_pending_mask[2] { 0, 0 }; BAN::Vector m_cmdline; BAN::Vector m_environ; diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index bae255407e..5e66069b20 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -79,6 +79,7 @@ namespace Kernel static pid_t current_tid(); Process& process(); + const Process& process() const; bool has_process() const { return m_process; } bool is_userspace() const { return m_is_userspace; } diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 721759bebb..2d757a10c8 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -1488,7 +1488,7 @@ namespace Kernel if (pid == m_pid) { - m_signal_pending_mask |= 1 << signal; + add_pending_signal(signal); return 0; } @@ -1501,7 +1501,7 @@ namespace Kernel found = true; if (signal) { - process.m_signal_pending_mask |= 1 << signal; + process.add_pending_signal(signal); // FIXME: This feels hacky Scheduler::get().unblock_thread(process.m_threads.front()->tid()); } diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 4f61e15487..d757673f78 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -140,6 +140,12 @@ namespace Kernel return *m_process; } + const Process& Thread::process() const + { + ASSERT(m_process); + return *m_process; + } + Thread::~Thread() { } @@ -241,7 +247,7 @@ namespace Kernel auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); if (!GDT::is_user_segment(interrupt_stack.cs)) return false; - uint64_t full_pending_mask = m_signal_pending_mask | m_process->m_signal_pending_mask; + uint64_t full_pending_mask = m_signal_pending_mask | process().signal_pending_mask();; return full_pending_mask & ~m_signal_block_mask; } @@ -265,7 +271,7 @@ namespace Kernel if (signal == 0) { - uint64_t full_pending_mask = m_signal_pending_mask | process().m_signal_pending_mask; + uint64_t full_pending_mask = m_signal_pending_mask | process().signal_pending_mask(); for (signal = _SIGMIN; signal <= _SIGMAX; signal++) { uint64_t mask = 1ull << signal; @@ -283,7 +289,7 @@ namespace Kernel vaddr_t signal_handler = process().m_signal_handlers[signal]; m_signal_pending_mask &= ~(1ull << signal); - process().m_signal_pending_mask &= ~(1ull << signal); + process().remove_pending_signal(signal); if (signal_handler == (vaddr_t)SIG_IGN) ; From 99e30a4d7dc75e36d369ccbf1f821ec4f55770af Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 02:48:26 +0200 Subject: [PATCH 24/46] Kernel: Replace i386 with i686 I don't really want to be working with i386 since it doesn't support compare exchange instruction --- LibELF/LibELF/LoadableELF.cpp | 2 +- LibELF/include/LibELF/ELF.h | 2 +- LibELF/include/LibELF/Types.h | 2 +- kernel/CMakeLists.txt | 22 ++++++++++---------- kernel/arch/{i386 => i686}/GDT.cpp | 0 kernel/arch/{i386 => i686}/IDT.cpp | 0 kernel/arch/{i386 => i686}/PageTable.cpp | 0 kernel/arch/{i386 => i686}/Signal.S | 2 ++ kernel/arch/{i386 => i686}/Syscall.S | 1 + kernel/arch/{i386 => i686}/Thread.S | 0 kernel/arch/{i386 => i686}/boot.S | 0 kernel/arch/{i386 => i686}/crt0.S | 0 kernel/arch/{i386 => i686}/crti.S | 0 kernel/arch/{i386 => i686}/crtn.S | 0 kernel/arch/{i386 => i686}/linker.ld | 0 kernel/include/kernel/Arch.h | 8 ++++--- kernel/include/kernel/FS/TmpFS/Definitions.h | 2 +- kernel/include/kernel/Memory/Types.h | 2 +- kernel/include/kernel/Processor.h | 2 +- kernel/kernel/Random.cpp | 2 +- kernel/kernel/Scheduler.cpp | 4 ++-- kernel/kernel/Thread.cpp | 2 +- libc/arch/{i386 => i686}/crt0.S | 0 libc/arch/{i386 => i686}/crti.S | 2 +- libc/arch/{i386 => i686}/crtn.S | 2 +- 25 files changed, 31 insertions(+), 26 deletions(-) rename kernel/arch/{i386 => i686}/GDT.cpp (100%) rename kernel/arch/{i386 => i686}/IDT.cpp (100%) rename kernel/arch/{i386 => i686}/PageTable.cpp (100%) rename kernel/arch/{i386 => i686}/Signal.S (98%) rename kernel/arch/{i386 => i686}/Syscall.S (98%) rename kernel/arch/{i386 => i686}/Thread.S (100%) rename kernel/arch/{i386 => i686}/boot.S (100%) rename kernel/arch/{i386 => i686}/crt0.S (100%) rename kernel/arch/{i386 => i686}/crti.S (100%) rename kernel/arch/{i386 => i686}/crtn.S (100%) rename kernel/arch/{i386 => i686}/linker.ld (100%) rename libc/arch/{i386 => i686}/crt0.S (100%) rename libc/arch/{i386 => i686}/crti.S (94%) rename libc/arch/{i386 => i686}/crtn.S (92%) diff --git a/LibELF/LibELF/LoadableELF.cpp b/LibELF/LibELF/LoadableELF.cpp index 83ef8ba35b..941929010d 100644 --- a/LibELF/LibELF/LoadableELF.cpp +++ b/LibELF/LibELF/LoadableELF.cpp @@ -86,7 +86,7 @@ namespace LibELF return BAN::Error::from_errno(ENOEXEC); } -#if ARCH(i386) +#if ARCH(i686) if (m_file_header.e_ident[EI_CLASS] != ELFCLASS32) #elif ARCH(x86_64) if (m_file_header.e_ident[EI_CLASS] != ELFCLASS64) diff --git a/LibELF/include/LibELF/ELF.h b/LibELF/include/LibELF/ELF.h index a36c9dd0b9..a3eea008b9 100644 --- a/LibELF/include/LibELF/ELF.h +++ b/LibELF/include/LibELF/ELF.h @@ -42,7 +42,7 @@ namespace LibELF const Elf32SectionHeader& section_header32(size_t) const; const char* lookup_section_name32(uint32_t) const; const char* lookup_string32(size_t, uint32_t) const; -#if ARCH(i386) +#if ARCH(i686) const Elf32FileHeader& file_header_native() const { return file_header32(); } const Elf32ProgramHeader& program_header_native(size_t index) const { return program_header32(index); } const Elf32SectionHeader& section_header_native(size_t index) const { return section_header32(index); } diff --git a/LibELF/include/LibELF/Types.h b/LibELF/include/LibELF/Types.h index bcaf0cc319..cf0c0a9ad2 100644 --- a/LibELF/include/LibELF/Types.h +++ b/LibELF/include/LibELF/Types.h @@ -155,7 +155,7 @@ namespace LibELF Elf64Xword p_align; }; -#if ARCH(i386) +#if ARCH(i686) using ElfNativeAddr = Elf32Addr; using ElfNativeOff = Elf32Off; using ElfNativeHalf = Elf32Half; diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index 2fb59ab17d..f5344d73fd 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -4,7 +4,7 @@ project(kernel CXX C ASM) if("${BANAN_ARCH}" STREQUAL "x86_64") set(ELF_FORMAT elf64-x86-64) -elseif("${BANAN_ARCH}" STREQUAL "i386") +elseif("${BANAN_ARCH}" STREQUAL "i686") set(ELF_FORMAT elf32-i386) endif() @@ -120,16 +120,16 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") ${LAI_SOURCES} kernel/lai_host.cpp ) -elseif("${BANAN_ARCH}" STREQUAL "i386") +elseif("${BANAN_ARCH}" STREQUAL "i686") set(KERNEL_SOURCES ${KERNEL_SOURCES} - arch/i386/boot.S - arch/i386/GDT.cpp - arch/i386/IDT.cpp - arch/i386/PageTable.cpp - arch/i386/Signal.S - arch/i386/Syscall.S - arch/i386/Thread.S + arch/i686/boot.S + arch/i686/GDT.cpp + arch/i686/IDT.cpp + arch/i686/PageTable.cpp + arch/i686/Signal.S + arch/i686/Syscall.S + arch/i686/Thread.S ) else() message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}") @@ -182,8 +182,8 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") target_compile_options(kernel PUBLIC -mcmodel=kernel -mno-red-zone) target_link_options(kernel PUBLIC LINKER:-z,max-page-size=4096) target_link_options(kernel PUBLIC LINKER:-T,${CMAKE_CURRENT_SOURCE_DIR}/arch/x86_64/linker.ld) -elseif("${BANAN_ARCH}" STREQUAL "i386") - target_link_options(kernel PUBLIC LINKER:-T,${CMAKE_CURRENT_SOURCE_DIR}/arch/i386/linker.ld) +elseif("${BANAN_ARCH}" STREQUAL "i686") + target_link_options(kernel PUBLIC LINKER:-T,${CMAKE_CURRENT_SOURCE_DIR}/arch/i686/linker.ld) endif() target_link_options(kernel PUBLIC -ffreestanding -nostdlib) diff --git a/kernel/arch/i386/GDT.cpp b/kernel/arch/i686/GDT.cpp similarity index 100% rename from kernel/arch/i386/GDT.cpp rename to kernel/arch/i686/GDT.cpp diff --git a/kernel/arch/i386/IDT.cpp b/kernel/arch/i686/IDT.cpp similarity index 100% rename from kernel/arch/i386/IDT.cpp rename to kernel/arch/i686/IDT.cpp diff --git a/kernel/arch/i386/PageTable.cpp b/kernel/arch/i686/PageTable.cpp similarity index 100% rename from kernel/arch/i386/PageTable.cpp rename to kernel/arch/i686/PageTable.cpp diff --git a/kernel/arch/i386/Signal.S b/kernel/arch/i686/Signal.S similarity index 98% rename from kernel/arch/i386/Signal.S rename to kernel/arch/i686/Signal.S index 41a3b834a7..c22d1c4ce1 100644 --- a/kernel/arch/i386/Signal.S +++ b/kernel/arch/i686/Signal.S @@ -7,6 +7,8 @@ .global signal_trampoline signal_trampoline: + ud2 + pushl %ebp movl %esp, %ebp subl $8, %esp diff --git a/kernel/arch/i386/Syscall.S b/kernel/arch/i686/Syscall.S similarity index 98% rename from kernel/arch/i386/Syscall.S rename to kernel/arch/i686/Syscall.S index f4a40759b7..39f892016a 100644 --- a/kernel/arch/i386/Syscall.S +++ b/kernel/arch/i686/Syscall.S @@ -1,5 +1,6 @@ .global sys_fork_trampoline sys_fork_trampoline: + ud2 subl $4, %esp pushl %ebx pushl %ebp diff --git a/kernel/arch/i386/Thread.S b/kernel/arch/i686/Thread.S similarity index 100% rename from kernel/arch/i386/Thread.S rename to kernel/arch/i686/Thread.S diff --git a/kernel/arch/i386/boot.S b/kernel/arch/i686/boot.S similarity index 100% rename from kernel/arch/i386/boot.S rename to kernel/arch/i686/boot.S diff --git a/kernel/arch/i386/crt0.S b/kernel/arch/i686/crt0.S similarity index 100% rename from kernel/arch/i386/crt0.S rename to kernel/arch/i686/crt0.S diff --git a/kernel/arch/i386/crti.S b/kernel/arch/i686/crti.S similarity index 100% rename from kernel/arch/i386/crti.S rename to kernel/arch/i686/crti.S diff --git a/kernel/arch/i386/crtn.S b/kernel/arch/i686/crtn.S similarity index 100% rename from kernel/arch/i386/crtn.S rename to kernel/arch/i686/crtn.S diff --git a/kernel/arch/i386/linker.ld b/kernel/arch/i686/linker.ld similarity index 100% rename from kernel/arch/i386/linker.ld rename to kernel/arch/i686/linker.ld diff --git a/kernel/include/kernel/Arch.h b/kernel/include/kernel/Arch.h index ee3b2e7add..9c9692b8fb 100644 --- a/kernel/include/kernel/Arch.h +++ b/kernel/include/kernel/Arch.h @@ -1,11 +1,11 @@ #pragma once #define x86_64 1 -#define i386 2 +#define i686 2 #define ARCH(arch) (__arch == arch) -#if !defined(__arch) || (__arch != x86_64 && __arch != i386) +#if !defined(__arch) || (__arch != x86_64 && __arch != i686) #error "Unsupported architecture" #endif @@ -13,10 +13,12 @@ #define read_rsp(rsp) asm volatile("movq %%rsp, %0" : "=r"(rsp)) #define push_callee_saved() asm volatile("pushq %rbx; pushq %rbp; pushq %r12; pushq %r13; pushq %r14; pushq %r15") #define pop_callee_saved() asm volatile("popq %r15; popq %r14; popq %r13; popq %r12; popq %rbp; popq %rbx") -#else +#elif ARCH(i686) #define read_rsp(rsp) asm volatile("movl %%esp, %0" : "=r"(rsp)) #define push_callee_saved() asm volatile("pushal") #define pop_callee_saved() asm volatile("popal") +#else + #error #endif #include diff --git a/kernel/include/kernel/FS/TmpFS/Definitions.h b/kernel/include/kernel/FS/TmpFS/Definitions.h index bfbe33d3d1..e2e50772a7 100644 --- a/kernel/include/kernel/FS/TmpFS/Definitions.h +++ b/kernel/include/kernel/FS/TmpFS/Definitions.h @@ -28,7 +28,7 @@ namespace Kernel // 1x triply indirect BAN::Array block; static constexpr size_t direct_block_count = 2; -#elif ARCH(i386) +#elif ARCH(i686) // 14x direct blocks // 1x singly indirect // 1x doubly indirect diff --git a/kernel/include/kernel/Memory/Types.h b/kernel/include/kernel/Memory/Types.h index 5c06c52e7b..4911b70db1 100644 --- a/kernel/include/kernel/Memory/Types.h +++ b/kernel/include/kernel/Memory/Types.h @@ -4,7 +4,7 @@ #if ARCH(x86_64) #define KERNEL_OFFSET 0xFFFFFFFF80000000 -#elif ARCH(i386) +#elif ARCH(i686) #define KERNEL_OFFSET 0xC0000000 #else #error diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index f559f308d0..f01f116320 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -19,7 +19,7 @@ namespace Kernel using ProcessorID = uint32_t; constexpr ProcessorID PROCESSOR_NONE = 0xFFFFFFFF; -#if ARCH(x86_64) || ARCH(i386) +#if ARCH(x86_64) || ARCH(i686) class Processor { BAN_NON_COPYABLE(Processor); diff --git a/kernel/kernel/Random.cpp b/kernel/kernel/Random.cpp index 447019ed3d..ce65370494 100644 --- a/kernel/kernel/Random.cpp +++ b/kernel/kernel/Random.cpp @@ -22,7 +22,7 @@ namespace Kernel { #if ARCH(x86_64) asm volatile("rdrand %0" : "=r"(s_rand_seed)); -#elif ARCH(i386) +#elif ARCH(i686) uint32_t lo, hi; asm volatile( "rdrand %[lo];" diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 4a64eae682..a2440e963d 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -21,7 +21,7 @@ namespace Kernel { #if ARCH(x86_64) asm volatile("movq %0, %%rsp" :: "rm"(Processor::current_stack_top())); -#elif ARCH(i386) +#elif ARCH(i686) asm volatile("movl %0, %%esp" :: "rm"(Processor::current_stack_top())); #else #error @@ -221,7 +221,7 @@ namespace Kernel "orq $(1 << 3), %rax;" "movq %rax, %cr0" ); -#elif ARCH(i386) +#elif ARCH(i686) asm volatile( "movl %cr0, %eax;" "orl $(1 << 3), %eax;" diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index d757673f78..413335adb6 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -111,7 +111,7 @@ namespace Kernel ); save_sse(); asm volatile("movq %0, %%cr0" :: "r"(cr0)); - #elif ARCH(i386) + #elif ARCH(i686) uintptr_t cr0; asm volatile( "movl %%cr0, %%eax;" diff --git a/libc/arch/i386/crt0.S b/libc/arch/i686/crt0.S similarity index 100% rename from libc/arch/i386/crt0.S rename to libc/arch/i686/crt0.S diff --git a/libc/arch/i386/crti.S b/libc/arch/i686/crti.S similarity index 94% rename from libc/arch/i386/crti.S rename to libc/arch/i686/crti.S index 75175fd594..64945d13a2 100644 --- a/libc/arch/i386/crti.S +++ b/libc/arch/i686/crti.S @@ -1,4 +1,4 @@ -/* i386 crti.s */ +/* i686 crti.s */ .section .init .global _init .type _init, @function diff --git a/libc/arch/i386/crtn.S b/libc/arch/i686/crtn.S similarity index 92% rename from libc/arch/i386/crtn.S rename to libc/arch/i686/crtn.S index 935c8d1b82..9adcd82053 100644 --- a/libc/arch/i386/crtn.S +++ b/libc/arch/i686/crtn.S @@ -1,4 +1,4 @@ -/* i386 crtn.s */ +/* i686 crtn.s */ .section .init /* gcc will nicely put the contents of crtend.o's .init section here. */ popl %ebp From 5dbe51a52e16e720883f9b44e1a8608a4311cd36 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 03:03:33 +0200 Subject: [PATCH 25/46] Userspace: Update printf formats to compile on 32 bit --- userspace/aoc2023/day19/main.cpp | 2 +- userspace/image/Netbpm.cpp | 2 +- userspace/mmap-shared-test/main.cpp | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/userspace/aoc2023/day19/main.cpp b/userspace/aoc2023/day19/main.cpp index 7a18fe0fea..d0c39d16ea 100644 --- a/userspace/aoc2023/day19/main.cpp +++ b/userspace/aoc2023/day19/main.cpp @@ -260,7 +260,7 @@ i64 puzzle2(FILE* fp) clock_gettime(CLOCK_MONOTONIC, &time_stop); u64 duration_us = (time_stop.tv_sec * 1'000'000 + time_stop.tv_nsec / 1'000) - (time_start.tv_sec * 1'000'000 + time_start.tv_nsec / 1'000); - printf("took %lu.%03lu ms, estimate %lu s\n", duration_us / 1000, duration_us % 1000, (values_sorted[0].size() - xi - 2) * duration_us / 1'000'000); + printf("step took %" PRIu64 ".%03" PRIu64 " ms, estimate %" PRIu64 " s\n", duration_us / 1000, duration_us % 1000, (values_sorted[0].size() - xi - 2) * duration_us / 1'000'000); } return result; diff --git a/userspace/image/Netbpm.cpp b/userspace/image/Netbpm.cpp index 485f295834..4d89ed7ed7 100644 --- a/userspace/image/Netbpm.cpp +++ b/userspace/image/Netbpm.cpp @@ -87,7 +87,7 @@ BAN::ErrorOr> load_netbpm(const void* mmap_addr, size_t size return BAN::Error::from_errno(EINVAL); } - printf("Netbpm image %" PRIuPTR "x%" PRIuPTR "\n", *width, *height); + printf("Netbpm image %" PRIu64 "x%" PRIu64 "\n", *width, *height); BAN::Vector bitmap; TRY(bitmap.resize(*width * *height)); diff --git a/userspace/mmap-shared-test/main.cpp b/userspace/mmap-shared-test/main.cpp index 68de5710d1..b27b3ccd81 100644 --- a/userspace/mmap-shared-test/main.cpp +++ b/userspace/mmap-shared-test/main.cpp @@ -152,7 +152,7 @@ int test2_job2() return 1; } - *(size_t*)addr = 0x123456789; + *(size_t*)addr = 0x12345678; if (msync(addr, sizeof(size_t), MS_SYNC) == -1) { From ca23360d079c36b473d3a298652af6fa85893f6f Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 03:04:57 +0200 Subject: [PATCH 26/46] Bootloader: Fix GDRT pointer size to 32 bits --- bootloader/bios/boot.S | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootloader/bios/boot.S b/bootloader/bios/boot.S index 4d61652e2a..583f71c684 100644 --- a/bootloader/bios/boot.S +++ b/bootloader/bios/boot.S @@ -161,7 +161,7 @@ gdt: .quad 0x00CF9A000000FFFF # 32-bit code gdtr: .short . - gdt - 1 - .quad gdt + .long gdt banan_boot_info: boot_command_line: From 84ef2161a109e67b30d3194c494eaf7eb8817aaf Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 03:18:54 +0200 Subject: [PATCH 27/46] BuildSystem: Allow running qemu with i686 target --- script/qemu.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/script/qemu.sh b/script/qemu.sh index a34c23136d..9a49fc3cd2 100755 --- a/script/qemu.sh +++ b/script/qemu.sh @@ -19,7 +19,12 @@ else DISK_ARGS="-device ahci,id=ahci -device ide-hd,drive=disk,bus=ahci.0" fi -qemu-system-$BANAN_ARCH \ +QEMU_ARCH=$BANAN_ARCH +if [ $BANAN_ARCH = "i686" ]; then + QEMU_ARCH=i386 +fi + +qemu-system-$QEMU_ARCH \ -m 1G \ -smp 4 \ $BIOS_ARGS \ From af050cc729910d7f306295f1d66e671dc52fdee5 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 13:25:22 +0200 Subject: [PATCH 28/46] Kernel: Fix boot code for x86_32 Boot assembly now initializes processor and jumps to kernel --- kernel/arch/i686/boot.S | 72 +++++++++++++++++++++++++++++------------ 1 file changed, 52 insertions(+), 20 deletions(-) diff --git a/kernel/arch/i686/boot.S b/kernel/arch/i686/boot.S index ea69fff792..118d90977e 100644 --- a/kernel/arch/i686/boot.S +++ b/kernel/arch/i686/boot.S @@ -53,6 +53,7 @@ bananboot_start: bananboot_end: .section .bss, "aw", @nobits + .align 4096 boot_stack_bottom: .skip 4096 * 4 boot_stack_top: @@ -68,11 +69,23 @@ bananboot_end: .section .data +# Map first GiB to 0x00000000 and 0xC0000000 +.align 32 +boot_pdpt: + .long V2P(boot_pd) + (PG_PRESENT) + .long 0 + .quad 0 + .quad 0 + .long V2P(boot_pd) + (PG_PRESENT) + .long 0 .align 4096 -boot_pml4: -boot_pdpt_lo: -boot_pdpt_hi: boot_pd: + .set i, 0 + .rept 512 + .long i + (PG_PAGE_SIZE | PG_READ_WRITE | PG_PRESENT) + .long 0 + .set i, i + 0x200000 + .endr boot_gdt: .quad 0x0000000000000000 # null descriptor @@ -106,9 +119,25 @@ has_cpuid: testl $0x00200000, %eax ret +has_pae: + movl $0, %eax + cpuid + testl $(1 << 6), %edx + ret + +has_sse: + movl $1, %eax + cpuid + testl $(1 << 25), %edx + ret + check_requirements: call has_cpuid jz .exit + call has_pae + jz .exit + call has_sse + jz .exit ret .exit: jmp system_halt @@ -126,16 +155,16 @@ enable_sse: initialize_paging: # enable PAE movl %cr4, %ecx - orl $0x20, %ecx + orl $(1 << 5), %ecx movl %ecx, %cr4 - # set address of paging structures - movl $V2P(boot_pml4), %ecx + # load page tables + movl $V2P(boot_pdpt), %ecx movl %ecx, %cr3 # enable paging movl %cr0, %ecx - orl $0x80000000, %ecx + orl $(1 << 31), %ecx movl %ecx, %cr0 ret @@ -145,30 +174,30 @@ initialize_paging: _start: cli; cld - # Initialize stack and multiboot info + # save bootloader magic and info movl %eax, V2P(bootloader_magic) movl %ebx, V2P(bootloader_info) + # load boot stack movl $V2P(boot_stack_top), %esp - call check_requirements - call enable_sse - - call initialize_paging - - # flush gdt + # load boot GDT lgdt V2P(boot_gdtr) ljmpl $0x08, $V2P(gdt_flush) - gdt_flush: + # set correct segment registers movw $0x10, %ax movw %ax, %ds movw %ax, %ss movw %ax, %es - # move stack pointer to higher half - movl %esp, %esp - addl $KERNEL_OFFSET, %esp + # do processor initialization + call check_requirements + call enable_sse + call initialize_paging + + # load higher half stack pointer + movl $boot_stack_top, %esp # jump to higher half leal higher_half, %ecx @@ -181,9 +210,11 @@ higher_half: # call to the kernel itself (clear ebp for stacktrace) xorl %ebp, %ebp - movl V2P(bootloader_magic), %edi - movl V2P(bootloader_info), %esi + subl $8, %esp + pushl bootloader_info + pushl bootloader_magic call kernel_main + addl $16, %esp # call global destructors call _fini @@ -200,6 +231,7 @@ system_halt: .code16 .global ap_trampoline ap_trampoline: + ud2 jmp 1f .align 8 ap_stack_ptr: From 1943c3e7a139f1446bb026fafffc4f8281aa5f7b Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 13:59:09 +0200 Subject: [PATCH 29/46] Kernel: Unify IDT and GDT code between x86_64 and x86_32 The code is pretty much the same, so there are just couple macros differiating initialization. --- kernel/CMakeLists.txt | 7 +- kernel/arch/i686/GDT.cpp | 21 --- kernel/arch/i686/IDT.cpp | 31 ----- kernel/arch/i686/interrupts.S | 134 ++++++++++++++++++++ kernel/arch/x86_64/GDT.cpp | 56 -------- kernel/include/kernel/GDT.h | 80 ++++++++++-- kernel/include/kernel/IDT.h | 26 +++- kernel/include/kernel/InterruptController.h | 1 + kernel/kernel/GDT.cpp | 69 ++++++++++ kernel/{arch/x86_64 => kernel}/IDT.cpp | 39 +++--- kernel/kernel/InterruptController.cpp | 5 + kernel/kernel/Processor.cpp | 13 +- 12 files changed, 335 insertions(+), 147 deletions(-) delete mode 100644 kernel/arch/i686/GDT.cpp delete mode 100644 kernel/arch/i686/IDT.cpp create mode 100644 kernel/arch/i686/interrupts.S delete mode 100644 kernel/arch/x86_64/GDT.cpp create mode 100644 kernel/kernel/GDT.cpp rename kernel/{arch/x86_64 => kernel}/IDT.cpp (93%) diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index f5344d73fd..6dcdefd88a 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -33,6 +33,8 @@ set(KERNEL_SOURCES kernel/FS/TmpFS/FileSystem.cpp kernel/FS/TmpFS/Inode.cpp kernel/FS/VirtualFileSystem.cpp + kernel/GDT.cpp + kernel/IDT.cpp kernel/Input/KeyboardLayout.cpp kernel/Input/KeyEvent.cpp kernel/Input/PS2/Controller.cpp @@ -105,8 +107,6 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") set(KERNEL_SOURCES ${KERNEL_SOURCES} arch/x86_64/boot.S - arch/x86_64/GDT.cpp - arch/x86_64/IDT.cpp arch/x86_64/interrupts.S arch/x86_64/PageTable.cpp arch/x86_64/Signal.S @@ -124,8 +124,7 @@ elseif("${BANAN_ARCH}" STREQUAL "i686") set(KERNEL_SOURCES ${KERNEL_SOURCES} arch/i686/boot.S - arch/i686/GDT.cpp - arch/i686/IDT.cpp + arch/i686/interrupts.S arch/i686/PageTable.cpp arch/i686/Signal.S arch/i686/Syscall.S diff --git a/kernel/arch/i686/GDT.cpp b/kernel/arch/i686/GDT.cpp deleted file mode 100644 index f76d47f025..0000000000 --- a/kernel/arch/i686/GDT.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include - -namespace Kernel -{ - - GDT* GDT::create() - { - ASSERT_NOT_REACHED(); - } - - void GDT::write_entry(uint8_t, uint32_t, uint32_t, uint8_t, uint8_t) - { - ASSERT_NOT_REACHED(); - } - - void GDT::write_tss() - { - ASSERT_NOT_REACHED(); - } - -} diff --git a/kernel/arch/i686/IDT.cpp b/kernel/arch/i686/IDT.cpp deleted file mode 100644 index b11241fda7..0000000000 --- a/kernel/arch/i686/IDT.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include - -namespace Kernel -{ - - IDT* IDT::create() - { - ASSERT_NOT_REACHED(); - } - - [[noreturn]] void IDT::force_triple_fault() - { - ASSERT_NOT_REACHED(); - } - - void IDT::register_irq_handler(uint8_t, Interruptable*) - { - ASSERT_NOT_REACHED(); - } - - void IDT::register_interrupt_handler(uint8_t, void (*)()) - { - ASSERT_NOT_REACHED(); - } - - void IDT::register_syscall_handler(uint8_t, void (*)()) - { - ASSERT_NOT_REACHED(); - } - -} diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S new file mode 100644 index 0000000000..0b74df9cc1 --- /dev/null +++ b/kernel/arch/i686/interrupts.S @@ -0,0 +1,134 @@ +isr_stub: + ud2 + pusha + call cpp_isr_handler + popa + iret + +irq_stub: + ud2 + pusha + call cpp_irq_handler + popa + iret + +// arguments in EAX, EBX, ECX, EDX, ESI, EDI +.global syscall_asm +syscall_asm: + ud2 + pusha + + pushl %esp + addl $36, (%esp) + + pushl %edi + pushl %esi + pushl %edx + pushl %ecx + pushl %ebx + pushl %eax + + call cpp_syscall_handler + + addl $60, %esp + + popl %edi + popl %esi + popl %ebp + addl $4, %esp + popl %ebx + popl %edx + popl %ecx + addl $4, %esp + + iret + +.macro isr n + .global isr\n + isr\n: + pushl $0 + pushl $\n + jmp isr_stub +.endm + +.macro isr_err n + .global isr\n + isr\n: + pushl $\n + jmp isr_stub +.endm + +.macro irq n + .global irq\n + irq\n: + pushl $0 + pushl $\n + jmp irq_stub +.endm + +isr 0 +isr 1 +isr 2 +isr 3 +isr 4 +isr 5 +isr 6 +isr 7 +isr_err 8 +isr 9 +isr_err 10 +isr_err 11 +isr_err 12 +isr_err 13 +isr_err 14 +isr 15 +isr 16 +isr_err 17 +isr 18 +isr 19 +isr 20 +isr 21 +isr 22 +isr 23 +isr 24 +isr 25 +isr 26 +isr 27 +isr 28 +isr 29 +isr 30 +isr 31 + +irq 0 +irq 1 +irq 2 +irq 3 +irq 4 +irq 5 +irq 6 +irq 7 +irq 8 +irq 9 +irq 10 +irq 11 +irq 12 +irq 13 +irq 14 +irq 15 +irq 16 +irq 17 +irq 18 +irq 19 +irq 20 +irq 21 +irq 22 +irq 23 +irq 24 +irq 25 +irq 26 +irq 27 +irq 28 +irq 29 +irq 30 +irq 31 +irq 32 diff --git a/kernel/arch/x86_64/GDT.cpp b/kernel/arch/x86_64/GDT.cpp deleted file mode 100644 index 18f0b61ac2..0000000000 --- a/kernel/arch/x86_64/GDT.cpp +++ /dev/null @@ -1,56 +0,0 @@ -#include -#include - -#include - -namespace Kernel -{ - - GDT* GDT::create() - { - auto* gdt = new GDT(); - ASSERT(gdt); - - gdt->write_entry(0x00, 0x00000000, 0x00000, 0x00, 0x0); // null - gdt->write_entry(0x08, 0x00000000, 0xFFFFF, 0x9A, 0xA); // kernel code - gdt->write_entry(0x10, 0x00000000, 0xFFFFF, 0x92, 0xC); // kernel data - gdt->write_entry(0x18, 0x00000000, 0xFFFFF, 0xFA, 0xA); // user code - gdt->write_entry(0x20, 0x00000000, 0xFFFFF, 0xF2, 0xC); // user data - gdt->write_tss(); - - return gdt; - } - - void GDT::write_entry(uint8_t offset, uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) - { - ASSERT(offset % sizeof(SegmentDescriptor) == 0); - uint8_t idx = offset / sizeof(SegmentDescriptor); - - auto& desc = m_gdt[idx]; - desc.base1 = (base >> 0) & 0xFFFF; - desc.base2 = (base >> 16) & 0xFF; - desc.base3 = (base >> 24) & 0xFF; - - desc.limit1 = (limit >> 0) & 0xFFFF; - desc.limit2 = (limit >> 16) & 0x0F; - - desc.access = access & 0xFF; - - desc.flags = flags & 0x0F; - } - - void GDT::write_tss() - { - memset(&m_tss, 0x00, sizeof(TaskStateSegment)); - m_tss.iopb = sizeof(TaskStateSegment); - - uint64_t base = reinterpret_cast(&m_tss); - - write_entry(0x28, (uint32_t)base, sizeof(TaskStateSegment), 0x89, 0x0); - - auto& desc = m_gdt[0x30 / sizeof(SegmentDescriptor)]; - desc.low = base >> 32; - desc.high = 0; - } - -} diff --git a/kernel/include/kernel/GDT.h b/kernel/include/kernel/GDT.h index a4bb623e00..56efc2cced 100644 --- a/kernel/include/kernel/GDT.h +++ b/kernel/include/kernel/GDT.h @@ -2,12 +2,14 @@ #include #include +#include #include namespace Kernel { +#if ARCH(x86_64) struct TaskStateSegment { uint32_t reserved1; @@ -26,6 +28,54 @@ namespace Kernel uint16_t reserved4; uint16_t iopb; } __attribute__((packed)); + static_assert(sizeof(TaskStateSegment) == 104); +#elif ARCH(i686) + struct TaskStateSegment + { + uint16_t link; + uint16_t __reserved0; + uint32_t esp0; + uint16_t ss0; + uint16_t __reserved1; + uint32_t esp1; + uint16_t ss1; + uint16_t __reserved2; + uint32_t esp2; + uint16_t ss2; + uint16_t __reserved3; + uint32_t cr3; + uint32_t eip; + uint32_t eflags; + uint32_t eax; + uint32_t ecx; + uint32_t edx; + uint32_t ebx; + uint32_t esp; + uint32_t ebp; + uint32_t esi; + uint32_t edi; + uint16_t es; + uint16_t __reserved4; + uint16_t cs; + uint16_t __reserved5; + uint16_t ss; + uint16_t __reserved6; + uint16_t ds; + uint16_t __reserved7; + uint16_t fs; + uint16_t __reserved8; + uint16_t gs; + uint16_t __reserved9; + uint16_t ldtr; + uint16_t __reserved10; + uint16_t __reserved11; + uint16_t iopb; + uint32_t ssp; + }; + static_assert(sizeof(TaskStateSegment) == 108); +#else + #error +#endif union SegmentDescriptor { @@ -38,20 +88,20 @@ namespace Kernel uint8_t limit2 : 4; uint8_t flags : 4; uint8_t base3; - } __attribute__((packed)); + }; struct { uint32_t low; uint32_t high; - } __attribute__((packed)); - - } __attribute__((packed)); + }; + }; + static_assert(sizeof(SegmentDescriptor) == 8); struct GDTR { uint16_t size; - uint64_t address; + uintptr_t address; } __attribute__((packed)); class GDT @@ -60,7 +110,7 @@ namespace Kernel BAN_NON_MOVABLE(GDT); public: - static GDT* create(); + static GDT* create(void* processor); void load() { flush_gdt(); flush_tss(); } static constexpr inline bool is_user_segment(uint8_t segment) @@ -68,9 +118,13 @@ namespace Kernel return (segment & 3) == 3; } - void set_tss_stack(uintptr_t rsp) + void set_tss_stack(uintptr_t sp) { - m_tss.rsp0 = rsp; +#if ARCH(x86_64) + m_tss.rsp0 = sp; +#elif ARCH(i686) + m_tss.esp0 = sp; +#endif } private: @@ -86,15 +140,21 @@ namespace Kernel void flush_tss() { - asm volatile("ltr %0" :: "rm"((uint16_t)0x28) : "memory"); + asm volatile("ltr %0" :: "rm"(m_tss_offset) : "memory"); } private: +#if ARCH(x86_64) BAN::Array m_gdt; // null, kernel code, kernel data, user code, user data, tss low, tss high + static constexpr uint16_t m_tss_offset = 0x28; +#elif ARCH(i686) + BAN::Array m_gdt; // null, kernel code, kernel data, user code, user data, processor data, tss + static constexpr uint16_t m_tss_offset = 0x30; +#endif TaskStateSegment m_tss; const GDTR m_gdtr { .size = m_gdt.size() * sizeof(SegmentDescriptor) - 1, - .address = reinterpret_cast(m_gdt.data()) + .address = reinterpret_cast(m_gdt.data()) }; }; diff --git a/kernel/include/kernel/IDT.h b/kernel/include/kernel/IDT.h index b99a66f298..1c8b9db484 100644 --- a/kernel/include/kernel/IDT.h +++ b/kernel/include/kernel/IDT.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -12,21 +13,36 @@ constexpr uint8_t IRQ_IPI = 32; namespace Kernel { +#if ARCH(x86_64) struct GateDescriptor { - uint16_t offset1; + uint16_t offset0; uint16_t selector; uint8_t IST; uint8_t flags; - uint16_t offset2; - uint32_t offset3; + uint16_t offset1; + uint32_t offset2; uint32_t reserved; - } __attribute__((packed)); + }; + static_assert(sizeof(GateDescriptor) == 16); +#elif ARCH(i686) + struct GateDescriptor + { + uint16_t offset0; + uint16_t selector; + uint8_t reserved; + uint8_t flags; + uint16_t offset1; + }; + static_assert(sizeof(GateDescriptor) == 8); +#else + #error +#endif struct IDTR { uint16_t size; - uint64_t offset; + uintptr_t offset; } __attribute__((packed)); class IDT diff --git a/kernel/include/kernel/InterruptController.h b/kernel/include/kernel/InterruptController.h index 115d011220..b430e9837f 100644 --- a/kernel/include/kernel/InterruptController.h +++ b/kernel/include/kernel/InterruptController.h @@ -18,6 +18,7 @@ namespace Kernel virtual bool is_in_service(uint8_t) = 0; static void initialize(bool force_pic); + static bool is_initialized(); static InterruptController& get(); virtual void initialize_multiprocessor() = 0; diff --git a/kernel/kernel/GDT.cpp b/kernel/kernel/GDT.cpp new file mode 100644 index 0000000000..302a82ed8b --- /dev/null +++ b/kernel/kernel/GDT.cpp @@ -0,0 +1,69 @@ +#include +#include + +#include + +namespace Kernel +{ + + GDT* GDT::create([[maybe_unused]] void* processor) + { + auto* gdt = new GDT(); + ASSERT(gdt); + +#if ARCH(x86_64) + constexpr uint8_t code_flags = 0xA; + constexpr uint8_t data_flags = 0xC; +#elif ARCH(i686) + constexpr uint8_t code_flags = 0xC; + constexpr uint8_t data_flags = 0xC; +#endif + + gdt->write_entry(0x00, 0x00000000, 0x00000, 0x00, 0x0); // null + gdt->write_entry(0x08, 0x00000000, 0xFFFFF, 0x9A, code_flags); // kernel code + gdt->write_entry(0x10, 0x00000000, 0xFFFFF, 0x92, data_flags); // kernel data + gdt->write_entry(0x18, 0x00000000, 0xFFFFF, 0xFA, code_flags); // user code + gdt->write_entry(0x20, 0x00000000, 0xFFFFF, 0xF2, data_flags); // user data +#if ARCH(i686) + gdt->write_entry(0x28, reinterpret_cast(processor), sizeof(Processor), 0x92, 0x4); // processor data +#endif + gdt->write_tss(); + + return gdt; + } + + void GDT::write_entry(uint8_t offset, uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) + { + ASSERT(offset % sizeof(SegmentDescriptor) == 0); + uint8_t idx = offset / sizeof(SegmentDescriptor); + + auto& desc = m_gdt[idx]; + desc.base1 = (base >> 0) & 0xFFFF; + desc.base2 = (base >> 16) & 0xFF; + desc.base3 = (base >> 24) & 0xFF; + + desc.limit1 = (limit >> 0) & 0xFFFF; + desc.limit2 = (limit >> 16) & 0x0F; + + desc.access = access & 0xFF; + + desc.flags = flags & 0x0F; + } + + void GDT::write_tss() + { + memset(&m_tss, 0x00, sizeof(TaskStateSegment)); + m_tss.iopb = sizeof(TaskStateSegment); + + uintptr_t base = reinterpret_cast(&m_tss); + + write_entry(m_tss_offset, (uint32_t)base, sizeof(TaskStateSegment), 0x89, 0x0); + +#if ARCH(x86_64) + auto& desc = m_gdt[(m_tss_offset + 8) / sizeof(SegmentDescriptor)]; + desc.low = base >> 32; + desc.high = 0; +#endif + } + +} diff --git a/kernel/arch/x86_64/IDT.cpp b/kernel/kernel/IDT.cpp similarity index 93% rename from kernel/arch/x86_64/IDT.cpp rename to kernel/kernel/IDT.cpp index 0cf1531654..844a93e78a 100644 --- a/kernel/arch/x86_64/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -141,21 +141,16 @@ namespace Kernel "Unkown Exception 0x1F", }; - extern "C" void cpp_isr_handler(uint64_t isr, uint64_t error, InterruptStack& interrupt_stack, const Registers* regs) + extern "C" void cpp_isr_handler(uint32_t isr, uint32_t error, InterruptStack& interrupt_stack, const Registers* regs) { if (g_paniced) { dprintln("Processor {} halted", Processor::current_id()); - InterruptController::get().broadcast_ipi(); + if (InterruptController::is_initialized()) + InterruptController::get().broadcast_ipi(); asm volatile("cli; 1: hlt; jmp 1b"); } -#if __enable_sse - bool from_userspace = (interrupt_stack.cs & 0b11) == 0b11; - if (from_userspace) - Thread::current().save_sse(); -#endif - pid_t tid = Scheduler::current_tid(); pid_t pid = tid ? Process::current().pid() : 0; @@ -209,11 +204,19 @@ namespace Kernel #if __enable_sse else if (isr == ISR::DeviceNotAvailable) { +#if ARCH(x86_64) asm volatile( "movq %cr0, %rax;" "andq $~(1 << 3), %rax;" "movq %rax, %cr0;" ); +#elif ARCH(i686) + asm volatile( + "movl %cr0, %eax;" + "andl $~(1 << 3), %eax;" + "movl %eax, %cr0;" + ); +#endif if (auto* current = &Thread::current(); current != Thread::sse_thread()) { if (auto* sse = Thread::sse_thread()) @@ -302,7 +305,8 @@ done: if (g_paniced) { dprintln("Processor {} halted", Processor::current_id()); - InterruptController::get().broadcast_ipi(); + if (InterruptController::is_initialized()) + InterruptController::get().broadcast_ipi(); asm volatile("cli; 1: hlt; jmp 1b"); } @@ -332,14 +336,17 @@ done: void IDT::register_interrupt_handler(uint8_t index, void (*handler)()) { - auto& descriptor = m_idt[index]; - descriptor.offset1 = (uint16_t)((uint64_t)handler >> 0); - descriptor.offset2 = (uint16_t)((uint64_t)handler >> 16); - descriptor.offset3 = (uint32_t)((uint64_t)handler >> 32); + auto& desc = m_idt[index]; + memset(&desc, 0, sizeof(GateDescriptor)); - descriptor.selector = 0x08; - descriptor.IST = 0; - descriptor.flags = 0x8E; + desc.offset0 = (uint16_t)((uintptr_t)handler >> 0); + desc.offset1 = (uint16_t)((uintptr_t)handler >> 16); +#if ARCH(x86_64) + desc.offset2 = (uint32_t)((uintptr_t)handler >> 32); +#endif + + desc.selector = 0x08; + desc.flags = 0x8E; } void IDT::register_syscall_handler(uint8_t index, void (*handler)()) diff --git a/kernel/kernel/InterruptController.cpp b/kernel/kernel/InterruptController.cpp index 6cd043925f..5aa8e19d17 100644 --- a/kernel/kernel/InterruptController.cpp +++ b/kernel/kernel/InterruptController.cpp @@ -40,6 +40,11 @@ namespace Kernel s_instance->m_using_apic = false; } + bool InterruptController::is_initialized() + { + return s_instance; + } + void InterruptController::enter_acpi_mode() { #if ARCH(x86_64) diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index a9b6acd819..2fbab9b826 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -38,7 +38,7 @@ namespace Kernel processor.m_stack = kmalloc(s_stack_size, 4096, true); ASSERT(processor.m_stack); - processor.m_gdt = GDT::create(); + processor.m_gdt = GDT::create(&processor); ASSERT(processor.m_gdt); processor.m_idt = IDT::create(); @@ -52,14 +52,19 @@ namespace Kernel auto id = read_processor_id(); auto& processor = s_processors[id]; + ASSERT(processor.m_gdt); + processor.m_gdt->load(); + + // initialize GS +#if ARCH(x86_64) // set gs base to pointer to this processor uint64_t ptr = reinterpret_cast(&processor); uint32_t ptr_hi = ptr >> 32; uint32_t ptr_lo = ptr & 0xFFFFFFFF; asm volatile("wrmsr" :: "d"(ptr_hi), "a"(ptr_lo), "c"(MSR_IA32_GS_BASE)); - - ASSERT(processor.m_gdt); - processor.gdt().load(); +#elif ARCH(i686) + asm volatile("movw $0x28, %%ax; movw %%ax, %%gs" ::: "ax"); +#endif ASSERT(processor.m_idt); processor.idt().load(); From d7bf34ecd078dcb10c89a75d809116c1afc6c24a Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 16:36:51 +0200 Subject: [PATCH 30/46] Kernel: Write isr handler for x86_32 and cleanup x86_64 --- kernel/arch/i686/interrupts.S | 31 +++++++++++++- kernel/arch/x86_64/interrupts.S | 38 ++++++++---------- kernel/kernel/IDT.cpp | 71 ++++++++++++++++++++++++--------- 3 files changed, 97 insertions(+), 43 deletions(-) diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 0b74df9cc1..f48bcab18d 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -1,15 +1,42 @@ isr_stub: - ud2 pusha + + movl %cr0, %eax; pushl %eax + movl %cr2, %eax; pushl %eax + movl %cr3, %eax; pushl %eax + movl %cr4, %eax; pushl %eax + + movl %esp, %eax // register ptr + leal 56(%esp), %ebx // interrupt stack ptr + movl 52(%esp), %ecx // error code + movl 48(%esp), %edx // isr number + + subl $12, %esp + pushl %eax + pushl %ebx + pushl %ecx + pushl %edx call cpp_isr_handler + addl $44, %esp + popa + addl $8, %esp iret irq_stub: - ud2 pusha + + leal 40(%esp), %eax // interrupt stack ptr + movl 32(%esp), %ebx // irq number + + subl $12, %esp + pushl %eax + pushl %ebx call cpp_irq_handler + addl $20, %esp + popa + addl $8, %esp iret // arguments in EAX, EBX, ECX, EDX, ESI, EDI diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index 7a78e818ce..a9aabc053f 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -1,11 +1,11 @@ .macro pushaq pushq %rax - pushq %rbx pushq %rcx pushq %rdx + pushq %rbx pushq %rbp - pushq %rdi pushq %rsi + pushq %rdi pushq %r8 pushq %r9 pushq %r10 @@ -25,12 +25,12 @@ popq %r10 popq %r9 popq %r8 - popq %rsi popq %rdi + popq %rsi popq %rbp + popq %rbx popq %rdx popq %rcx - popq %rbx popq %rax .endm @@ -43,35 +43,29 @@ popq %r10 popq %r9 popq %r8 - popq %rsi popq %rdi + popq %rsi popq %rbp + popq %rbx popq %rdx popq %rcx - popq %rbx .endm isr_stub: pushaq - movq %cr0, %rax; pushq %rax - movq %cr2, %rax; pushq %rax - movq %cr3, %rax; pushq %rax - movq %cr4, %rax; pushq %rax - movq 184(%rsp), %rax; pushq %rax - movq 176(%rsp), %rax; pushq %rax - movq 208(%rsp), %rax; pushq %rax - - movq 176(%rsp), %rdi - movq 184(%rsp), %rsi - - movq %rsp, %rdx - addq $192, %rdx - - movq %rsp, %rcx + movq %cr0, %rax; pushq %rax + movq %cr2, %rax; pushq %rax + movq %cr3, %rax; pushq %rax + movq %cr4, %rax; pushq %rax + movq 152(%rsp), %rdi // isr number + movq 160(%rsp), %rsi // error code + leaq 168(%rsp), %rdx // interrupt stack ptr + movq %rsp, %rcx // register ptr call cpp_isr_handler - addq $56, %rsp + addq $32, %rsp + popaq addq $16, %rsp iretq diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index 844a93e78a..069a8410d9 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -15,11 +15,9 @@ namespace Kernel { +#if ARCH(x86_64) struct Registers { - uint64_t rsp; - uint64_t rip; - uint64_t rflags; uint64_t cr4; uint64_t cr3; uint64_t cr2; @@ -33,14 +31,33 @@ namespace Kernel uint64_t r10; uint64_t r9; uint64_t r8; - uint64_t rsi; + uint64_t rdi; + uint64_t rsi; uint64_t rbp; + uint64_t rbx; uint64_t rdx; uint64_t rcx; - uint64_t rbx; uint64_t rax; }; +#elif ARCH(i686) + struct Registers + { + uint32_t cr4; + uint32_t cr3; + uint32_t cr2; + uint32_t cr0; + + uint32_t edi; + uint32_t esi; + uint32_t ebp; + uint32_t unused; + uint32_t ebx; + uint32_t edx; + uint32_t ecx; + uint32_t eax; + }; +#endif #define X(num) 1 + static BAN::Array s_interruptables; @@ -141,7 +158,7 @@ namespace Kernel "Unkown Exception 0x1F", }; - extern "C" void cpp_isr_handler(uint32_t isr, uint32_t error, InterruptStack& interrupt_stack, const Registers* regs) + extern "C" void cpp_isr_handler(uint32_t isr, uint32_t error, InterruptStack* interrupt_stack, const Registers* regs) { if (g_paniced) { @@ -156,24 +173,24 @@ namespace Kernel if (tid) { - Thread::current().set_return_sp(interrupt_stack.sp); - Thread::current().set_return_ip(interrupt_stack.ip); + Thread::current().set_return_sp(interrupt_stack->sp); + Thread::current().set_return_ip(interrupt_stack->ip); if (isr == ISR::PageFault) { // Check if stack is OOB auto& stack = Thread::current().stack(); auto& istack = Thread::current().interrupt_stack(); - if (stack.vaddr() < interrupt_stack.sp && interrupt_stack.sp <= stack.vaddr() + stack.size()) + if (stack.vaddr() < interrupt_stack->sp && interrupt_stack->sp <= stack.vaddr() + stack.size()) ; // using normal stack - else if (istack.vaddr() < interrupt_stack.sp && interrupt_stack.sp <= istack.vaddr() + istack.size()) + else if (istack.vaddr() < interrupt_stack->sp && interrupt_stack->sp <= istack.vaddr() + istack.size()) ; // using interrupt stack else { derrorln("Stack pointer out of bounds!"); - derrorln("rip {H}", interrupt_stack.ip); + derrorln("rip {H}", interrupt_stack->ip); derrorln("rsp {H}, stack {H}->{H}, istack {H}->{H}", - interrupt_stack.sp, + interrupt_stack->sp, stack.vaddr(), stack.vaddr() + stack.size(), istack.vaddr(), istack.vaddr() + istack.size() ); @@ -228,9 +245,9 @@ namespace Kernel #endif } - if (PageTable::current().get_page_flags(interrupt_stack.ip & PAGE_ADDR_MASK) & PageTable::Flags::Present) + if (PageTable::current().get_page_flags(interrupt_stack->ip & PAGE_ADDR_MASK) & PageTable::Flags::Present) { - auto* machine_code = (const uint8_t*)interrupt_stack.ip; + auto* machine_code = (const uint8_t*)interrupt_stack->ip; dwarnln("While executing: {2H}{2H}{2H}{2H}{2H}{2H}{2H}{2H}", machine_code[0], machine_code[1], @@ -243,6 +260,7 @@ namespace Kernel ); } +#if ARCH(x86_64) dwarnln( "{} (error code: 0x{16H}), pid {}, tid {}\r\n" "Register dump\r\n" @@ -252,10 +270,25 @@ namespace Kernel "cr0=0x{16H}, cr2=0x{16H}, cr3=0x{16H}, cr4=0x{16H}", isr_exceptions[isr], error, pid, tid, regs->rax, regs->rbx, regs->rcx, regs->rdx, - regs->rsp, regs->rbp, regs->rdi, regs->rsi, - regs->rip, regs->rflags, + interrupt_stack->sp, regs->rbp, regs->rdi, regs->rsi, + interrupt_stack->ip, interrupt_stack->flags, regs->cr0, regs->cr2, regs->cr3, regs->cr4 ); +#elif ARCH(i686) + dwarnln( + "{} (error code: 0x{16H}), pid {}, tid {}\r\n" + "Register dump\r\n" + "eax=0x{8H}, ebx=0x{8H}, ecx=0x{8H}, edx=0x{8H}\r\n" + "esp=0x{8H}, ebp=0x{8H}, edi=0x{8H}, esi=0x{8H}\r\n" + "eip=0x{8H}, eflags=0x{8H}\r\n" + "cr0=0x{8H}, cr2=0x{8H}, cr3=0x{8H}, cr4=0x{8H}", + isr_exceptions[isr], error, pid, tid, + regs->eax, regs->ebx, regs->ecx, regs->edx, + interrupt_stack->sp, regs->ebp, regs->edi, regs->esi, + interrupt_stack->ip, interrupt_stack->flags, + regs->cr0, regs->cr2, regs->cr3, regs->cr4 + ); +#endif if (isr == ISR::PageFault) PageTable::current().debug_dump(); Debug::dump_stack_trace(); @@ -300,7 +333,7 @@ done: return; } - extern "C" void cpp_irq_handler(uint64_t irq, InterruptStack& interrupt_stack) + extern "C" void cpp_irq_handler(uint32_t irq, InterruptStack* interrupt_stack) { if (g_paniced) { @@ -312,8 +345,8 @@ done: if (Scheduler::current_tid()) { - Thread::current().set_return_sp(interrupt_stack.sp); - Thread::current().set_return_ip(interrupt_stack.ip); + Thread::current().set_return_sp(interrupt_stack->sp); + Thread::current().set_return_ip(interrupt_stack->ip); } if (!InterruptController::get().is_in_service(irq)) From 4d1f0e77f229d4f14203f97f380b821e7bea8816 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 19:05:34 +0200 Subject: [PATCH 31/46] Kernel: Fix physical address size for x86_32 Having 32 bit address space does not mean physical address space is also only 32 bits... --- kernel/include/kernel/FS/TmpFS/Definitions.h | 7 ++++--- kernel/include/kernel/Memory/Types.h | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/kernel/include/kernel/FS/TmpFS/Definitions.h b/kernel/include/kernel/FS/TmpFS/Definitions.h index e2e50772a7..1b57c0d6c6 100644 --- a/kernel/include/kernel/FS/TmpFS/Definitions.h +++ b/kernel/include/kernel/FS/TmpFS/Definitions.h @@ -29,12 +29,13 @@ namespace Kernel BAN::Array block; static constexpr size_t direct_block_count = 2; #elif ARCH(i686) - // 14x direct blocks + uint32_t __padding; + // 5x direct blocks // 1x singly indirect // 1x doubly indirect // 1x triply indirect - BAN::Array block; - static constexpr size_t direct_block_count = 14; + BAN::Array block; + static constexpr size_t direct_block_count = 5; #else #error #endif diff --git a/kernel/include/kernel/Memory/Types.h b/kernel/include/kernel/Memory/Types.h index 4911b70db1..364635fbbe 100644 --- a/kernel/include/kernel/Memory/Types.h +++ b/kernel/include/kernel/Memory/Types.h @@ -21,6 +21,6 @@ namespace Kernel { using vaddr_t = uintptr_t; - using paddr_t = uintptr_t; + using paddr_t = uint64_t; } From c12d1e9bd98981ca2b0c221c32d8542d831ce5a0 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 19:06:37 +0200 Subject: [PATCH 32/46] Kernel: Implement PageTable for x86_32 This is mostly copied from x86_64 with necessary modifications --- kernel/arch/i686/PageTable.cpp | 688 ++++++++++++++++++++++++++++----- 1 file changed, 587 insertions(+), 101 deletions(-) diff --git a/kernel/arch/i686/PageTable.cpp b/kernel/arch/i686/PageTable.cpp index fe28366c3a..a5e60c5967 100644 --- a/kernel/arch/i686/PageTable.cpp +++ b/kernel/arch/i686/PageTable.cpp @@ -1,144 +1,630 @@ -#include +#include #include +#include +#include + +extern uint8_t g_kernel_start[]; +extern uint8_t g_kernel_end[]; + +extern uint8_t g_kernel_execute_start[]; +extern uint8_t g_kernel_execute_end[]; + +extern uint8_t g_userspace_start[]; +extern uint8_t g_userspace_end[]; namespace Kernel { RecursiveSpinLock PageTable::s_fast_page_lock; + static PageTable* s_kernel = nullptr; + static bool s_has_nxe = false; + static bool s_has_pge = false; + + static paddr_t s_global_pdpte = 0; + + static inline PageTable::flags_t parse_flags(uint64_t entry) + { + using Flags = PageTable::Flags; + + PageTable::flags_t result = 0; + if (s_has_nxe && !(entry & (1ull << 63))) + result |= Flags::Execute; + if (entry & Flags::Reserved) + result |= Flags::Reserved; + if (entry & Flags::CacheDisable) + result |= Flags::CacheDisable; + if (entry & Flags::UserSupervisor) + result |= Flags::UserSupervisor; + if (entry & Flags::ReadWrite) + result |= Flags::ReadWrite; + if (entry & Flags::Present) + result |= Flags::Present; + return result; + } + void PageTable::initialize() { - ASSERT_NOT_REACHED(); - } + if (CPUID::has_nxe()) + s_has_nxe = true; - PageTable& PageTable::kernel() - { - ASSERT_NOT_REACHED(); - } + if (CPUID::has_pge()) + s_has_pge = true; - bool PageTable::is_valid_pointer(uintptr_t) - { - ASSERT_NOT_REACHED(); - } + ASSERT(s_kernel == nullptr); + s_kernel = new PageTable(); + ASSERT(s_kernel); - BAN::ErrorOr PageTable::create_userspace() - { - ASSERT_NOT_REACHED(); - } - - PageTable::~PageTable() - { - ASSERT_NOT_REACHED(); - } - - void PageTable::unmap_page(vaddr_t) - { - ASSERT_NOT_REACHED(); - } - - void PageTable::unmap_range(vaddr_t, size_t) - { - ASSERT_NOT_REACHED(); - } - - void PageTable::map_range_at(paddr_t, vaddr_t, size_t, flags_t) - { - ASSERT_NOT_REACHED(); - } - - void PageTable::map_page_at(paddr_t, vaddr_t, flags_t) - { - ASSERT_NOT_REACHED(); - } - - paddr_t PageTable::physical_address_of(vaddr_t) const - { - ASSERT_NOT_REACHED(); - } - - PageTable::flags_t PageTable::get_page_flags(vaddr_t) const - { - ASSERT_NOT_REACHED(); - } - - bool PageTable::is_page_free(vaddr_t) const - { - ASSERT_NOT_REACHED(); - } - - bool PageTable::is_range_free(vaddr_t, size_t) const - { - ASSERT_NOT_REACHED(); - } - - bool PageTable::reserve_page(vaddr_t, bool) - { - ASSERT_NOT_REACHED(); - } - - bool PageTable::reserve_range(vaddr_t, size_t, bool) - { - ASSERT_NOT_REACHED(); - } - - vaddr_t PageTable::reserve_free_page(vaddr_t, vaddr_t) - { - ASSERT_NOT_REACHED(); - } - - vaddr_t PageTable::reserve_free_contiguous_pages(size_t, vaddr_t, vaddr_t) - { - ASSERT_NOT_REACHED(); - } - - void PageTable::load() - { - ASSERT_NOT_REACHED(); + s_kernel->initialize_kernel(); + s_kernel->initial_load(); } void PageTable::initial_load() { - ASSERT_NOT_REACHED(); + if (s_has_nxe) + { + asm volatile( + "movl $0xC0000080, %%ecx;" + "rdmsr;" + "orl $0x800, %%eax;" + "wrmsr" + ::: "eax", "ecx", "edx", "memory" + ); + } + + if (s_has_pge) + { + asm volatile( + "movl %%cr4, %%eax;" + "orl $0x80, %%eax;" + "movl %%eax, %%cr4;" + ::: "eax" + ); + } + + // enable write protect + asm volatile( + "movl %%cr0, %%eax;" + "orl $0x10000, %%eax;" + "movl %%eax, %%cr0;" + ::: "rax" + ); + + load(); } - void PageTable::debug_dump() + PageTable& PageTable::kernel() { - ASSERT_NOT_REACHED(); + ASSERT(s_kernel); + return *s_kernel; } - uint64_t PageTable::get_page_data(vaddr_t) const + bool PageTable::is_valid_pointer(uintptr_t) { - ASSERT_NOT_REACHED(); + return true; + } + + static uint64_t* allocate_zeroed_page_aligned_page() + { + void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true); + ASSERT(page); + memset(page, 0, PAGE_SIZE); + return (uint64_t*)page; } void PageTable::initialize_kernel() { - ASSERT_NOT_REACHED(); - } + ASSERT(s_global_pdpte == 0); + s_global_pdpte = V2P(allocate_zeroed_page_aligned_page()); - void PageTable::map_kernel_memory() - { - ASSERT_NOT_REACHED(); + ASSERT(m_highest_paging_struct == 0); + m_highest_paging_struct = V2P(kmalloc(32, 32, true)); + ASSERT(m_highest_paging_struct); + + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + pdpt[0] = 0; + pdpt[1] = 0; + pdpt[2] = 0; + pdpt[3] = s_global_pdpte; + static_assert(KERNEL_OFFSET == 0xC0000000); + + prepare_fast_page(); + + // Map main bios area below 1 MiB + map_range_at( + 0x000E0000, + P2V(0x000E0000), + 0x00100000 - 0x000E0000, + PageTable::Flags::Present + ); + + // Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end) + ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0); + map_range_at( + V2P(g_kernel_start), + (vaddr_t)g_kernel_start, + g_kernel_end - g_kernel_start, + Flags::ReadWrite | Flags::Present + ); + + // Map executable kernel memory as executable + map_range_at( + V2P(g_kernel_execute_start), + (vaddr_t)g_kernel_execute_start, + g_kernel_execute_end - g_kernel_execute_start, + Flags::Execute | Flags::Present + ); + + // Map userspace memory + map_range_at( + V2P(g_userspace_start), + (vaddr_t)g_userspace_start, + g_userspace_end - g_userspace_start, + Flags::Execute | Flags::UserSupervisor | Flags::Present + ); } void PageTable::prepare_fast_page() { - ASSERT_NOT_REACHED(); + constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; + constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; + constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; + + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + ASSERT(!(pdpt[pdpte] & Flags::Present)); + pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present; + + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte]) & PAGE_ADDR_MASK); + ASSERT(!(pd[pde] & Flags::Present)); + pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present; + + uint64_t* pt = reinterpret_cast(P2V(pd[pde]) & PAGE_ADDR_MASK); + ASSERT(!(pt[pte] & Flags::Present)); + pt[pte] = V2P(allocate_zeroed_page_aligned_page()); } - void PageTable::invalidate(vaddr_t) + void PageTable::map_fast_page(paddr_t paddr) { - ASSERT_NOT_REACHED(); - } + ASSERT(s_kernel); + ASSERT(paddr); - void PageTable::map_fast_page(paddr_t) - { - ASSERT_NOT_REACHED(); + SpinLockGuard _(s_fast_page_lock); + + constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; + constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; + constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; + + uint64_t* pdpt = reinterpret_cast(P2V(s_kernel->m_highest_paging_struct)); + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); + uint64_t* pt = reinterpret_cast(P2V(pd[pde] & PAGE_ADDR_MASK)); + + ASSERT(!(pt[pte] & Flags::Present)); + pt[pte] = paddr | Flags::ReadWrite | Flags::Present; + + invalidate(fast_page()); } void PageTable::unmap_fast_page() { + ASSERT(s_kernel); + + SpinLockGuard _(s_fast_page_lock); + + constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF; + constexpr uint64_t pde = (fast_page() >> 21) & 0x1FF; + constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; + + uint64_t* pdpt = reinterpret_cast(P2V(s_kernel->m_highest_paging_struct)); + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); + uint64_t* pt = reinterpret_cast(P2V(pd[pde] & PAGE_ADDR_MASK)); + + ASSERT(pt[pte] & Flags::Present); + pt[pte] = 0; + + invalidate(fast_page()); + } + + BAN::ErrorOr PageTable::create_userspace() + { + SpinLockGuard _(s_kernel->m_lock); + PageTable* page_table = new PageTable; + if (page_table == nullptr) + return BAN::Error::from_errno(ENOMEM); + page_table->map_kernel_memory(); + return page_table; + } + + void PageTable::map_kernel_memory() + { + ASSERT(s_kernel); + ASSERT(s_global_pdpte); + + ASSERT(m_highest_paging_struct == 0); + m_highest_paging_struct = V2P(kmalloc(32, 32, true)); + ASSERT(m_highest_paging_struct); + + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + pdpt[0] = 0; + pdpt[1] = 0; + pdpt[2] = 0; + pdpt[3] = s_global_pdpte; + static_assert(KERNEL_OFFSET == 0xC0000000); + } + + PageTable::~PageTable() + { + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + + for (uint32_t pdpte = 0; pdpte < 3; pdpte++) + { + if (!(pdpt[pdpte] & Flags::Present)) + continue; + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); + for (uint32_t pde = 0; pde < 512; pde++) + { + if (!(pd[pde] & Flags::Present)) + continue; + kfree(reinterpret_cast(P2V(pd[pde] & PAGE_ADDR_MASK))); + } + kfree(pd); + } + kfree(pdpt); + } + + void PageTable::load() + { + SpinLockGuard _(m_lock); + ASSERT(m_highest_paging_struct < 0x100000000); + const uint32_t pdpt_lo = m_highest_paging_struct; + asm volatile("movl %0, %%cr3" :: "r"(pdpt_lo)); + Processor::set_current_page_table(this); + } + + void PageTable::invalidate(vaddr_t vaddr) + { + ASSERT(vaddr % PAGE_SIZE == 0); + asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory"); + } + + void PageTable::unmap_page(vaddr_t vaddr) + { + ASSERT(vaddr); + ASSERT(vaddr % PAGE_SIZE == 0); + ASSERT(vaddr != fast_page()); + if (vaddr >= KERNEL_OFFSET) + ASSERT(vaddr >= (vaddr_t)g_kernel_start); + if ((vaddr >= KERNEL_OFFSET) != (this == s_kernel)) + Kernel::panic("unmapping {8H}, kernel: {}", vaddr, this == s_kernel); + + const uint64_t pdpte = (vaddr >> 30) & 0x1FF; + const uint64_t pde = (vaddr >> 21) & 0x1FF; + const uint64_t pte = (vaddr >> 12) & 0x1FF; + + SpinLockGuard _(m_lock); + + if (is_page_free(vaddr)) + { + dwarnln("unmapping unmapped page {8H}", vaddr); + return; + } + + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); + uint64_t* pt = reinterpret_cast(P2V(pd[pde] & PAGE_ADDR_MASK)); + + pt[pte] = 0; + invalidate(vaddr); + } + + void PageTable::unmap_range(vaddr_t vaddr, size_t size) + { + vaddr_t s_page = vaddr / PAGE_SIZE; + vaddr_t e_page = BAN::Math::div_round_up(vaddr + size, PAGE_SIZE); + + SpinLockGuard _(m_lock); + for (vaddr_t page = s_page; page < e_page; page++) + unmap_page(page * PAGE_SIZE); + } + + void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags) + { + ASSERT(vaddr); + ASSERT(vaddr != fast_page()); + if ((vaddr >= KERNEL_OFFSET) != (this == s_kernel)) + Kernel::panic("mapping {8H} to {8H}, kernel: {}", paddr, vaddr, this == s_kernel); + + ASSERT(paddr % PAGE_SIZE == 0); + ASSERT(vaddr % PAGE_SIZE == 0); + ASSERT(flags & Flags::Used); + + const uint64_t pdpte = (vaddr >> 30) & 0x1FF; + const uint64_t pde = (vaddr >> 21) & 0x1FF; + const uint64_t pte = (vaddr >> 12) & 0x1FF; + + uint64_t extra_flags = 0; + if (s_has_pge && vaddr >= KERNEL_OFFSET) // Map kernel memory as global + extra_flags |= 1ull << 8; + if (s_has_nxe && !(flags & Flags::Execute)) + extra_flags |= 1ull << 63; + if (flags & Flags::Reserved) + extra_flags |= Flags::Reserved; + if (flags & Flags::CacheDisable) + extra_flags |= Flags::CacheDisable; + + // NOTE: we add present here, since it has to be available in higher level structures + flags_t uwr_flags = (flags & (Flags::UserSupervisor | Flags::ReadWrite)) | Flags::Present; + + SpinLockGuard _(m_lock); + + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + if (!(pdpt[pdpte] & Flags::Present)) + pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present; + + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); + if ((pd[pde] & uwr_flags) != uwr_flags) + { + if (!(pd[pde] & Flags::Present)) + pd[pde] = V2P(allocate_zeroed_page_aligned_page()); + pd[pde] |= uwr_flags; + } + + if (!(flags & Flags::Present)) + uwr_flags &= ~Flags::Present; + + uint64_t* pt = reinterpret_cast(P2V(pd[pde] & PAGE_ADDR_MASK)); + pt[pte] = paddr | uwr_flags | extra_flags; + + invalidate(vaddr); + } + + void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags) + { + ASSERT(vaddr); + ASSERT(paddr % PAGE_SIZE == 0); + ASSERT(vaddr % PAGE_SIZE == 0); + + size_t page_count = range_page_count(vaddr, size); + + SpinLockGuard _(m_lock); + for (size_t page = 0; page < page_count; page++) + map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags); + } + + uint64_t PageTable::get_page_data(vaddr_t vaddr) const + { + ASSERT(vaddr % PAGE_SIZE == 0); + + const uint64_t pdpte = (vaddr >> 30) & 0x1FF; + const uint64_t pde = (vaddr >> 21) & 0x1FF; + const uint64_t pte = (vaddr >> 12) & 0x1FF; + + SpinLockGuard _(m_lock); + + uint64_t* pdpt = (uint64_t*)P2V(m_highest_paging_struct); + if (!(pdpt[pdpte] & Flags::Present)) + return 0; + + uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK); + if (!(pd[pde] & Flags::Present)) + return 0; + + uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK); + if (!(pt[pte] & Flags::Used)) + return 0; + + return pt[pte]; + } + + PageTable::flags_t PageTable::get_page_flags(vaddr_t vaddr) const + { + return parse_flags(get_page_data(vaddr)); + } + + paddr_t PageTable::physical_address_of(vaddr_t vaddr) const + { + uint64_t page_data = get_page_data(vaddr); + return (page_data & PAGE_ADDR_MASK) & ~(1ull << 63); + } + + bool PageTable::is_page_free(vaddr_t vaddr) const + { + ASSERT(vaddr % PAGE_SIZE == 0); + return !(get_page_flags(vaddr) & Flags::Used); + } + + bool PageTable::is_range_free(vaddr_t vaddr, size_t size) const + { + vaddr_t s_page = vaddr / PAGE_SIZE; + vaddr_t e_page = BAN::Math::div_round_up(vaddr + size, PAGE_SIZE); + + SpinLockGuard _(m_lock); + for (vaddr_t page = s_page; page < e_page; page++) + if (!is_page_free(page * PAGE_SIZE)) + return false; + return true; + } + + bool PageTable::reserve_page(vaddr_t vaddr, bool only_free) + { + SpinLockGuard _(m_lock); + ASSERT(vaddr % PAGE_SIZE == 0); + if (only_free && !is_page_free(vaddr)) + return false; + map_page_at(0, vaddr, Flags::Reserved); + return true; + } + + bool PageTable::reserve_range(vaddr_t vaddr, size_t bytes, bool only_free) + { + if (size_t rem = bytes % PAGE_SIZE) + bytes += PAGE_SIZE - rem; + ASSERT(vaddr % PAGE_SIZE == 0); + + SpinLockGuard _(m_lock); + if (only_free && !is_range_free(vaddr, bytes)) + return false; + for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE) + reserve_page(vaddr + offset); + return true; + } + + vaddr_t PageTable::reserve_free_page(vaddr_t first_address, vaddr_t last_address) + { + if (first_address >= KERNEL_OFFSET && first_address < (vaddr_t)g_kernel_end) + first_address = (vaddr_t)g_kernel_end; + if (size_t rem = first_address % PAGE_SIZE) + first_address += PAGE_SIZE - rem; + if (size_t rem = last_address % PAGE_SIZE) + last_address -= rem; + + const uint32_t s_pdpte = (first_address >> 30) & 0x1FF; + const uint32_t s_pde = (first_address >> 21) & 0x1FF; + const uint32_t s_pte = (first_address >> 12) & 0x1FF; + + const uint32_t e_pdpte = (last_address >> 30) & 0x1FF; + const uint32_t e_pde = (last_address >> 21) & 0x1FF; + const uint32_t e_pte = (last_address >> 12) & 0x1FF; + + SpinLockGuard _(m_lock); + + // Try to find free page that can be mapped without + // allocations (page table with unused entries) + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + for (uint32_t pdpte = s_pdpte; pdpte < 4; pdpte++) + { + if (pdpte > e_pdpte) + break; + if (!(pdpt[pdpte] & Flags::Present)) + continue; + uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte] & PAGE_ADDR_MASK)); + for (uint32_t pde = s_pde; pde < 512; pde++) + { + if (pdpte == e_pdpte && pde > e_pde) + break; + if (!(pd[pde] & Flags::Present)) + continue; + uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK); + for (uint32_t pte = s_pte; pte < 512; pte++) + { + if (pdpte == e_pdpte && pde == e_pde && pte >= e_pte) + break; + if (!(pt[pte] & Flags::Used)) + { + vaddr_t vaddr = 0; + vaddr |= (vaddr_t)pdpte << 30; + vaddr |= (vaddr_t)pde << 21; + vaddr |= (vaddr_t)pte << 12; + ASSERT(reserve_page(vaddr)); + return vaddr; + } + } + } + } + + // Find any free page + for (vaddr_t vaddr = first_address; vaddr < last_address; vaddr += PAGE_SIZE) + { + if (is_page_free(vaddr)) + { + ASSERT(reserve_page(vaddr)); + return vaddr; + } + } + ASSERT_NOT_REACHED(); } + vaddr_t PageTable::reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address) + { + if (first_address >= KERNEL_OFFSET && first_address < (vaddr_t)g_kernel_start) + first_address = (vaddr_t)g_kernel_start; + if (size_t rem = first_address % PAGE_SIZE) + first_address += PAGE_SIZE - rem; + if (size_t rem = last_address % PAGE_SIZE) + last_address -= rem; + + SpinLockGuard _(m_lock); + + for (vaddr_t vaddr = first_address; vaddr < last_address;) + { + bool valid { true }; + for (size_t page = 0; page < page_count; page++) + { + if (!is_page_free(vaddr + page * PAGE_SIZE)) + { + vaddr += (page + 1) * PAGE_SIZE; + valid = false; + break; + } + } + if (valid) + { + ASSERT(reserve_range(vaddr, page_count * PAGE_SIZE)); + return vaddr; + } + } + + ASSERT_NOT_REACHED(); + } + + static void dump_range(vaddr_t start, vaddr_t end, PageTable::flags_t flags) + { + if (start == 0) + return; + dprintln("{}-{}: {}{}{}{}", + (void*)(start), (void*)(end - 1), + flags & PageTable::Flags::Execute ? 'x' : '-', + flags & PageTable::Flags::UserSupervisor ? 'u' : '-', + flags & PageTable::Flags::ReadWrite ? 'w' : '-', + flags & PageTable::Flags::Present ? 'r' : '-' + ); + } + + void PageTable::debug_dump() + { + SpinLockGuard _(m_lock); + + flags_t flags = 0; + vaddr_t start = 0; + + uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); + for (uint32_t pdpte = 0; pdpte < 4; pdpte++) + { + if (!(pdpt[pdpte] & Flags::Present)) + { + dump_range(start, (pdpte << 30), flags); + start = 0; + continue; + } + uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK); + for (uint64_t pde = 0; pde < 512; pde++) + { + if (!(pd[pde] & Flags::Present)) + { + dump_range(start, (pdpte << 30) | (pde << 21), flags); + start = 0; + continue; + } + uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK); + for (uint64_t pte = 0; pte < 512; pte++) + { + if (parse_flags(pt[pte]) != flags) + { + dump_range(start, (pdpte << 30) | (pde << 21) | (pte << 12), flags); + start = 0; + } + + if (!(pt[pte] & Flags::Used)) + continue; + + if (start == 0) + { + flags = parse_flags(pt[pte]); + start = (pdpte << 30) | (pde << 21) | (pte << 12); + } + } + } + } + } + } From 96babec22afe328dc35d681ae9a929ff16b9e1e9 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 19:42:14 +0200 Subject: [PATCH 33/46] Kernel: Implement Thread trampolines for x86_32 --- kernel/arch/i686/Thread.S | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/kernel/arch/i686/Thread.S b/kernel/arch/i686/Thread.S index d0b6a04f90..42fc440071 100644 --- a/kernel/arch/i686/Thread.S +++ b/kernel/arch/i686/Thread.S @@ -7,12 +7,35 @@ read_ip: # void start_thread(uint32_t sp, uint32_t ip) .global start_thread start_thread: - ud2 + movl 8(%esp), %edi # ip + movl 4(%esp), %esp # sp + + # STACK LAYOUT + # NULL + # thread ptr + # &Thread::on_exit + # data + + xorl %ebp, %ebp + + sti + call *%edi + + movl 4(%esp), %edi # &Thread::on_exit + + movl 8(%esp), %eax # thread ptr + movl %eax, (%esp) + + call *%edi + # void continue_thread(uint32_t sp, uint32_t ip) .global continue_thread continue_thread: - ud2 + movl 8(%esp), %edi # ip + movl 4(%esp), %esp # sp + xorl %eax, %eax + jmp *%edi # void thread_userspace_trampoline(uint32_t sp, uint32_t ip, int argc, char** argv, char** envp) .global thread_userspace_trampoline From 7c2933aae1985b3372e0c8f2650ba988a8f95a87 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 26 Mar 2024 20:10:09 +0200 Subject: [PATCH 34/46] Kernel: Fix ISR error code formatting to 32 bit --- kernel/kernel/IDT.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index 069a8410d9..1b8ea33d93 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -262,7 +262,7 @@ namespace Kernel #if ARCH(x86_64) dwarnln( - "{} (error code: 0x{16H}), pid {}, tid {}\r\n" + "{} (error code: 0x{8H}), pid {}, tid {}\r\n" "Register dump\r\n" "rax=0x{16H}, rbx=0x{16H}, rcx=0x{16H}, rdx=0x{16H}\r\n" "rsp=0x{16H}, rbp=0x{16H}, rdi=0x{16H}, rsi=0x{16H}\r\n" @@ -276,7 +276,7 @@ namespace Kernel ); #elif ARCH(i686) dwarnln( - "{} (error code: 0x{16H}), pid {}, tid {}\r\n" + "{} (error code: 0x{8H}), pid {}, tid {}\r\n" "Register dump\r\n" "eax=0x{8H}, ebx=0x{8H}, ecx=0x{8H}, edx=0x{8H}\r\n" "esp=0x{8H}, ebp=0x{8H}, edi=0x{8H}, esi=0x{8H}\r\n" From 1b65f850ee47ec1bef16b41dc951191351a0dad3 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 27 Mar 2024 15:06:24 +0200 Subject: [PATCH 35/46] Kernel: Rename thread stacks to more appropriate names --- kernel/include/kernel/Thread.h | 21 ++++++++++---------- kernel/kernel/IDT.cpp | 17 ++++++++-------- kernel/kernel/Process.cpp | 6 +++--- kernel/kernel/Scheduler.cpp | 2 +- kernel/kernel/Thread.cpp | 36 +++++++++++++++++----------------- 5 files changed, 40 insertions(+), 42 deletions(-) diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index 5e66069b20..dd2e276360 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -67,13 +67,13 @@ namespace Kernel void set_started() { ASSERT(m_state == State::NotStarted); m_state = State::Executing; } State state() const { return m_state; } - vaddr_t stack_base() const { return m_stack->vaddr(); } - size_t stack_size() const { return m_stack->size(); } - VirtualRange& stack() { return *m_stack; } - VirtualRange& interrupt_stack() { return *m_interrupt_stack; } + vaddr_t kernel_stack_bottom() const { return m_kernel_stack->vaddr(); } + vaddr_t kernel_stack_top() const { return m_kernel_stack->vaddr() + m_kernel_stack->size(); } + VirtualRange& kernel_stack() { return *m_kernel_stack; } - vaddr_t interrupt_stack_base() const { return m_interrupt_stack ? m_interrupt_stack->vaddr() : 0; } - size_t interrupt_stack_size() const { return m_interrupt_stack ? m_interrupt_stack->size() : 0; } + vaddr_t userspace_stack_bottom() const { return is_userspace() ? m_userspace_stack->vaddr() : 0; } + vaddr_t userspace_stack_top() const { return is_userspace() ? m_userspace_stack->vaddr() + m_userspace_stack->size() : 0; } + VirtualRange& userspace_stack() { ASSERT(is_userspace()); return *m_userspace_stack; } static Thread& current(); static pid_t current_tid(); @@ -84,7 +84,7 @@ namespace Kernel bool is_userspace() const { return m_is_userspace; } - size_t virtual_page_count() const { return m_stack->size() / PAGE_SIZE; } + size_t virtual_page_count() const { return (m_kernel_stack->size() / PAGE_SIZE) + (m_userspace_stack->size() / PAGE_SIZE); } size_t physical_page_count() const { return virtual_page_count(); } #if __enable_sse @@ -101,10 +101,9 @@ namespace Kernel private: static constexpr size_t m_kernel_stack_size = PAGE_SIZE * 4; - static constexpr size_t m_userspace_stack_size = PAGE_SIZE * 2; - static constexpr size_t m_interrupt_stack_size = PAGE_SIZE * 2; - BAN::UniqPtr m_interrupt_stack; - BAN::UniqPtr m_stack; + static constexpr size_t m_userspace_stack_size = PAGE_SIZE * 4; + BAN::UniqPtr m_kernel_stack; + BAN::UniqPtr m_userspace_stack; uintptr_t m_ip { 0 }; uintptr_t m_sp { 0 }; const pid_t m_tid { 0 }; diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index 1b8ea33d93..2d52b2a0e3 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -179,20 +179,19 @@ namespace Kernel if (isr == ISR::PageFault) { // Check if stack is OOB - auto& stack = Thread::current().stack(); - auto& istack = Thread::current().interrupt_stack(); - if (stack.vaddr() < interrupt_stack->sp && interrupt_stack->sp <= stack.vaddr() + stack.size()) - ; // using normal stack - else if (istack.vaddr() < interrupt_stack->sp && interrupt_stack->sp <= istack.vaddr() + istack.size()) - ; // using interrupt stack + auto& thread = Thread::current(); + if (thread.userspace_stack_bottom() < interrupt_stack->sp && interrupt_stack->sp <= thread.userspace_stack_top()) + ; // using userspace stack + else if (thread.kernel_stack_bottom() < interrupt_stack->sp && interrupt_stack->sp <= thread.kernel_stack_top()) + ; // using kernel stack else { derrorln("Stack pointer out of bounds!"); derrorln("rip {H}", interrupt_stack->ip); - derrorln("rsp {H}, stack {H}->{H}, istack {H}->{H}", + derrorln("rsp {H}, userspace stack {H}->{H}, kernel stack {H}->{H}", interrupt_stack->sp, - stack.vaddr(), stack.vaddr() + stack.size(), - istack.vaddr(), istack.vaddr() + istack.size() + thread.userspace_stack_bottom(), thread.userspace_stack_top(), + thread.kernel_stack_bottom(), thread.kernel_stack_top() ); Thread::current().handle_signal(SIGKILL); goto done; diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 2d757a10c8..927ac3fa95 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -676,9 +676,9 @@ namespace Kernel LockGuard _(m_process_lock); - if (Thread::current().stack().contains(address)) + if (Thread::current().userspace_stack().contains(address)) { - TRY(Thread::current().stack().allocate_page_for_demand_paging(address)); + TRY(Thread::current().userspace_stack().allocate_page_for_demand_paging(address)); return true; } @@ -1879,7 +1879,7 @@ namespace Kernel if (vaddr == 0) return {}; - if (vaddr >= thread.stack_base() && vaddr + size <= thread.stack_base() + thread.stack_size()) + if (vaddr >= thread.userspace_stack_bottom() && vaddr + size <= thread.userspace_stack_top()) return {}; // FIXME: should we allow cross mapping access? diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index a2440e963d..cf59a8a304 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -251,7 +251,7 @@ namespace Kernel if (current->has_process()) { current->process().page_table().load(); - Processor::gdt().set_tss_stack(current->interrupt_stack_base() + current->interrupt_stack_size()); + Processor::gdt().set_tss_stack(current->kernel_stack_top()); } else PageTable::kernel().load(); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 413335adb6..7a38469dfa 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -38,7 +38,7 @@ namespace Kernel BAN::ScopeGuard thread_deleter([thread] { delete thread; }); // Initialize stack and registers - thread->m_stack = TRY(VirtualRange::create_to_vaddr_range( + thread->m_kernel_stack = TRY(VirtualRange::create_to_vaddr_range( PageTable::kernel(), KERNEL_OFFSET, ~(uintptr_t)0, @@ -46,7 +46,7 @@ namespace Kernel PageTable::Flags::ReadWrite | PageTable::Flags::Present, true )); - thread->m_sp = thread->stack_base() + thread->stack_size(); + thread->m_sp = thread->kernel_stack_top(); thread->m_ip = (uintptr_t)entry; // Initialize stack for returning @@ -72,7 +72,7 @@ namespace Kernel thread->m_is_userspace = true; - thread->m_stack = TRY(VirtualRange::create_to_vaddr_range( + thread->m_userspace_stack = TRY(VirtualRange::create_to_vaddr_range( process->page_table(), 0x300000, KERNEL_OFFSET, m_userspace_stack_size, @@ -80,10 +80,10 @@ namespace Kernel true )); - thread->m_interrupt_stack = TRY(VirtualRange::create_to_vaddr_range( + thread->m_kernel_stack = TRY(VirtualRange::create_to_vaddr_range( process->page_table(), 0x300000, KERNEL_OFFSET, - m_interrupt_stack_size, + m_kernel_stack_size, PageTable::Flags::ReadWrite | PageTable::Flags::Present, true )); @@ -162,8 +162,8 @@ namespace Kernel thread->m_is_userspace = true; - thread->m_interrupt_stack = TRY(m_interrupt_stack->clone(new_process->page_table())); - thread->m_stack = TRY(m_stack->clone(new_process->page_table())); + thread->m_kernel_stack = TRY(m_kernel_stack->clone(new_process->page_table())); + thread->m_userspace_stack = TRY(m_userspace_stack->clone(new_process->page_table())); thread->m_state = State::Executing; @@ -183,11 +183,11 @@ namespace Kernel [](void*) { const auto& info = Process::current().userspace_info(); - thread_userspace_trampoline(Thread::current().sp(), info.entry, info.argc, info.argv, info.envp); + thread_userspace_trampoline(Thread::current().userspace_stack_top(), info.entry, info.argc, info.argv, info.envp); ASSERT_NOT_REACHED(); } ); - m_sp = stack_base() + stack_size(); + m_sp = kernel_stack_top(); m_ip = (uintptr_t)entry_trampoline; // Signal mask is inherited @@ -216,7 +216,7 @@ namespace Kernel ASSERT_NOT_REACHED(); } ); - m_sp = stack_base() + stack_size(); + m_sp = kernel_stack_top(); m_ip = (uintptr_t)entry; m_signal_pending_mask = 0; @@ -244,7 +244,7 @@ namespace Kernel { if (!is_userspace() || m_state != State::Executing) return false; - auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); + auto& interrupt_stack = *reinterpret_cast(kernel_stack_top() - sizeof(InterruptStack)); if (!GDT::is_user_segment(interrupt_stack.cs)) return false; uint64_t full_pending_mask = m_signal_pending_mask | process().signal_pending_mask();; @@ -255,7 +255,7 @@ namespace Kernel { if (!is_userspace() || m_state != State::Executing) return false; - auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); + auto& interrupt_stack = *reinterpret_cast(kernel_stack_top() - sizeof(InterruptStack)); return interrupt_stack.ip == (uintptr_t)signal_trampoline; } @@ -266,7 +266,7 @@ namespace Kernel SpinLockGuard _(m_signal_lock); - auto& interrupt_stack = *reinterpret_cast(interrupt_stack_base() + interrupt_stack_size() - sizeof(InterruptStack)); + auto& interrupt_stack = *reinterpret_cast(kernel_stack_top() - sizeof(InterruptStack)); ASSERT(GDT::is_user_segment(interrupt_stack.cs)); if (signal == 0) @@ -398,13 +398,13 @@ namespace Kernel void Thread::validate_stack() const { - if (stack_base() <= m_sp && m_sp <= stack_base() + stack_size()) + if (kernel_stack_bottom() <= m_sp && m_sp <= kernel_stack_top()) return; - if (interrupt_stack_base() <= m_sp && m_sp <= interrupt_stack_base() + interrupt_stack_size()) + if (userspace_stack_bottom() <= m_sp && m_sp <= userspace_stack_top()) return; - Kernel::panic("sp {8H}, stack {8H}->{8H}, interrupt_stack {8H}->{8H}", m_sp, - stack_base(), stack_base() + stack_size(), - interrupt_stack_base(), interrupt_stack_base() + interrupt_stack_size() + Kernel::panic("sp {8H}, kernel stack {8H}->{8H}, userspace stack {8H}->{8H}", m_sp, + kernel_stack_bottom(), kernel_stack_top(), + userspace_stack_bottom(), userspace_stack_top() ); } From 5050047cef0683e16fc67c054a9e362a6580e850 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Fri, 29 Mar 2024 18:02:12 +0200 Subject: [PATCH 36/46] Kernel: Rewrite whole scheduler Current context saving was very hacky and dependant on compiler behaviour that was not consistent. Now we always use iret for context saving. This makes everything more clean. --- kernel/arch/x86_64/Thread.S | 45 +--- kernel/arch/x86_64/interrupts.S | 14 +- kernel/include/kernel/InterruptStack.h | 20 ++ kernel/include/kernel/Lock/Mutex.h | 4 +- kernel/include/kernel/Processor.h | 9 + kernel/include/kernel/Scheduler.h | 13 +- kernel/include/kernel/SchedulerQueue.h | 1 + kernel/include/kernel/Thread.h | 29 +- kernel/kernel/IDT.cpp | 31 +-- kernel/kernel/Networking/ARPTable.cpp | 2 +- kernel/kernel/Networking/UNIX/Socket.cpp | 2 +- kernel/kernel/Process.cpp | 30 ++- kernel/kernel/Processor.cpp | 30 +++ kernel/kernel/Scheduler.cpp | 312 +++++++++------------- kernel/kernel/Storage/ATA/AHCI/Device.cpp | 2 +- kernel/kernel/Syscall.cpp | 3 - kernel/kernel/Thread.cpp | 146 ++++++---- libc/arch/x86_64/crt0.S | 35 ++- 18 files changed, 364 insertions(+), 364 deletions(-) diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index de1e59ab52..e7ec70154a 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -4,36 +4,19 @@ read_ip: popq %rax jmp *%rax -exit_thread_trampoline: +# void start_thread() +.global start_kernel_thread +start_kernel_thread: + # STACK LAYOUT + # on_exit arg + # on_exit func + # entry arg + # entry func + movq 8(%rsp), %rdi - ret + movq 0(%rsp), %rsi + call *%rsi -# void start_thread(uint64_t sp, uint64_t ip) -.global start_thread -start_thread: - movq %rdi, %rsp - popq %rdi - movq $0, %rbp - pushq $exit_thread_trampoline - sti - jmp *%rsi - -# void continue_thread(uint64_t sp, uint64_t ip) -.global continue_thread -continue_thread: - movq %rdi, %rsp - movq $0, %rax - jmp *%rsi - -# void thread_userspace_trampoline(uint64_t sp, uint64_t ip, int argc, char** argv, char** envp) -.global thread_userspace_trampoline -thread_userspace_trampoline: - pushq $0x23 - pushq %rdi - pushfq - pushq $0x1B - pushq %rsi - movq %rdx, %rdi - movq %rcx, %rsi - movq %r8, %rdx - iretq + movq 24(%rsp), %rdi + movq 16(%rsp), %rsi + call *%rsi diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index a9aabc053f..7d3e5f4d9a 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -72,9 +72,7 @@ isr_stub: irq_stub: pushaq - movq 0x78(%rsp), %rdi # irq number - movq %rsp, %rsi - addq $136, %rsi + movq 120(%rsp), %rdi # irq number call cpp_irq_handler popaq addq $16, %rsp @@ -168,7 +166,15 @@ irq 28 irq 29 irq 30 irq 31 -irq 32 + +.global asm_reschedule_handler +asm_reschedule_handler: + pushaq + leaq 120(%rsp), %rdi # interrupt stack ptr + movq %rsp, %rsi # interrupt register ptr + call cpp_reschedule_handler + popaq + iretq // arguments in RAX, RBX, RCX, RDX, RSI, RDI // System V ABI: RDI, RSI, RDX, RCX, R8, R9 diff --git a/kernel/include/kernel/InterruptStack.h b/kernel/include/kernel/InterruptStack.h index 6a112874e4..cd91c0fb77 100644 --- a/kernel/include/kernel/InterruptStack.h +++ b/kernel/include/kernel/InterruptStack.h @@ -14,4 +14,24 @@ namespace Kernel uintptr_t ss; }; + struct InterruptRegisters + { + uintptr_t r15; + uintptr_t r14; + uintptr_t r13; + uintptr_t r12; + uintptr_t r11; + uintptr_t r10; + uintptr_t r9; + uintptr_t r8; + + uintptr_t rdi; + uintptr_t rsi; + uintptr_t rbp; + uintptr_t rbx; + uintptr_t rdx; + uintptr_t rcx; + uintptr_t rax; + }; + } diff --git a/kernel/include/kernel/Lock/Mutex.h b/kernel/include/kernel/Lock/Mutex.h index 257c10509b..a4c895a224 100644 --- a/kernel/include/kernel/Lock/Mutex.h +++ b/kernel/include/kernel/Lock/Mutex.h @@ -25,7 +25,7 @@ namespace Kernel else { while (!m_locker.compare_exchange(-1, tid)) - Scheduler::get().reschedule(); + Scheduler::get().yield(); ASSERT(m_lock_depth == 0); } m_lock_depth++; @@ -81,7 +81,7 @@ namespace Kernel if (has_priority) m_queue_length++; while (!(has_priority || m_queue_length == 0) || !m_locker.compare_exchange(-1, tid)) - Scheduler::get().reschedule(); + Scheduler::get().yield(); ASSERT(m_lock_depth == 0); } m_lock_depth++; diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index f01f116320..c9d6234c5f 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace Kernel @@ -68,6 +69,11 @@ namespace Kernel static SchedulerQueue::Node* get_current_thread() { return reinterpret_cast(read_gs_ptr(offsetof(Processor, m_current_thread))); } static void set_current_thread(SchedulerQueue::Node* thread) { write_gs_ptr(offsetof(Processor, m_current_thread), thread); } + static void enter_interrupt(InterruptStack*, InterruptRegisters*); + static void leave_interrupt(); + static InterruptStack& get_interrupt_stack(); + static InterruptRegisters& get_interrupt_registers(); + private: Processor() = default; ~Processor() { ASSERT_NOT_REACHED(); } @@ -121,6 +127,9 @@ namespace Kernel Thread* m_idle_thread { nullptr }; SchedulerQueue::Node* m_current_thread { nullptr }; + InterruptStack* m_interrupt_stack { nullptr }; + InterruptRegisters* m_interrupt_registers { nullptr }; + void* m_current_page_table { nullptr }; friend class BAN::Array; diff --git a/kernel/include/kernel/Scheduler.h b/kernel/include/kernel/Scheduler.h index c2091d68e3..2784d269b3 100644 --- a/kernel/include/kernel/Scheduler.h +++ b/kernel/include/kernel/Scheduler.h @@ -16,8 +16,10 @@ namespace Kernel [[noreturn]] void start(); + void yield(); + void timer_reschedule(); - void reschedule(); + void irq_reschedule(); void reschedule_if_idling(); void set_current_thread_sleeping(uint64_t wake_time); @@ -30,9 +32,6 @@ namespace Kernel Thread& current_thread(); static pid_t current_tid(); - [[noreturn]] void execute_current_thread(); - [[noreturn]] void delete_current_process_and_thread(); - // This is no return if called on current thread void terminate_thread(Thread*); @@ -41,11 +40,7 @@ namespace Kernel void set_current_thread_sleeping_impl(Semaphore* semaphore, uint64_t wake_time); - [[nodiscard]] bool save_current_thread(); - void advance_current_thread(); - - [[noreturn]] void execute_current_thread_locked(); - [[noreturn]] void execute_current_thread_stack_loaded(); + void setup_next_thread(); BAN::ErrorOr add_thread(Thread*); diff --git a/kernel/include/kernel/SchedulerQueue.h b/kernel/include/kernel/SchedulerQueue.h index c43455df3a..8c6d51a9aa 100644 --- a/kernel/include/kernel/SchedulerQueue.h +++ b/kernel/include/kernel/SchedulerQueue.h @@ -26,6 +26,7 @@ namespace Kernel Thread* thread; uint64_t wake_time { 0 }; Semaphore* semaphore { nullptr }; + bool should_block { false }; private: Node* next { nullptr }; diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index dd2e276360..fb3b0d799e 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -25,7 +26,7 @@ namespace Kernel { NotStarted, Executing, - Terminated + Terminated, }; public: @@ -52,19 +53,8 @@ namespace Kernel BAN::ErrorOr block_or_eintr_or_timeout(Semaphore& semaphore, uint64_t timeout_ms, bool etimedout); BAN::ErrorOr block_or_eintr_or_waketime(Semaphore& semaphore, uint64_t wake_time_ms, bool etimedout); - void set_return_sp(uintptr_t& sp) { m_return_sp = &sp; } - void set_return_ip(uintptr_t& ip) { m_return_ip = &ip; } - uintptr_t return_sp() { ASSERT(m_return_sp); return *m_return_sp; } - uintptr_t return_ip() { ASSERT(m_return_ip); return *m_return_ip; } - pid_t tid() const { return m_tid; } - void set_sp(uintptr_t sp) { m_sp = sp; validate_stack(); } - void set_ip(uintptr_t ip) { m_ip = ip; } - uintptr_t sp() const { return m_sp; } - uintptr_t ip() const { return m_ip; } - - void set_started() { ASSERT(m_state == State::NotStarted); m_state = State::Executing; } State state() const { return m_state; } vaddr_t kernel_stack_bottom() const { return m_kernel_stack->vaddr(); } @@ -87,6 +77,10 @@ namespace Kernel size_t virtual_page_count() const { return (m_kernel_stack->size() / PAGE_SIZE) + (m_userspace_stack->size() / PAGE_SIZE); } size_t physical_page_count() const { return virtual_page_count(); } + uintptr_t& interrupt_sp() { return m_interrupt_sp; } + InterruptStack& interrupt_stack() { return m_interrupt_stack; } + InterruptRegisters& interrupt_registers() { return m_interrupt_registers; } + #if __enable_sse void save_sse(); void load_sse(); @@ -97,22 +91,20 @@ namespace Kernel Thread(pid_t tid, Process*); void on_exit(); - void validate_stack() const; - private: static constexpr size_t m_kernel_stack_size = PAGE_SIZE * 4; static constexpr size_t m_userspace_stack_size = PAGE_SIZE * 4; BAN::UniqPtr m_kernel_stack; BAN::UniqPtr m_userspace_stack; - uintptr_t m_ip { 0 }; - uintptr_t m_sp { 0 }; const pid_t m_tid { 0 }; State m_state { State::NotStarted }; Process* m_process { nullptr }; bool m_is_userspace { false }; + bool m_delete_process { false }; - uintptr_t* m_return_sp { nullptr }; - uintptr_t* m_return_ip { nullptr }; + InterruptStack m_interrupt_stack { }; + InterruptRegisters m_interrupt_registers { }; + uintptr_t m_interrupt_sp { }; uint64_t m_signal_pending_mask { 0 }; uint64_t m_signal_block_mask { 0 }; @@ -123,6 +115,7 @@ namespace Kernel alignas(16) uint8_t m_sse_storage[512] {}; #endif + friend class Process; friend class Scheduler; }; diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index 2d52b2a0e3..434aef33e0 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -10,7 +10,7 @@ #include #define ISR_LIST_X X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19) X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29) X(30) X(31) -#define IRQ_LIST_X X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19) X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29) X(30) X(31) X(32) +#define IRQ_LIST_X X(0) X(1) X(2) X(3) X(4) X(5) X(6) X(7) X(8) X(9) X(10) X(11) X(12) X(13) X(14) X(15) X(16) X(17) X(18) X(19) X(20) X(21) X(22) X(23) X(24) X(25) X(26) X(27) X(28) X(29) X(30) X(31) namespace Kernel { @@ -173,9 +173,6 @@ namespace Kernel if (tid) { - Thread::current().set_return_sp(interrupt_stack->sp); - Thread::current().set_return_ip(interrupt_stack->ip); - if (isr == ISR::PageFault) { // Check if stack is OOB @@ -202,9 +199,9 @@ namespace Kernel page_fault_error.raw = error; if (!page_fault_error.present) { - asm volatile("sti"); + Processor::set_interrupt_state(InterruptState::Enabled); auto result = Process::current().allocate_page_for_demand_paging(regs->cr2); - asm volatile("cli"); + Processor::set_interrupt_state(InterruptState::Disabled); if (!result.is_error() && result.value()) goto done; @@ -332,7 +329,14 @@ done: return; } - extern "C" void cpp_irq_handler(uint32_t irq, InterruptStack* interrupt_stack) + extern "C" void cpp_reschedule_handler(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) + { + Processor::enter_interrupt(interrupt_stack, interrupt_registers); + Scheduler::get().irq_reschedule(); + Processor::leave_interrupt(); + } + + extern "C" void cpp_irq_handler(uint32_t irq) { if (g_paniced) { @@ -342,20 +346,14 @@ done: asm volatile("cli; 1: hlt; jmp 1b"); } - if (Scheduler::current_tid()) - { - Thread::current().set_return_sp(interrupt_stack->sp); - Thread::current().set_return_ip(interrupt_stack->ip); - } + ASSERT(irq != IRQ_IPI); if (!InterruptController::get().is_in_service(irq)) dprintln("spurious irq 0x{2H}", irq); else { InterruptController::get().eoi(irq); - if (irq == IRQ_IPI) - Scheduler::get().reschedule(); - else if (auto* handler = s_interruptables[irq]) + if (auto* handler = s_interruptables[irq]) handler->handle_irq(); else dprintln("no handler for irq 0x{2H}", irq); @@ -402,6 +400,7 @@ done: IRQ_LIST_X #undef X + extern "C" void asm_reschedule_handler(); extern "C" void syscall_asm(); IDT* IDT::create() @@ -419,6 +418,8 @@ done: IRQ_LIST_X #undef X + idt->register_interrupt_handler(IRQ_VECTOR_BASE + IRQ_IPI, asm_reschedule_handler); + idt->register_syscall_handler(0x80, syscall_asm); return idt; diff --git a/kernel/kernel/Networking/ARPTable.cpp b/kernel/kernel/Networking/ARPTable.cpp index bdf62f88b2..38deb15285 100644 --- a/kernel/kernel/Networking/ARPTable.cpp +++ b/kernel/kernel/Networking/ARPTable.cpp @@ -79,7 +79,7 @@ namespace Kernel if (it != m_arp_table.end()) return it->value; } - Scheduler::get().reschedule(); + Scheduler::get().yield(); } return BAN::Error::from_errno(ETIMEDOUT); diff --git a/kernel/kernel/Networking/UNIX/Socket.cpp b/kernel/kernel/Networking/UNIX/Socket.cpp index e986cbc10d..16a0e8cea3 100644 --- a/kernel/kernel/Networking/UNIX/Socket.cpp +++ b/kernel/kernel/Networking/UNIX/Socket.cpp @@ -164,7 +164,7 @@ namespace Kernel } while (!connection_info.connection_done) - Scheduler::get().reschedule(); + Scheduler::get().yield(); return {}; } diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 927ac3fa95..74bd73e489 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -127,9 +127,6 @@ namespace Kernel } process->m_loadable_elf->reserve_address_space(); - process->m_is_userspace = true; - process->m_userspace_info.entry = process->m_loadable_elf->entry_point(); - char** argv = nullptr; { size_t needed_bytes = sizeof(char*) * 2 + path.size() + 1; @@ -155,6 +152,8 @@ namespace Kernel MUST(process->m_mapped_regions.push_back(BAN::move(argv_region))); } + process->m_is_userspace = true; + process->m_userspace_info.entry = process->m_loadable_elf->entry_point(); process->m_userspace_info.argc = 1; process->m_userspace_info.argv = argv; process->m_userspace_info.envp = nullptr; @@ -207,7 +206,7 @@ namespace Kernel m_exit_status.semaphore.unblock(); while (m_exit_status.waiting > 0) - Scheduler::get().reschedule(); + Scheduler::get().yield(); m_process_lock.lock(); @@ -220,7 +219,7 @@ namespace Kernel bool Process::on_thread_exit(Thread& thread) { - ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); + LockGuard _(m_process_lock); ASSERT(m_threads.size() > 0); @@ -228,8 +227,6 @@ namespace Kernel { ASSERT(m_threads.front() == &thread); m_threads.clear(); - - thread.setup_process_cleanup(); return true; } @@ -248,11 +245,18 @@ namespace Kernel void Process::exit(int status, int signal) { m_exit_status.exit_code = __WGENEXITCODE(status, signal); - for (auto* thread : m_threads) - if (thread != &Thread::current()) - Scheduler::get().terminate_thread(thread); - if (this == &Process::current()) - Scheduler::get().terminate_thread(&Thread::current()); + while (!m_threads.empty()) + m_threads.front()->on_exit(); + //for (auto* thread : m_threads) + // if (thread != &Thread::current()) + // Scheduler::get().terminate_thread(thread); + //if (this == &Process::current()) + //{ + // m_threads.clear(); + // Processor::set_interrupt_state(InterruptState::Disabled); + // Thread::current().setup_process_cleanup(); + // Scheduler::get().yield(); + //} } size_t Process::proc_meminfo(off_t offset, BAN::ByteSpan buffer) const @@ -533,7 +537,7 @@ namespace Kernel m_has_called_exec = true; m_threads.front()->setup_exec(); - Scheduler::get().execute_current_thread(); + Scheduler::get().yield(); ASSERT_NOT_REACHED(); } diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 2fbab9b826..12be1c2286 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -79,4 +79,34 @@ namespace Kernel write_gs_ptr(offsetof(Processor, m_idle_thread), idle_thread); } + void Processor::enter_interrupt(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) + { + ASSERT(get_interrupt_state() == InterruptState::Disabled); + ASSERT(read_gs_ptr(offsetof(Processor, m_interrupt_stack)) == nullptr); + write_gs_ptr(offsetof(Processor, m_interrupt_stack), interrupt_stack); + write_gs_ptr(offsetof(Processor, m_interrupt_registers), interrupt_registers); + } + + void Processor::leave_interrupt() + { + ASSERT(get_interrupt_state() == InterruptState::Disabled); + ASSERT(read_gs_ptr(offsetof(Processor, m_interrupt_stack)) != nullptr); + write_gs_ptr(offsetof(Processor, m_interrupt_stack), nullptr); + write_gs_ptr(offsetof(Processor, m_interrupt_registers), nullptr); + } + + InterruptStack& Processor::get_interrupt_stack() + { + ASSERT(get_interrupt_state() == InterruptState::Disabled); + ASSERT(read_gs_ptr(offsetof(Processor, m_interrupt_stack))); + return *read_gs_sized(offsetof(Processor, m_interrupt_stack)); + } + + InterruptRegisters& Processor::get_interrupt_registers() + { + ASSERT(get_interrupt_state() == InterruptState::Disabled); + ASSERT(read_gs_ptr(offsetof(Processor, m_interrupt_registers))); + return *read_gs_sized(offsetof(Processor, m_interrupt_registers)); + } + } diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index cf59a8a304..4e33af3343 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -11,9 +11,6 @@ namespace Kernel { - extern "C" [[noreturn]] void start_thread(uintptr_t sp, uintptr_t ip); - extern "C" [[noreturn]] void continue_thread(uintptr_t sp, uintptr_t ip); - static Scheduler* s_instance = nullptr; static BAN::Atomic s_started { false }; @@ -46,10 +43,8 @@ namespace Kernel void Scheduler::start() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); - m_lock.lock(); - s_started = true; - advance_current_thread(); - execute_current_thread_locked(); + ASSERT(!m_active_threads.empty()); + yield(); ASSERT_NOT_REACHED(); } @@ -71,41 +66,125 @@ namespace Kernel return Scheduler::get().current_thread().tid(); } + void Scheduler::setup_next_thread() + { + ASSERT(m_lock.current_processor_has_lock()); + + if (auto* current = Processor::get_current_thread()) + { + auto* thread = current->thread; + + if (thread->state() == Thread::State::Terminated) + { + PageTable::kernel().load(); + delete thread; + delete current; + } + else + { + // thread->state() can be NotStarted when calling exec or cleaning up process + if (thread->state() != Thread::State::NotStarted) + { + thread->interrupt_stack() = Processor::get_interrupt_stack(); + thread->interrupt_stack().sp = thread->interrupt_sp(); + thread->interrupt_registers() = Processor::get_interrupt_registers(); + } + + if (current->should_block) + { + current->should_block = false; + m_blocking_threads.add_with_wake_time(current); + } + else + { + m_active_threads.push_back(current); + } + } + } + + SchedulerQueue::Node* node = nullptr; + while (!m_active_threads.empty()) + { + node = m_active_threads.pop_front(); + if (node->thread->state() != Thread::State::Terminated) + break; + + PageTable::kernel().load(); + delete node->thread; + delete node; + node = nullptr; + } + + Processor::set_current_thread(node); + + auto* thread = node ? node->thread : Processor::idle_thread(); + + if (thread->has_process()) + thread->process().page_table().load(); + else + PageTable::kernel().load(); + + if (thread->state() == Thread::State::NotStarted) + thread->m_state = Thread::State::Executing; + + ASSERT(thread->interrupt_stack().ip); + ASSERT(thread->interrupt_stack().sp); + + Processor::gdt().set_tss_stack(thread->kernel_stack_top()); + + Processor::get_interrupt_stack() = thread->interrupt_stack(); + Processor::get_interrupt_registers() = thread->interrupt_registers(); + } + void Scheduler::timer_reschedule() { // Broadcast IPI to all other processors for them // to perform reschedule InterruptController::get().broadcast_ipi(); - auto state = m_lock.lock(); - m_blocking_threads.remove_with_wake_time(m_active_threads, SystemTimer::get().ms_since_boot()); - if (save_current_thread()) - return Processor::set_interrupt_state(state); - advance_current_thread(); - execute_current_thread_locked(); - ASSERT_NOT_REACHED(); + { + SpinLockGuard _(m_lock); + m_blocking_threads.remove_with_wake_time(m_active_threads, SystemTimer::get().ms_since_boot()); + } + + yield(); } - void Scheduler::reschedule() + void Scheduler::yield() { - auto state = m_lock.lock(); - if (save_current_thread()) - return Processor::set_interrupt_state(state); - advance_current_thread(); - execute_current_thread_locked(); - ASSERT_NOT_REACHED(); + auto state = Processor::get_interrupt_state(); + Processor::set_interrupt_state(InterruptState::Disabled); + + asm volatile( + "movq %%rsp, %[save_sp];" + "movq %[load_sp], %%rsp;" + "int %[ipi];" + : [save_sp]"=m"(Thread::current().interrupt_sp()) + : [load_sp]"r"(Processor::current_stack_top()), + [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI) + : "memory" + ); + + Processor::set_interrupt_state(state); + } + + void Scheduler::irq_reschedule() + { + SpinLockGuard _(m_lock); + setup_next_thread(); } void Scheduler::reschedule_if_idling() { - auto state = m_lock.lock(); - if (m_active_threads.empty() || Processor::get_current_thread()) - return m_lock.unlock(state); - if (save_current_thread()) - return Processor::set_interrupt_state(state); - advance_current_thread(); - execute_current_thread_locked(); - ASSERT_NOT_REACHED(); + { + SpinLockGuard _(m_lock); + if (Processor::get_current_thread()) + return; + if (m_active_threads.empty()) + return; + } + + yield(); } BAN::ErrorOr Scheduler::add_thread(Thread* thread) @@ -120,190 +199,49 @@ namespace Kernel void Scheduler::terminate_thread(Thread* thread) { - SpinLockGuard _(m_lock); + auto state = m_lock.lock(); + + ASSERT(thread->state() == Thread::State::Executing); thread->m_state = Thread::State::Terminated; - if (thread == ¤t_thread()) - execute_current_thread_locked(); - } + thread->interrupt_stack().sp = Processor::current_stack_top(); - void Scheduler::advance_current_thread() - { - ASSERT(m_lock.current_processor_has_lock()); + m_lock.unlock(InterruptState::Disabled); - if (auto* current = Processor::get_current_thread()) - m_active_threads.push_back(current); - Processor::set_current_thread(nullptr); + // actual deletion will be done while rescheduling - if (!m_active_threads.empty()) - Processor::set_current_thread(m_active_threads.pop_front()); - } - - // NOTE: this is declared always inline, so we don't corrupt the stack - // after getting the rsp - ALWAYS_INLINE bool Scheduler::save_current_thread() - { - ASSERT(m_lock.current_processor_has_lock()); - - uintptr_t sp, ip; - push_callee_saved(); - if (!(ip = read_ip())) + if (¤t_thread() == thread) { - pop_callee_saved(); - return true; - } - read_rsp(sp); - - Thread& current = current_thread(); - current.set_ip(ip); - current.set_sp(sp); - - load_temp_stack(); - - return false; - } - - void Scheduler::delete_current_process_and_thread() - { - m_lock.lock(); - - load_temp_stack(); - PageTable::kernel().load(); - - auto* current = Processor::get_current_thread(); - ASSERT(current); - delete ¤t->thread->process(); - delete current->thread; - delete current; - Processor::set_current_thread(nullptr); - - advance_current_thread(); - execute_current_thread_locked(); - ASSERT_NOT_REACHED(); - } - - void Scheduler::execute_current_thread() - { - m_lock.lock(); - load_temp_stack(); - PageTable::kernel().load(); - execute_current_thread_stack_loaded(); - ASSERT_NOT_REACHED(); - } - - void Scheduler::execute_current_thread_locked() - { - ASSERT(m_lock.current_processor_has_lock()); - load_temp_stack(); - PageTable::kernel().load(); - execute_current_thread_stack_loaded(); - ASSERT_NOT_REACHED(); - } - - NEVER_INLINE void Scheduler::execute_current_thread_stack_loaded() - { - ASSERT(m_lock.current_processor_has_lock()); - -#if SCHEDULER_VERIFY_STACK - vaddr_t rsp; - read_rsp(rsp); - ASSERT(Processor::current_stack_bottom() <= rsp && rsp <= Processor::current_stack_top()); - ASSERT(&PageTable::current() == &PageTable::kernel()); -#endif - - Thread* current = ¤t_thread(); - -#if __enable_sse - if (current != Thread::sse_thread()) - { -#if ARCH(x86_64) - asm volatile( - "movq %cr0, %rax;" - "orq $(1 << 3), %rax;" - "movq %rax, %cr0" - ); -#elif ARCH(i686) - asm volatile( - "movl %cr0, %eax;" - "orl $(1 << 3), %eax;" - "movl %eax, %cr0" - ); -#else - #error -#endif - } -#endif - - while (current->state() == Thread::State::Terminated) - { - auto* node = Processor::get_current_thread(); - if (node->thread->has_process()) - if (node->thread->process().on_thread_exit(*node->thread)) - break; - - delete node->thread; - delete node; - Processor::set_current_thread(nullptr); - - advance_current_thread(); - current = ¤t_thread(); + yield(); + ASSERT_NOT_REACHED(); } - if (current->has_process()) - { - current->process().page_table().load(); - Processor::gdt().set_tss_stack(current->kernel_stack_top()); - } - else - PageTable::kernel().load(); - - switch (current->state()) - { - case Thread::State::NotStarted: - current->set_started(); - m_lock.unlock(InterruptState::Disabled); - start_thread(current->sp(), current->ip()); - case Thread::State::Executing: - m_lock.unlock(InterruptState::Disabled); - while (current->can_add_signal_to_execute()) - current->handle_signal(); - continue_thread(current->sp(), current->ip()); - case Thread::State::Terminated: - ASSERT_NOT_REACHED(); - } - - ASSERT_NOT_REACHED(); + Processor::set_interrupt_state(state); } void Scheduler::set_current_thread_sleeping_impl(Semaphore* semaphore, uint64_t wake_time) { - ASSERT(m_lock.current_processor_has_lock()); - - if (save_current_thread()) - return; + auto state = m_lock.lock(); auto* current = Processor::get_current_thread(); current->semaphore = semaphore; current->wake_time = wake_time; - m_blocking_threads.add_with_wake_time(current); - Processor::set_current_thread(nullptr); + current->should_block = true; - advance_current_thread(); - execute_current_thread_locked(); - ASSERT_NOT_REACHED(); + m_lock.unlock(InterruptState::Disabled); + + yield(); + + Processor::set_interrupt_state(state); } void Scheduler::set_current_thread_sleeping(uint64_t wake_time) { - auto state = m_lock.lock(); set_current_thread_sleeping_impl(nullptr, wake_time); - Processor::set_interrupt_state(state); } void Scheduler::block_current_thread(Semaphore* semaphore, uint64_t wake_time) { - auto state = m_lock.lock(); set_current_thread_sleeping_impl(semaphore, wake_time); - Processor::set_interrupt_state(state); } void Scheduler::unblock_threads(Semaphore* semaphore) diff --git a/kernel/kernel/Storage/ATA/AHCI/Device.cpp b/kernel/kernel/Storage/ATA/AHCI/Device.cpp index c671f6337d..1bc945a832 100644 --- a/kernel/kernel/Storage/ATA/AHCI/Device.cpp +++ b/kernel/kernel/Storage/ATA/AHCI/Device.cpp @@ -168,7 +168,7 @@ namespace Kernel // This doesn't allow scheduler to go properly idle. while (SystemTimer::get().ms_since_boot() < start_time + s_ata_timeout) { - Scheduler::get().reschedule(); + Scheduler::get().yield(); if (!(m_port->ci & (1 << command_slot))) return {}; } diff --git a/kernel/kernel/Syscall.cpp b/kernel/kernel/Syscall.cpp index c941bb4db8..22ca55e161 100644 --- a/kernel/kernel/Syscall.cpp +++ b/kernel/kernel/Syscall.cpp @@ -32,9 +32,6 @@ namespace Kernel { ASSERT((interrupt_stack.cs & 0b11) == 0b11); - Thread::current().set_return_sp(interrupt_stack.sp); - Thread::current().set_return_ip(interrupt_stack.ip); - asm volatile("sti"); BAN::ErrorOr ret = BAN::Error::from_errno(ENOSYS); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 7a38469dfa..a7ad2e22f9 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -12,8 +12,8 @@ namespace Kernel { - extern "C" void thread_userspace_trampoline(uint64_t sp, uint64_t ip, int argc, char** argv, char** envp); - extern "C" uintptr_t read_ip(); + extern "C" [[noreturn]] void start_userspace_thread(); + extern "C" [[noreturn]] void start_kernel_thread(); extern "C" void signal_trampoline(); @@ -46,14 +46,21 @@ namespace Kernel PageTable::Flags::ReadWrite | PageTable::Flags::Present, true )); - thread->m_sp = thread->kernel_stack_top(); - thread->m_ip = (uintptr_t)entry; // Initialize stack for returning - write_to_stack(thread->m_sp, nullptr); // alignment - write_to_stack(thread->m_sp, thread); - write_to_stack(thread->m_sp, &Thread::on_exit); - write_to_stack(thread->m_sp, data); + uintptr_t sp = thread->kernel_stack_top(); + write_to_stack(sp, thread); + write_to_stack(sp, &Thread::on_exit); + write_to_stack(sp, data); + write_to_stack(sp, entry); + + thread->m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); + thread->m_interrupt_stack.cs = 0x08; + thread->m_interrupt_stack.flags = 0x202; + thread->m_interrupt_stack.sp = sp; + thread->m_interrupt_stack.ss = 0x10; + + memset(&thread->m_interrupt_registers, 0, sizeof(InterruptRegisters)); thread_deleter.disable(); @@ -72,14 +79,6 @@ namespace Kernel thread->m_is_userspace = true; - thread->m_userspace_stack = TRY(VirtualRange::create_to_vaddr_range( - process->page_table(), - 0x300000, KERNEL_OFFSET, - m_userspace_stack_size, - PageTable::Flags::UserSupervisor | PageTable::Flags::ReadWrite | PageTable::Flags::Present, - true - )); - thread->m_kernel_stack = TRY(VirtualRange::create_to_vaddr_range( process->page_table(), 0x300000, KERNEL_OFFSET, @@ -88,6 +87,14 @@ namespace Kernel true )); + thread->m_userspace_stack = TRY(VirtualRange::create_to_vaddr_range( + process->page_table(), + 0x300000, KERNEL_OFFSET, + m_userspace_stack_size, + PageTable::Flags::UserSupervisor | PageTable::Flags::ReadWrite | PageTable::Flags::Present, + true + )); + thread->setup_exec(); thread_deleter.disable(); @@ -148,6 +155,11 @@ namespace Kernel Thread::~Thread() { + if (m_delete_process) + { + ASSERT(m_process); + delete m_process; + } } BAN::ErrorOr Thread::clone(Process* new_process, uintptr_t sp, uintptr_t ip) @@ -165,10 +177,13 @@ namespace Kernel thread->m_kernel_stack = TRY(m_kernel_stack->clone(new_process->page_table())); thread->m_userspace_stack = TRY(m_userspace_stack->clone(new_process->page_table())); - thread->m_state = State::Executing; + thread->m_state = State::NotStarted; - thread->m_ip = ip; - thread->m_sp = sp; + thread->m_interrupt_stack.ip = ip; + thread->m_interrupt_stack.cs = 0x08; + thread->m_interrupt_stack.flags = 0x002; + thread->m_interrupt_stack.sp = sp; + thread->m_interrupt_stack.ss = 0x10; thread_deleter.disable(); @@ -179,58 +194,69 @@ namespace Kernel { ASSERT(is_userspace()); m_state = State::NotStarted; - static entry_t entry_trampoline( - [](void*) - { - const auto& info = Process::current().userspace_info(); - thread_userspace_trampoline(Thread::current().userspace_stack_top(), info.entry, info.argc, info.argv, info.envp); - ASSERT_NOT_REACHED(); - } - ); - m_sp = kernel_stack_top(); - m_ip = (uintptr_t)entry_trampoline; // Signal mask is inherited - // Setup stack for returning - ASSERT(m_sp % PAGE_SIZE == 0); - PageTable::with_fast_page(process().page_table().physical_address_of(m_sp - PAGE_SIZE), [&] { + auto& userspace_info = process().userspace_info(); + ASSERT(userspace_info.entry); + + // Initialize stack for returning + PageTable::with_fast_page(process().page_table().physical_address_of(userspace_stack_top() - PAGE_SIZE), [&] { uintptr_t sp = PageTable::fast_page() + PAGE_SIZE; - write_to_stack(sp, nullptr); // alignment - write_to_stack(sp, this); - write_to_stack(sp, &Thread::on_exit); write_to_stack(sp, nullptr); - m_sp -= 4 * sizeof(uintptr_t); + write_to_stack(sp, userspace_info.argc); + write_to_stack(sp, userspace_info.argv); + write_to_stack(sp, userspace_info.envp); }); + + m_interrupt_stack.ip = userspace_info.entry; + m_interrupt_stack.cs = 0x18 | 3; + m_interrupt_stack.flags = 0x202; + m_interrupt_stack.sp = userspace_stack_top() - 4 * sizeof(uintptr_t); + m_interrupt_stack.ss = 0x20 | 3; + + memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); } void Thread::setup_process_cleanup() { + ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); + m_state = State::NotStarted; static entry_t entry( [](void* process_ptr) { - auto& process = *reinterpret_cast(process_ptr); - process.cleanup_function(); - Scheduler::get().delete_current_process_and_thread(); - ASSERT_NOT_REACHED(); + auto* thread = &Thread::current(); + auto* process = static_cast(process_ptr); + + ASSERT(thread->m_process == process); + + process->cleanup_function(); + + thread->m_delete_process = true; + + // will call on thread exit after return } ); - m_sp = kernel_stack_top(); - m_ip = (uintptr_t)entry; m_signal_pending_mask = 0; m_signal_block_mask = ~0ull; - ASSERT(m_sp % PAGE_SIZE == 0); - PageTable::with_fast_page(process().page_table().physical_address_of(m_sp - PAGE_SIZE), [&] { + PageTable::with_fast_page(process().page_table().physical_address_of(kernel_stack_top() - PAGE_SIZE), [&] { uintptr_t sp = PageTable::fast_page() + PAGE_SIZE; - write_to_stack(sp, nullptr); // alignment write_to_stack(sp, this); write_to_stack(sp, &Thread::on_exit); write_to_stack(sp, m_process); - m_sp -= 4 * sizeof(uintptr_t); + write_to_stack(sp, entry); }); + + m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); + m_interrupt_stack.cs = 0x08; + m_interrupt_stack.flags = 0x202; + m_interrupt_stack.sp = kernel_stack_top() - 4 * sizeof(uintptr_t); + m_interrupt_stack.ss = 0x10; + + memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); } bool Thread::is_interrupted_by_signal() @@ -396,22 +422,24 @@ namespace Kernel return {}; } - void Thread::validate_stack() const - { - if (kernel_stack_bottom() <= m_sp && m_sp <= kernel_stack_top()) - return; - if (userspace_stack_bottom() <= m_sp && m_sp <= userspace_stack_top()) - return; - Kernel::panic("sp {8H}, kernel stack {8H}->{8H}, userspace stack {8H}->{8H}", m_sp, - kernel_stack_bottom(), kernel_stack_top(), - userspace_stack_bottom(), userspace_stack_top() - ); - } - void Thread::on_exit() { ASSERT(this == &Thread::current()); - Scheduler::get().terminate_thread(this); + if (!m_delete_process && has_process()) + { + if (process().on_thread_exit(*this)) + { + Processor::set_interrupt_state(InterruptState::Disabled); + setup_process_cleanup(); + Scheduler::get().yield(); + } + else + Scheduler::get().terminate_thread(this); + } + else + { + Scheduler::get().terminate_thread(this); + } ASSERT_NOT_REACHED(); } diff --git a/libc/arch/x86_64/crt0.S b/libc/arch/x86_64/crt0.S index 67804f1990..474a8bf4d4 100644 --- a/libc/arch/x86_64/crt0.S +++ b/libc/arch/x86_64/crt0.S @@ -2,34 +2,29 @@ .global _start _start: - # Set up end of the stack frame linked list. - movq $0, %rbp - pushq %rbp # rip=0 - pushq %rbp # rbp=0 - movq %rsp, %rbp + # STACK LAYOUT + # null + # argc + # argv + # envp - # Save argc, argv, environ - pushq %rdx - pushq %rsi - pushq %rdi + xorq %rbp, %rbp - # Prepare malloc, environment - movq %rdx, %rdi + # init libc + movq 0(%rsp), %rdi call _init_libc - # Call global constructos + # call global constructors call _init - # Restore argc, argv, environ - popq %rdi - popq %rsi - popq %rdx - - # Run main + # call main + movq 16(%rsp), %rdi + movq 8(%rsp), %rsi + movq 0(%rsp), %rdx call main - # Cleanly exit the process - movl %eax, %edi + # call exit + movq %rax, %rdi call exit .size _start, . - _start From 2106a9e3736b5d27a1bcabffe22ef8403d730fa6 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 2 Apr 2024 12:34:42 +0300 Subject: [PATCH 37/46] Kernel: Rework scheduler/processor stacks. --- kernel/arch/i686/Thread.S | 51 +++++++++++--------------- kernel/arch/i686/interrupts.S | 23 +++++++++--- kernel/arch/x86_64/Thread.S | 6 ++- kernel/include/kernel/InterruptStack.h | 14 +++++++ kernel/include/kernel/Thread.h | 4 +- kernel/kernel/Scheduler.cpp | 41 ++++++++++++--------- kernel/kernel/Thread.cpp | 19 +++++----- 7 files changed, 94 insertions(+), 64 deletions(-) diff --git a/kernel/arch/i686/Thread.S b/kernel/arch/i686/Thread.S index 42fc440071..c6f78c89dc 100644 --- a/kernel/arch/i686/Thread.S +++ b/kernel/arch/i686/Thread.S @@ -4,40 +4,31 @@ read_ip: popl %eax jmp *%eax -# void start_thread(uint32_t sp, uint32_t ip) -.global start_thread -start_thread: - movl 8(%esp), %edi # ip - movl 4(%esp), %esp # sp +# void start_kernel_thread() +.global start_kernel_thread +start_kernel_thread: + call get_start_kernel_thread_sp + movl %eax, %esp # STACK LAYOUT - # NULL - # thread ptr - # &Thread::on_exit - # data + # on_exit arg + # on_exit func + # entry arg + # entry func - xorl %ebp, %ebp + movl 4(%esp), %edi + movl 0(%esp), %esi + subl $12, %esp + pushl %edi sti - call *%edi + call *%esi + addl $16, %esp - movl 4(%esp), %edi # &Thread::on_exit + movl 12(%esp), %edi + movl 8(%esp), %esi - movl 8(%esp), %eax # thread ptr - movl %eax, (%esp) - - call *%edi - - -# void continue_thread(uint32_t sp, uint32_t ip) -.global continue_thread -continue_thread: - movl 8(%esp), %edi # ip - movl 4(%esp), %esp # sp - xorl %eax, %eax - jmp *%edi - -# void thread_userspace_trampoline(uint32_t sp, uint32_t ip, int argc, char** argv, char** envp) -.global thread_userspace_trampoline -thread_userspace_trampoline: - ud2 + subl $12, %esp + pushl %edi + call *%esi + addl $16, %esp diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index f48bcab18d..335dd45315 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -26,17 +26,31 @@ isr_stub: irq_stub: pusha - leal 40(%esp), %eax // interrupt stack ptr - movl 32(%esp), %ebx // irq number + movl 32(%esp), %eax # interrupt number + + subl $8, %esp + pushl %eax + call cpp_irq_handler + addl $12, %esp + + popa + addl $8, %esp + iret + +.global asm_reschedule_handler +asm_reschedule_handler: + pusha + + movl %esp, %eax # interrupt registers ptr + leal 32(%esp), %ebx # interrupt stack ptr subl $12, %esp pushl %eax pushl %ebx - call cpp_irq_handler + call cpp_reschedule_handler addl $20, %esp popa - addl $8, %esp iret // arguments in EAX, EBX, ECX, EDX, ESI, EDI @@ -158,4 +172,3 @@ irq 28 irq 29 irq 30 irq 31 -irq 32 diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index e7ec70154a..46eea1b08b 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -4,9 +4,12 @@ read_ip: popq %rax jmp *%rax -# void start_thread() +# void start_kernel_thread() .global start_kernel_thread start_kernel_thread: + call get_start_kernel_thread_sp + movq %rax, %rsp + # STACK LAYOUT # on_exit arg # on_exit func @@ -15,6 +18,7 @@ start_kernel_thread: movq 8(%rsp), %rdi movq 0(%rsp), %rsi + sti call *%rsi movq 24(%rsp), %rdi diff --git a/kernel/include/kernel/InterruptStack.h b/kernel/include/kernel/InterruptStack.h index cd91c0fb77..f08581ca91 100644 --- a/kernel/include/kernel/InterruptStack.h +++ b/kernel/include/kernel/InterruptStack.h @@ -14,6 +14,7 @@ namespace Kernel uintptr_t ss; }; +#if ARCH(x86_64) struct InterruptRegisters { uintptr_t r15; @@ -33,5 +34,18 @@ namespace Kernel uintptr_t rcx; uintptr_t rax; }; +#elif ARCH(i686) + struct InterruptRegisters + { + uintptr_t edi; + uintptr_t esi; + uintptr_t ebp; + uintptr_t unused; + uintptr_t ebx; + uintptr_t edx; + uintptr_t ecx; + uintptr_t eax; + }; +#endif } diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index fb3b0d799e..f232bdccc6 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -77,7 +77,6 @@ namespace Kernel size_t virtual_page_count() const { return (m_kernel_stack->size() / PAGE_SIZE) + (m_userspace_stack->size() / PAGE_SIZE); } size_t physical_page_count() const { return virtual_page_count(); } - uintptr_t& interrupt_sp() { return m_interrupt_sp; } InterruptStack& interrupt_stack() { return m_interrupt_stack; } InterruptRegisters& interrupt_registers() { return m_interrupt_registers; } @@ -89,6 +88,8 @@ namespace Kernel private: Thread(pid_t tid, Process*); + + static void on_exit_trampoline(Thread*); void on_exit(); private: @@ -104,7 +105,6 @@ namespace Kernel InterruptStack m_interrupt_stack { }; InterruptRegisters m_interrupt_registers { }; - uintptr_t m_interrupt_sp { }; uint64_t m_signal_pending_mask { 0 }; uint64_t m_signal_block_mask { 0 }; diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 4e33af3343..0031c9ba5d 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -14,17 +14,6 @@ namespace Kernel static Scheduler* s_instance = nullptr; static BAN::Atomic s_started { false }; - ALWAYS_INLINE static void load_temp_stack() - { -#if ARCH(x86_64) - asm volatile("movq %0, %%rsp" :: "rm"(Processor::current_stack_top())); -#elif ARCH(i686) - asm volatile("movl %0, %%esp" :: "rm"(Processor::current_stack_top())); -#else - #error -#endif - } - BAN::ErrorOr Scheduler::initialize() { ASSERT(s_instance == nullptr); @@ -40,6 +29,11 @@ namespace Kernel return *s_instance; } + extern "C" uintptr_t get_start_kernel_thread_sp() + { + return Scheduler::get().current_thread().kernel_stack_top() - 4 * sizeof(uintptr_t); + } + void Scheduler::start() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); @@ -86,7 +80,6 @@ namespace Kernel if (thread->state() != Thread::State::NotStarted) { thread->interrupt_stack() = Processor::get_interrupt_stack(); - thread->interrupt_stack().sp = thread->interrupt_sp(); thread->interrupt_registers() = Processor::get_interrupt_registers(); } @@ -155,15 +148,29 @@ namespace Kernel auto state = Processor::get_interrupt_state(); Processor::set_interrupt_state(InterruptState::Disabled); +#if ARCH(x86_64) asm volatile( - "movq %%rsp, %[save_sp];" + "movq %%rsp, %%rcx;" "movq %[load_sp], %%rsp;" "int %[ipi];" - : [save_sp]"=m"(Thread::current().interrupt_sp()) - : [load_sp]"r"(Processor::current_stack_top()), - [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI) - : "memory" + "movq %%rcx, %%rsp;" + :: [load_sp]"r"(Processor::current_stack_top()), + [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI) + : "memory", "rcx" ); +#elif ARCH(i686) + asm volatile( + "movl %%esp, %%ecx;" + "movl %[load_sp], %%esp;" + "int %[ipi];" + "movl %%ecx, %%esp;" + :: [load_sp]"r"(Processor::current_stack_top()), + [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI) + : "memory", "ecx" + ); +#else + #error +#endif Processor::set_interrupt_state(state); } diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index a7ad2e22f9..c75dfc2f3d 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -12,19 +12,15 @@ namespace Kernel { - extern "C" [[noreturn]] void start_userspace_thread(); extern "C" [[noreturn]] void start_kernel_thread(); extern "C" void signal_trampoline(); template - static void write_to_stack(uintptr_t& rsp, const T& value) + static void write_to_stack(uintptr_t& rsp, const T& value) requires(sizeof(T) <= sizeof(uintptr_t)) { rsp -= sizeof(uintptr_t); - if constexpr(sizeof(T) < sizeof(uintptr_t)) - *(uintptr_t*)rsp = (uintptr_t)value; - else - memcpy((void*)rsp, (void*)&value, sizeof(uintptr_t)); + *(uintptr_t*)rsp = (uintptr_t)value; } static pid_t s_next_tid = 1; @@ -50,13 +46,13 @@ namespace Kernel // Initialize stack for returning uintptr_t sp = thread->kernel_stack_top(); write_to_stack(sp, thread); - write_to_stack(sp, &Thread::on_exit); + write_to_stack(sp, &Thread::on_exit_trampoline); write_to_stack(sp, data); write_to_stack(sp, entry); thread->m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); thread->m_interrupt_stack.cs = 0x08; - thread->m_interrupt_stack.flags = 0x202; + thread->m_interrupt_stack.flags = 0x002; thread->m_interrupt_stack.sp = sp; thread->m_interrupt_stack.ss = 0x10; @@ -245,7 +241,7 @@ namespace Kernel PageTable::with_fast_page(process().page_table().physical_address_of(kernel_stack_top() - PAGE_SIZE), [&] { uintptr_t sp = PageTable::fast_page() + PAGE_SIZE; write_to_stack(sp, this); - write_to_stack(sp, &Thread::on_exit); + write_to_stack(sp, &Thread::on_exit_trampoline); write_to_stack(sp, m_process); write_to_stack(sp, entry); }); @@ -422,6 +418,11 @@ namespace Kernel return {}; } + void Thread::on_exit_trampoline(Thread* thread) + { + thread->on_exit(); + } + void Thread::on_exit() { ASSERT(this == &Thread::current()); From b35cad0c2e4afde4bfae50d21551837c6185b301 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 2 Apr 2024 12:48:35 +0300 Subject: [PATCH 38/46] Bootloader allow installation when BANAN_ARCH=i686 --- bootloader/installer/CMakeLists.txt | 17 +++++++++++------ bootloader/installer/ELF.cpp | 2 +- script/install-bootloader.sh | 11 ++++++++--- 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/bootloader/installer/CMakeLists.txt b/bootloader/installer/CMakeLists.txt index f2abc69aff..d5efc80fa9 100644 --- a/bootloader/installer/CMakeLists.txt +++ b/bootloader/installer/CMakeLists.txt @@ -1,6 +1,11 @@ cmake_minimum_required(VERSION 3.26) -project(x86_64-banan_os-bootloader-installer CXX) +if (NOT DEFINED ENV{BANAN_ARCH}) + message(FATAL_ERROR "environment variable BANAN_ARCH not defined") +endif () +set(BANAN_ARCH $ENV{BANAN_ARCH}) + +project(banan_os-bootloader-installer CXX) set(SOURCES crc32.cpp @@ -10,8 +15,8 @@ set(SOURCES main.cpp ) -add_executable(x86_64-banan_os-bootloader-installer ${SOURCES}) -target_compile_options(x86_64-banan_os-bootloader-installer PRIVATE -O2 -std=c++20) -target_compile_definitions(x86_64-banan_os-bootloader-installer PRIVATE __arch=x86_64) -target_include_directories(x86_64-banan_os-bootloader-installer PRIVATE ${CMAKE_SOURCE_DIR}/../../LibELF/include) -target_include_directories(x86_64-banan_os-bootloader-installer PRIVATE ${CMAKE_SOURCE_DIR}/../../kernel/include) +add_executable(banan_os-bootloader-installer ${SOURCES}) +target_compile_options(banan_os-bootloader-installer PRIVATE -O2 -std=c++20) +target_compile_definitions(banan_os-bootloader-installer PRIVATE __arch=${BANAN_ARCH}) +target_include_directories(banan_os-bootloader-installer PRIVATE ${CMAKE_SOURCE_DIR}/../../LibELF/include) +target_include_directories(banan_os-bootloader-installer PRIVATE ${CMAKE_SOURCE_DIR}/../../kernel/include) diff --git a/bootloader/installer/ELF.cpp b/bootloader/installer/ELF.cpp index bc2d0845ea..99c5b4d564 100644 --- a/bootloader/installer/ELF.cpp +++ b/bootloader/installer/ELF.cpp @@ -81,7 +81,7 @@ bool ELFFile::validate_elf_header() const #if ARCH(x86_64) if (elf_header.e_ident[EI_CLASS] != ELFCLASS64) -#elif ARCH(i386) +#elif ARCH(i686) if (elf_header.e_ident[EI_CLASS] != ELFCLASS32) #endif { diff --git a/script/install-bootloader.sh b/script/install-bootloader.sh index 204190c4d3..410c8c6226 100755 --- a/script/install-bootloader.sh +++ b/script/install-bootloader.sh @@ -2,6 +2,11 @@ set -e +if [[ -z $BANAN_ARCH ]]; then + echo "You must set the BANAN_ARCH environment variable" >&2 + exit 1 +fi + if [[ -z $BANAN_DISK_IMAGE_PATH ]]; then echo "You must set the BANAN_DISK_IMAGE_PATH environment variable" >&2 exit 1 @@ -26,7 +31,7 @@ ROOT_PARTITION_INDEX=2 ROOT_PARTITION_INFO=$(fdisk -x $BANAN_DISK_IMAGE_PATH | grep "^$BANAN_DISK_IMAGE_PATH" | head -$ROOT_PARTITION_INDEX | tail -1) ROOT_PARTITION_GUID=$(echo $ROOT_PARTITION_INFO | cut -d' ' -f6) -INSTALLER_BUILD_DIR=$BANAN_ROOT_DIR/bootloader/installer/build +INSTALLER_BUILD_DIR=$BANAN_ROOT_DIR/bootloader/installer/build/$BANAN_ARCH BOOTLOADER_ELF=$BANAN_BUILD_DIR/bootloader/bios/bootloader if ! [ -f $BOOTLOADER_ELF ]; then @@ -37,11 +42,11 @@ fi if ! [ -d $INSTALLER_BUILD_DIR ]; then mkdir -p $INSTALLER_BUILD_DIR cd $INSTALLER_BUILD_DIR - $CMAKE_COMMAND .. + $CMAKE_COMMAND ../.. fi cd $INSTALLER_BUILD_DIR make echo installing bootloader -$INSTALLER_BUILD_DIR/x86_64-banan_os-bootloader-installer $BOOTLOADER_ELF $BANAN_DISK_IMAGE_PATH $ROOT_PARTITION_GUID +$INSTALLER_BUILD_DIR/banan_os-bootloader-installer $BOOTLOADER_ELF $BANAN_DISK_IMAGE_PATH $ROOT_PARTITION_GUID From 6ac36816043c7abd659cff5a6b02bc0b8b521ecb Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 2 Apr 2024 15:04:40 +0300 Subject: [PATCH 39/46] Bootloader: Implement loading for 32 bit ELF files. --- bootloader/bios/elf.S | 235 +++++++++++++++++++++++++++++++----------- 1 file changed, 175 insertions(+), 60 deletions(-) diff --git a/bootloader/bios/elf.S b/bootloader/bios/elf.S index 3499e5ee30..e1d2d2c2c0 100644 --- a/bootloader/bios/elf.S +++ b/bootloader/bios/elf.S @@ -5,15 +5,26 @@ .set e_machine, 18 .set e_version, 20 .set e_entry, 24 -.set e_phoff, 32 -.set e_shoff, 40 -.set e_flags, 48 -.set e_ehsize, 52 -.set e_phentsize, 54 -.set e_phnum, 56 -.set e_shentsize, 58 -.set e_shnum, 60 -.set e_shstrndx, 62 + +.set e32_phoff, 28 +.set e32_shoff, 32 +.set e32_flags, 36 +.set e32_ehsize, 40 +.set e32_phentsize, 42 +.set e32_phnum, 44 +.set e32_shentsize, 46 +.set e32_shnum, 48 +.set e32_shstrndx, 50 + +.set e64_phoff, 32 +.set e64_shoff, 40 +.set e64_flags, 48 +.set e64_ehsize, 52 +.set e64_phentsize, 54 +.set e64_phnum, 56 +.set e64_shentsize, 58 +.set e64_shnum, 60 +.set e64_shstrndx, 62 # e_ident offsets .set EI_CLASS, 4 @@ -22,6 +33,7 @@ # e_ident constants .set ELFMAGIC, 0x464C457F +.set ELFCLASS32, 1 .set ELFCLASS64, 2 .set ELFDATA2LSB, 1 .set EV_CURRENT, 1 @@ -31,18 +43,30 @@ # program header field offsets .set p_type, 0 -.set p_flags, 4 -.set p_offset, 8 -.set p_vaddr, 16 -.set p_paddr, 24 -.set p_filesz, 32 -.set p_memsz, 40 -.set p_align, 48 + +.set p32_offset, 4 +.set p32_vaddr, 8 +.set p32_paddr, 12 +.set p32_filesz, 16 +.set p32_memsz, 20 +.set p32_flags, 24 +.set p32_align, 28 + +.set p64_flags, 4 +.set p64_offset, 8 +.set p64_vaddr, 16 +.set p64_paddr, 24 +.set p64_filesz, 32 +.set p64_memsz, 40 +.set p64_align, 48 # p_type constants .set PT_NULL, 0 .set PT_LOAD, 1 +# mask for entry point and segment loading +.set LOAD_MASK, 0x07FFFFFF + .code16 .section .stage2 @@ -52,8 +76,12 @@ elf_validate_file_header: cmpl $ELFMAGIC, (elf_file_header) jne .elf_validate_file_header_invalid_magic + cmpb $ELFCLASS32, (elf_file_header + EI_CLASS) + je .elf_validate_file_header_class_valid cmpb $ELFCLASS64, (elf_file_header + EI_CLASS) - jne .elf_validate_file_header_only_64bit_supported + je .elf_validate_file_header_class_valid + jmp .elf_validate_file_header_invalid_class + .elf_validate_file_header_class_valid: cmpb $ELFDATA2LSB, (elf_file_header + EI_DATA) jne .elf_validate_file_header_only_little_endian_supported @@ -72,8 +100,8 @@ elf_validate_file_header: .elf_validate_file_header_invalid_magic: movw $elf_validate_file_header_invalid_magic_msg, %si jmp print_and_halt - .elf_validate_file_header_only_64bit_supported: - movw $elf_validate_file_header_only_64bit_supported_msg, %si + .elf_validate_file_header_invalid_class: + movw $elf_validate_file_header_invalid_class_msg, %si jmp print_and_halt .elf_validate_file_header_only_little_endian_supported: movw $elf_validate_file_header_only_little_endian_supported_msg, %si @@ -86,6 +114,77 @@ elf_validate_file_header: jmp print_and_halt +# sets memory to zero +# edi: start address +# ecx: byte count +# on return +# edi: start address + byte count +# ecx: 0 +elf_memset_zero: + test %ecx, %ecx + jz .elf_memset_zero_done + .elf_memset_zero_loop: + movb $0, (%edi) + incl %edi + decl %ecx + jnz .elf_memset_zero_loop + .elf_memset_zero_done: + ret + +# reads memory specified by 32 bit elf_program_header to memory +elf_read_program_header32_to_memory: + pushal + pushl %ebp + movl %esp, %ebp + + # memset p_filesz -> p_memsz to 0 + movl (elf_program_header + p32_filesz), %ebx + movl (elf_program_header + p32_vaddr), %edi + andl $LOAD_MASK, %edi + addl %ebx, %edi + movl (elf_program_header + p32_memsz), %ecx + subl %ebx, %ecx + call elf_memset_zero + + # read file specified in program header to memory + movl (elf_program_header + p32_offset), %eax + movl (elf_program_header + p32_vaddr), %edi + andl $LOAD_MASK, %edi + movl (elf_program_header + p32_filesz), %ecx + call *%esi + + leavel + popal + ret + + +# reads memory specified by 64 bit elf_program_header to memory +elf_read_program_header64_to_memory: + pushal + pushl %ebp + movl %esp, %ebp + + # memset p_filesz -> p_memsz to 0 + movl (elf_program_header + p64_filesz), %ebx + movl (elf_program_header + p64_vaddr), %edi + andl $LOAD_MASK, %edi + addl %ebx, %edi + movl (elf_program_header + p64_memsz), %ecx + subl %ebx, %ecx + call elf_memset_zero + + # read file specified in program header to memory + movl (elf_program_header + p64_offset), %eax + movl (elf_program_header + p64_vaddr), %edi + andl $LOAD_MASK, %edi + movl (elf_program_header + p64_filesz), %ecx + call *%esi + + leavel + popal + ret + + # read callback format # eax: first byte # ecx: byte count @@ -104,42 +203,72 @@ elf_read_kernel_to_memory: movl %esp, %ebp subl $2, %esp - # read file header + # read start of file header movl $0, %eax - movl $64, %ecx + movl $24, %ecx movl $elf_file_header, %edi call *%esi call elf_validate_file_header - cmpl $0, (elf_file_header + e_phoff + 4) + # determine file header size + movl $52, %ecx + movl $64, %edx + cmpb $ELFCLASS64, (elf_file_header + EI_CLASS) + cmovel %edx, %ecx + + # read full file header + movl $0, %eax + movl $elf_file_header, %edi + call *%esi + + # verify that e_phoff fits in 32 bits + cmpb $ELFCLASS64, (elf_file_header + EI_CLASS) + jne .elf_read_kernel_to_memory_valid_offset + cmpl $0, (elf_file_header + e64_phoff + 4) jnz .elf_read_kernel_to_memory_unsupported_offset + .elf_read_kernel_to_memory_valid_offset: + + # read architecture phentsize and phnum to fixed locations + movw (elf_file_header + e32_phentsize), %ax + movw (elf_file_header + e32_phnum), %bx + movl (elf_file_header + e32_phoff), %ecx + cmpb $ELFCLASS64, (elf_file_header + EI_CLASS) + cmovew (elf_file_header + e64_phentsize), %ax + cmovew (elf_file_header + e64_phnum), %bx + cmovel (elf_file_header + e64_phoff), %ecx + movw %ax, (elf_file_header_phentsize) + movw %bx, (elf_file_header_phnum) + movl %ecx, (elf_file_header_phoff) # current program header movw $0, -2(%ebp) .elf_read_kernel_to_memory_loop_program_headers: movw -2(%ebp), %cx - cmpw (elf_file_header + e_phnum), %cx + cmpw (elf_file_header_phnum), %cx jae .elf_read_kernel_to_memory_done # eax := program_header_index * e_phentsize + e_phoff xorl %eax, %eax movw %cx, %ax xorl %ebx, %ebx - movw (elf_file_header + e_phentsize), %bx + movw (elf_file_header_phentsize), %bx mull %ebx - addl (elf_file_header + e_phoff), %eax + addl (elf_file_header_phoff), %eax jc .elf_read_kernel_to_memory_unsupported_offset - # setup program header size and address - movl $56, %ecx - movl $elf_program_header, %edi + # determine program header size + movl $32, %ecx + movl $56, %edx + cmpb $ELFCLASS64, (elf_file_header + EI_CLASS) + cmovel %edx, %ecx - # read the program header + # read program header + movl $elf_program_header, %edi call *%esi - # test if program header is empty + # test if program header is NULL header cmpl $PT_NULL, (elf_program_header + p_type) je .elf_read_kernel_to_memory_null_program_header @@ -147,33 +276,12 @@ elf_read_kernel_to_memory: cmpl $PT_LOAD, (elf_program_header + p_type) jne .elf_read_kernel_to_memory_not_loadable_header - # memset p_filesz -> p_memsz to 0 - movl (elf_program_header + p_filesz), %ebx - - movl (elf_program_header + p_vaddr), %edi - andl $0x7FFFFFFF, %edi - addl %ebx, %edi - - movl (elf_program_header + p_memsz), %ecx - subl %ebx, %ecx - jz .elf_read_kernel_to_memory_memset_done - - .elf_read_kernel_to_memory_memset: - movb $0, (%edi) - incl %edi - decl %ecx - jnz .elf_read_kernel_to_memory_memset - .elf_read_kernel_to_memory_memset_done: - - # read file specified in program header to memory - movl (elf_program_header + p_offset), %eax - movl (elf_program_header + p_vaddr), %edi - andl $0x7FFFFFFF, %edi - movl (elf_program_header + p_filesz), %ecx - - #call print_hex32; call print_newline - - call *%esi + # read program header to memory + movl $elf_read_program_header32_to_memory, %eax + movl $elf_read_program_header64_to_memory, %ebx + cmpb $ELFCLASS64, (elf_file_header + EI_CLASS) + cmovel %ebx, %eax + call *%eax .elf_read_kernel_to_memory_null_program_header: incw -2(%ebp) @@ -185,7 +293,7 @@ elf_read_kernel_to_memory: # set kernel entry address movl (elf_file_header + e_entry), %eax - andl $0x7FFFFF, %eax + andl $LOAD_MASK, %eax ret @@ -200,8 +308,8 @@ elf_read_kernel_to_memory: elf_validate_file_header_invalid_magic_msg: .asciz "ELF: file has invalid ELF magic" -elf_validate_file_header_only_64bit_supported_msg: - .asciz "ELF: file is not targettint 64 bit" +elf_validate_file_header_invalid_class_msg: + .asciz "ELF: file has invalid ELF class" elf_validate_file_header_only_little_endian_supported_msg: .asciz "ELF: file is not in little endian format" elf_validate_file_header_not_current_version_msg: @@ -219,5 +327,12 @@ elf_read_kernel_to_memory_not_loadable_header_msg: elf_file_header: .skip 64 +elf_file_header_phentsize: + .skip 2 +elf_file_header_phnum: + .skip 2 +elf_file_header_phoff: + .skip 4 # NOTE: only 32 bit offsets are supported + elf_program_header: .skip 56 From d9b8391968575f6b8abc1dd6d9f3ab9676531092 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 00:40:16 +0300 Subject: [PATCH 40/46] Kernel: Fix i686 page table global mappings --- kernel/arch/i686/PageTable.cpp | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/kernel/arch/i686/PageTable.cpp b/kernel/arch/i686/PageTable.cpp index a5e60c5967..869ade0fd3 100644 --- a/kernel/arch/i686/PageTable.cpp +++ b/kernel/arch/i686/PageTable.cpp @@ -117,16 +117,7 @@ namespace Kernel ASSERT(s_global_pdpte == 0); s_global_pdpte = V2P(allocate_zeroed_page_aligned_page()); - ASSERT(m_highest_paging_struct == 0); - m_highest_paging_struct = V2P(kmalloc(32, 32, true)); - ASSERT(m_highest_paging_struct); - - uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); - pdpt[0] = 0; - pdpt[1] = 0; - pdpt[2] = 0; - pdpt[3] = s_global_pdpte; - static_assert(KERNEL_OFFSET == 0xC0000000); + map_kernel_memory(); prepare_fast_page(); @@ -171,8 +162,7 @@ namespace Kernel constexpr uint64_t pte = (fast_page() >> 12) & 0x1FF; uint64_t* pdpt = reinterpret_cast(P2V(m_highest_paging_struct)); - ASSERT(!(pdpt[pdpte] & Flags::Present)); - pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present; + ASSERT(pdpt[pdpte] & Flags::Present); uint64_t* pd = reinterpret_cast(P2V(pdpt[pdpte]) & PAGE_ADDR_MASK); ASSERT(!(pd[pde] & Flags::Present)); @@ -247,7 +237,7 @@ namespace Kernel pdpt[0] = 0; pdpt[1] = 0; pdpt[2] = 0; - pdpt[3] = s_global_pdpte; + pdpt[3] = s_global_pdpte | Flags::Present; static_assert(KERNEL_OFFSET == 0xC0000000); } From 4d70322eab7fedccc315d5e10494d8fdb369d343 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 00:41:13 +0300 Subject: [PATCH 41/46] Kernel: Save segment registers on all interrupts on i686 --- kernel/arch/i686/interrupts.S | 55 ++++++++++++++++++++++++++--------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 335dd45315..60dd8b8060 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -1,5 +1,32 @@ +.macro push_userspace + pushw %gs + pushw %fs + pushw %es + pushw %ds + pushal +.endm + +.macro load_kernel_segments + movw $0x10, %ax + movw %ax, %ds + movw %ax, %es + movw %ax, %fs + + movw $0x28, %ax + movw %ax, %gs +.endm + +.macro pop_userspace + popal + popw %ds + popw %es + popw %fs + popw %gs +.endm + isr_stub: - pusha + push_userspace + load_kernel_segments movl %cr0, %eax; pushl %eax movl %cr2, %eax; pushl %eax @@ -7,9 +34,9 @@ isr_stub: movl %cr4, %eax; pushl %eax movl %esp, %eax // register ptr - leal 56(%esp), %ebx // interrupt stack ptr - movl 52(%esp), %ecx // error code - movl 48(%esp), %edx // isr number + leal 64(%esp), %ebx // interrupt stack ptr + movl 60(%esp), %ecx // error code + movl 56(%esp), %edx // isr number subl $12, %esp pushl %eax @@ -19,30 +46,32 @@ isr_stub: call cpp_isr_handler addl $44, %esp - popa + pop_userspace addl $8, %esp iret irq_stub: - pusha + push_userspace + load_kernel_segments - movl 32(%esp), %eax # interrupt number + movl 40(%esp), %eax # interrupt number - subl $8, %esp + subl $12, %esp pushl %eax call cpp_irq_handler - addl $12, %esp + addl $16, %esp - popa + pop_userspace addl $8, %esp iret .global asm_reschedule_handler asm_reschedule_handler: - pusha + push_userspace + load_kernel_segments movl %esp, %eax # interrupt registers ptr - leal 32(%esp), %ebx # interrupt stack ptr + leal 40(%esp), %ebx # interrupt stack ptr subl $12, %esp pushl %eax @@ -50,7 +79,7 @@ asm_reschedule_handler: call cpp_reschedule_handler addl $20, %esp - popa + pop_userspace iret // arguments in EAX, EBX, ECX, EDX, ESI, EDI From fe386fa819d4f62bd9d895bd4945617842bf7456 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 00:42:39 +0300 Subject: [PATCH 42/46] Kernel: Implement thread start trampoline for userspace This is needed on i686 to set segment registers. --- kernel/arch/x86_64/Thread.S | 26 ++++++++++++++++++++++++++ kernel/kernel/Scheduler.cpp | 5 ----- kernel/kernel/Thread.cpp | 25 ++++++++++++++++++------- libc/arch/x86_64/crt0.S | 5 +++++ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index 46eea1b08b..ad30770977 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -24,3 +24,29 @@ start_kernel_thread: movq 24(%rsp), %rdi movq 16(%rsp), %rsi call *%rsi + +.global start_userspace_thread +start_userspace_thread: + call get_start_kernel_thread_sp + movq %rax, %rsp + + # STACK LAYOUT + # entry + # argc + # argv + # envp + # userspace stack + + call get_userspace_thread_stack_top + + popq %rdx + popq %rsi + popq %rdi + popq %rcx + + pushq $(0x20 | 3) + pushq %rax + pushq $0x202 + pushq $(0x18 | 3) + pushq %rcx + iretq diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 0031c9ba5d..0f0ff63bd9 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -29,11 +29,6 @@ namespace Kernel return *s_instance; } - extern "C" uintptr_t get_start_kernel_thread_sp() - { - return Scheduler::get().current_thread().kernel_stack_top() - 4 * sizeof(uintptr_t); - } - void Scheduler::start() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index c75dfc2f3d..06041d06cd 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -13,6 +13,7 @@ namespace Kernel { extern "C" [[noreturn]] void start_kernel_thread(); + extern "C" [[noreturn]] void start_userspace_thread(); extern "C" void signal_trampoline(); @@ -23,6 +24,16 @@ namespace Kernel *(uintptr_t*)rsp = (uintptr_t)value; } + extern "C" uintptr_t get_start_kernel_thread_sp() + { + return Thread::current().kernel_stack_top() - 4 * sizeof(uintptr_t); + } + + extern "C" uintptr_t get_userspace_thread_stack_top() + { + return Thread::current().userspace_stack_top() - 4 * sizeof(uintptr_t); + } + static pid_t s_next_tid = 1; BAN::ErrorOr Thread::create_kernel(entry_t entry, void* data, Process* process) @@ -197,19 +208,19 @@ namespace Kernel ASSERT(userspace_info.entry); // Initialize stack for returning - PageTable::with_fast_page(process().page_table().physical_address_of(userspace_stack_top() - PAGE_SIZE), [&] { + PageTable::with_fast_page(process().page_table().physical_address_of(kernel_stack_top() - PAGE_SIZE), [&] { uintptr_t sp = PageTable::fast_page() + PAGE_SIZE; - write_to_stack(sp, nullptr); + write_to_stack(sp, userspace_info.entry); write_to_stack(sp, userspace_info.argc); write_to_stack(sp, userspace_info.argv); write_to_stack(sp, userspace_info.envp); }); - m_interrupt_stack.ip = userspace_info.entry; - m_interrupt_stack.cs = 0x18 | 3; - m_interrupt_stack.flags = 0x202; - m_interrupt_stack.sp = userspace_stack_top() - 4 * sizeof(uintptr_t); - m_interrupt_stack.ss = 0x20 | 3; + m_interrupt_stack.ip = reinterpret_cast(start_userspace_thread);; + m_interrupt_stack.cs = 0x08; + m_interrupt_stack.flags = 0x002; + m_interrupt_stack.sp = kernel_stack_top() - 4 * sizeof(uintptr_t); + m_interrupt_stack.ss = 0x10; memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); } diff --git a/libc/arch/x86_64/crt0.S b/libc/arch/x86_64/crt0.S index 474a8bf4d4..e4176a9fa5 100644 --- a/libc/arch/x86_64/crt0.S +++ b/libc/arch/x86_64/crt0.S @@ -2,6 +2,11 @@ .global _start _start: + pushq $0 + pushq %rdi + pushq %rsi + pushq %rdx + # STACK LAYOUT # null # argc From c95a271821ac8055cd1726215121957491bb8591 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 00:43:38 +0300 Subject: [PATCH 43/46] Kernel: Set ss in i686 tss --- kernel/include/kernel/GDT.h | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel/include/kernel/GDT.h b/kernel/include/kernel/GDT.h index 56efc2cced..bf8bfa5e91 100644 --- a/kernel/include/kernel/GDT.h +++ b/kernel/include/kernel/GDT.h @@ -124,6 +124,7 @@ namespace Kernel m_tss.rsp0 = sp; #elif ARCH(i686) m_tss.esp0 = sp; + m_tss.ss0 = 0x10; #endif } From 9e073e9fa0dd73432f4345d45962d1b0323f3740 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 00:45:22 +0300 Subject: [PATCH 44/46] Kernel: Add offset for interrupt stack in Scheduler::yield() This allows accessing (garbage) sp and ss in interrupt stack. --- kernel/kernel/Scheduler.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 0f0ff63bd9..5f8aa57343 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -115,11 +115,7 @@ namespace Kernel if (thread->state() == Thread::State::NotStarted) thread->m_state = Thread::State::Executing; - ASSERT(thread->interrupt_stack().ip); - ASSERT(thread->interrupt_stack().sp); - Processor::gdt().set_tss_stack(thread->kernel_stack_top()); - Processor::get_interrupt_stack() = thread->interrupt_stack(); Processor::get_interrupt_registers() = thread->interrupt_registers(); } @@ -149,7 +145,9 @@ namespace Kernel "movq %[load_sp], %%rsp;" "int %[ipi];" "movq %%rcx, %%rsp;" - :: [load_sp]"r"(Processor::current_stack_top()), + // NOTE: This is offset by 2 pointers since interrupt without PL change + // does not push SP and SS. This allows accessing "whole" interrupt stack. + :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI) : "memory", "rcx" ); @@ -159,7 +157,9 @@ namespace Kernel "movl %[load_sp], %%esp;" "int %[ipi];" "movl %%ecx, %%esp;" - :: [load_sp]"r"(Processor::current_stack_top()), + // NOTE: This is offset by 2 pointers since interrupt without PL change + // does not push SP and SS. This allows accessing "whole" interrupt stack. + :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI) : "memory", "ecx" ); From 0dd74e3c9dc9e808c3fda88967417d0d5fdd017e Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 02:23:23 +0300 Subject: [PATCH 45/46] Kernel: Implement syscalls for i686 and cleanup x86_64 This actually allows i686 to boot properly! --- kernel/arch/i686/Syscall.S | 28 ++++++++++++++++++-------- kernel/arch/i686/Thread.S | 36 ++++++++++++++++++++++++++++++++- kernel/arch/i686/interrupts.S | 38 ++++++++++++++++++++++------------- kernel/arch/x86_64/Syscall.S | 13 ++++++++++-- kernel/arch/x86_64/Thread.S | 4 ++-- kernel/kernel/Syscall.cpp | 4 ++-- kernel/kernel/Thread.cpp | 10 +++++++-- libc/arch/i686/crt0.S | 35 ++++++++++++++++---------------- 8 files changed, 120 insertions(+), 48 deletions(-) diff --git a/kernel/arch/i686/Syscall.S b/kernel/arch/i686/Syscall.S index 39f892016a..7a31dcd173 100644 --- a/kernel/arch/i686/Syscall.S +++ b/kernel/arch/i686/Syscall.S @@ -1,19 +1,31 @@ .global sys_fork_trampoline sys_fork_trampoline: - ud2 - subl $4, %esp - pushl %ebx pushl %ebp + pushl %ebx + pushl %esi + pushl %edi + call read_ip testl %eax, %eax - je .done + jz .reload_stack + + movl %esp, %ebx + subl $8, %esp pushl %eax - pushl %esp + pushl %ebx call sys_fork addl $16, %esp -.done: - popl %ebp + + .done: + popl %edi + popl %esi popl %ebx - addl $4, %esp + popl %ebp ret + + .reload_stack: + call get_thread_start_sp + movl %eax, %esp + xorl %eax, %eax + jmp .done diff --git a/kernel/arch/i686/Thread.S b/kernel/arch/i686/Thread.S index c6f78c89dc..ef8dd7d705 100644 --- a/kernel/arch/i686/Thread.S +++ b/kernel/arch/i686/Thread.S @@ -7,7 +7,7 @@ read_ip: # void start_kernel_thread() .global start_kernel_thread start_kernel_thread: - call get_start_kernel_thread_sp + call get_thread_start_sp movl %eax, %esp # STACK LAYOUT @@ -32,3 +32,37 @@ start_kernel_thread: pushl %edi call *%esi addl $16, %esp + + +.global start_userspace_thread +start_userspace_thread: + call get_thread_start_sp + movl %eax, %esp + + # STACK LAYOUT + # entry + # argc + # argv + # envp + # userspace stack + + call get_userspace_thread_stack_top + + movw $(0x20 | 3), %bx + movw %bx, %ds + movw %bx, %es + movw %bx, %fs + movw %bx, %gs + xorw %bx, %bx + + popl %edx + popl %esi + popl %edi + popl %ecx + + pushl $(0x20 | 3) + pushl %eax + pushl $0x202 + pushl $(0x18 | 3) + pushl %ecx + iret diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 60dd8b8060..9328ad1f84 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -24,6 +24,22 @@ popw %gs .endm +.macro pop_userspace_skip_eax + popl %edi + popl %esi + popl %ebp + addl $4, %esp + popl %ebx + popl %edx + popl %ecx + addl $4, %esp + + popw %ds + popw %es + popw %fs + popw %gs +.endm + isr_stub: push_userspace load_kernel_segments @@ -85,11 +101,12 @@ asm_reschedule_handler: // arguments in EAX, EBX, ECX, EDX, ESI, EDI .global syscall_asm syscall_asm: - ud2 - pusha + push_userspace + + subl $8, %esp pushl %esp - addl $36, (%esp) + addl $48, (%esp) pushl %edi pushl %esi @@ -98,19 +115,12 @@ syscall_asm: pushl %ebx pushl %eax + load_kernel_segments + call cpp_syscall_handler + addl $36, %esp - addl $60, %esp - - popl %edi - popl %esi - popl %ebp - addl $4, %esp - popl %ebx - popl %edx - popl %ecx - addl $4, %esp - + pop_userspace_skip_eax iret .macro isr n diff --git a/kernel/arch/x86_64/Syscall.S b/kernel/arch/x86_64/Syscall.S index 45c1656086..91cc85fdb8 100644 --- a/kernel/arch/x86_64/Syscall.S +++ b/kernel/arch/x86_64/Syscall.S @@ -6,13 +6,16 @@ sys_fork_trampoline: pushq %r13 pushq %r14 pushq %r15 + call read_ip testq %rax, %rax - je .done + je .reload_stack + movq %rax, %rsi movq %rsp, %rdi call sys_fork -.done: + + .done: popq %r15 popq %r14 popq %r13 @@ -20,3 +23,9 @@ sys_fork_trampoline: popq %rbp popq %rbx ret + + .reload_stack: + call get_thread_start_sp + movq %rax, %rsp + xorq %rax, %rax + jmp .done diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index ad30770977..a1757157f5 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -7,7 +7,7 @@ read_ip: # void start_kernel_thread() .global start_kernel_thread start_kernel_thread: - call get_start_kernel_thread_sp + call get_thread_start_sp movq %rax, %rsp # STACK LAYOUT @@ -27,7 +27,7 @@ start_kernel_thread: .global start_userspace_thread start_userspace_thread: - call get_start_kernel_thread_sp + call get_thread_start_sp movq %rax, %rsp # STACK LAYOUT diff --git a/kernel/kernel/Syscall.cpp b/kernel/kernel/Syscall.cpp index 22ca55e161..ab979b68b6 100644 --- a/kernel/kernel/Syscall.cpp +++ b/kernel/kernel/Syscall.cpp @@ -28,9 +28,9 @@ namespace Kernel #undef O }; - extern "C" long cpp_syscall_handler(int syscall, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5, InterruptStack& interrupt_stack) + extern "C" long cpp_syscall_handler(int syscall, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5, InterruptStack* interrupt_stack) { - ASSERT((interrupt_stack.cs & 0b11) == 0b11); + ASSERT(GDT::is_user_segment(interrupt_stack->cs)); asm volatile("sti"); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 06041d06cd..12fd45891a 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -24,9 +24,9 @@ namespace Kernel *(uintptr_t*)rsp = (uintptr_t)value; } - extern "C" uintptr_t get_start_kernel_thread_sp() + extern "C" uintptr_t get_thread_start_sp() { - return Thread::current().kernel_stack_top() - 4 * sizeof(uintptr_t); + return Thread::current().interrupt_stack().sp; } extern "C" uintptr_t get_userspace_thread_stack_top() @@ -192,6 +192,12 @@ namespace Kernel thread->m_interrupt_stack.sp = sp; thread->m_interrupt_stack.ss = 0x10; +#if ARCH(x86_64) + thread->m_interrupt_registers.rax = 0; +#elif ARCH(i686) + thread->m_interrupt_registers.eax = 0; +#endif + thread_deleter.disable(); return thread; diff --git a/libc/arch/i686/crt0.S b/libc/arch/i686/crt0.S index 6667894507..d44ffdce6c 100644 --- a/libc/arch/i686/crt0.S +++ b/libc/arch/i686/crt0.S @@ -2,31 +2,32 @@ .global _start _start: - # zero out stack frame pushl $0 - pushl $0 - movl %esp, %ebp - - # FIXME: handle stack alignment - ud2 - - # push argc, argv, environ for call to main - pushl %edx - pushl %esi pushl %edi - - # initialize libc + pushl %esi pushl %edx - call _init_libc - addl $4, %esp - # call global constructos + # STACK LAYOUT + # null + # argc + # argv + # envp + + xorl %ebp, %ebp + + # init libc (envp already as argument) + call _init_libc + + # call global constructors call _init - # call main, arguments are already on stack + # call main + movl 0(%esp), %eax + xchgl %eax, 8(%esp) + movl %eax, (%esp) call main - # cleanly exit the process + subl $12, %esp pushl %eax call exit From d2df55b1ac799e4830f3450b161f2ed70384af73 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Wed, 3 Apr 2024 02:30:38 +0300 Subject: [PATCH 46/46] Kernel: Allow booting with multiple processors on i686 Also remove unnecessary ds clearing in x86_64. --- kernel/arch/i686/boot.S | 4 ---- kernel/arch/x86_64/boot.S | 3 --- 2 files changed, 7 deletions(-) diff --git a/kernel/arch/i686/boot.S b/kernel/arch/i686/boot.S index 118d90977e..7e35c728c1 100644 --- a/kernel/arch/i686/boot.S +++ b/kernel/arch/i686/boot.S @@ -231,7 +231,6 @@ system_halt: .code16 .global ap_trampoline ap_trampoline: - ud2 jmp 1f .align 8 ap_stack_ptr: @@ -239,10 +238,7 @@ ap_stack_ptr: 1: cli; cld ljmpl $0x00, $ap_cs_clear - ap_cs_clear: - xorw %ax, %ax - movw %ax, %ds # load ap gdt and enter protected mode lgdt ap_gdtr diff --git a/kernel/arch/x86_64/boot.S b/kernel/arch/x86_64/boot.S index 4d4d1d00f9..0309dc2f49 100644 --- a/kernel/arch/x86_64/boot.S +++ b/kernel/arch/x86_64/boot.S @@ -249,10 +249,7 @@ ap_stack_ptr: 1: cli; cld ljmpl $0x00, $ap_cs_clear - ap_cs_clear: - xorw %ax, %ax - movw %ax, %ds # load ap gdt and enter protected mode lgdt ap_gdtr