diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index ee038c24..46e306fa 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -137,6 +137,7 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") arch/x86_64/Signal.S arch/x86_64/Syscall.S arch/x86_64/Thread.S + arch/x86_64/Yield.S ) elseif("${BANAN_ARCH}" STREQUAL "i686") set(KERNEL_SOURCES @@ -147,6 +148,7 @@ elseif("${BANAN_ARCH}" STREQUAL "i686") arch/i686/Signal.S arch/i686/Syscall.S arch/i686/Thread.S + arch/i686/Yield.S ) else() message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}") diff --git a/kernel/arch/i686/Yield.S b/kernel/arch/i686/Yield.S new file mode 100644 index 00000000..68b352be --- /dev/null +++ b/kernel/arch/i686/Yield.S @@ -0,0 +1,25 @@ +.global asm_yield_trampoline +asm_yield_trampoline: + movl %esp, %ecx + movl 4(%esp), %esp + + pushl (%ecx) + pushl %ecx + pushl %eax + pushl %ebx + pushl %esi + pushl %edi + pushl %ebp + + pushl %esp + call scheduler_on_yield + addl $4, %esp + + popl %ebp + popl %edi + popl %esi + popl %ebx + popl %eax + movl 4(%esp), %ecx + movl 0(%esp), %esp + jmp *%ecx diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 7bae605f..c355dbe7 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -83,28 +83,6 @@ irq_stub: addl $8, %esp iret -.global asm_yield_handler -asm_yield_handler: - # This can only be called from kernel, so no segment saving is needed - pushal - cld - - leal 32(%esp), %edi # interrupt stack ptr - movl %esp, %esi # interrupt registers ptr - - movl %esp, %ebp - andl $-16, %esp - - subl $8, %esp - pushl %esi - pushl %edi - call cpp_yield_handler - - movl %ebp, %esp - - popal - iret - .global asm_ipi_handler asm_ipi_handler: pushal diff --git a/kernel/arch/x86_64/Yield.S b/kernel/arch/x86_64/Yield.S new file mode 100644 index 00000000..bec190c4 --- /dev/null +++ b/kernel/arch/x86_64/Yield.S @@ -0,0 +1,29 @@ +.global asm_yield_trampoline +asm_yield_trampoline: + movq %rsp, %rcx + movq %rdi, %rsp + + subq $8, %rsp + pushq (%rcx) + pushq %rcx + pushq %rax + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + movq %rsp, %rdi + call scheduler_on_yield + + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + popq %rax + movq 8(%rsp), %rcx + movq 0(%rsp), %rsp + jmp *%rcx diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index 0995b353..401951fe 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -71,16 +71,6 @@ irq_stub: addq $16, %rsp iretq -.global asm_yield_handler -asm_yield_handler: - pushaq 8 - cld - leaq 120(%rsp), %rdi # interrupt stack ptr - movq %rsp, %rsi # interrupt register ptr - call cpp_yield_handler - popaq 8 - iretq - .global asm_ipi_handler asm_ipi_handler: pushaq 8 diff --git a/kernel/include/kernel/IDT.h b/kernel/include/kernel/IDT.h index f051dfcb..3da72577 100644 --- a/kernel/include/kernel/IDT.h +++ b/kernel/include/kernel/IDT.h @@ -22,9 +22,8 @@ namespace Kernel #if ARCH(i686) constexpr uint8_t IRQ_SYSCALL = 0xF0; #endif - constexpr uint8_t IRQ_YIELD = 0xF1; - constexpr uint8_t IRQ_IPI = 0xF2; - constexpr uint8_t IRQ_TIMER = 0xF3; + constexpr uint8_t IRQ_IPI = 0xF1; + constexpr uint8_t IRQ_TIMER = 0xF2; #if ARCH(x86_64) struct GateDescriptor diff --git a/kernel/include/kernel/InterruptStack.h b/kernel/include/kernel/InterruptStack.h index 4828bbf3..03006252 100644 --- a/kernel/include/kernel/InterruptStack.h +++ b/kernel/include/kernel/InterruptStack.h @@ -27,7 +27,6 @@ namespace Kernel uintptr_t r10; uintptr_t r9; uintptr_t r8; - uintptr_t rdi; uintptr_t rsi; uintptr_t rbp; @@ -36,6 +35,18 @@ namespace Kernel uintptr_t rcx; uintptr_t rax; }; + struct YieldRegisters + { + uintptr_t r15; + uintptr_t r14; + uintptr_t r13; + uintptr_t r12; + uintptr_t rbp; + uintptr_t rbx; + uintptr_t ret; + uintptr_t sp; + uintptr_t ip; + }; #elif ARCH(i686) struct InterruptRegisters { @@ -48,6 +59,16 @@ namespace Kernel uintptr_t ecx; uintptr_t eax; }; + struct YieldRegisters + { + uintptr_t ebp; + uintptr_t edi; + uintptr_t esi; + uintptr_t ebx; + uintptr_t ret; + uintptr_t sp; + uintptr_t ip; + }; #endif } diff --git a/kernel/include/kernel/Scheduler.h b/kernel/include/kernel/Scheduler.h index 5b3686ee..5b91a648 100644 --- a/kernel/include/kernel/Scheduler.h +++ b/kernel/include/kernel/Scheduler.h @@ -57,7 +57,7 @@ namespace Kernel static BAN::ErrorOr create(); BAN::ErrorOr initialize(); - void reschedule(InterruptStack*, InterruptRegisters*); + void reschedule(YieldRegisters*); void reschedule_if_idle(); void timer_interrupt(); diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index 454c6345..2f163dbe 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -132,8 +132,7 @@ namespace Kernel size_t virtual_page_count() const { return (m_kernel_stack ? (m_kernel_stack->size() / PAGE_SIZE) : 0) + (m_userspace_stack ? (m_userspace_stack->size() / PAGE_SIZE) : 0); } size_t physical_page_count() const { return virtual_page_count(); } - InterruptStack& interrupt_stack() { return m_interrupt_stack; } - InterruptRegisters& interrupt_registers() { return m_interrupt_registers; } + YieldRegisters& yield_registers() { return m_yield_registers; } void save_sse(); void load_sse(); @@ -173,8 +172,7 @@ namespace Kernel SchedulerQueue::Node* m_scheduler_node { nullptr }; - InterruptStack m_interrupt_stack { }; - InterruptRegisters m_interrupt_registers { }; + YieldRegisters m_yield_registers { }; siginfo_t m_signal_infos[_SIGMAX + 1] { }; uint64_t m_signal_pending_mask { 0 }; diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index ebdc5090..03b532e1 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -325,14 +325,6 @@ namespace Kernel Thread::current().load_sse(); } - extern "C" void cpp_yield_handler(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) - { - // yield is raised through kernel software interrupt - ASSERT(!InterruptController::get().is_in_service(IRQ_YIELD - IRQ_VECTOR_BASE)); - ASSERT(!GDT::is_user_segment(interrupt_stack->cs)); - Processor::scheduler().reschedule(interrupt_stack, interrupt_registers); - } - extern "C" void cpp_ipi_handler() { ASSERT(InterruptController::get().is_in_service(IRQ_IPI - IRQ_VECTOR_BASE)); @@ -477,7 +469,6 @@ namespace Kernel static_assert(DoubleFault == 8); #endif - idt->register_interrupt_handler(IRQ_YIELD, asm_yield_handler); idt->register_interrupt_handler(IRQ_IPI, asm_ipi_handler); idt->register_interrupt_handler(IRQ_TIMER, asm_timer_handler); #if ARCH(i686) diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 5513e9bb..1ff9d8fd 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -36,6 +36,7 @@ namespace Kernel static BAN::Array s_processor_ids { PROCESSOR_NONE }; extern "C" void asm_syscall_handler(); + extern "C" void asm_yield_trampoline(uintptr_t); ProcessorID Processor::read_processor_id() { @@ -556,33 +557,7 @@ namespace Kernel if (!scheduler().is_idle()) Thread::current().set_cpu_time_stop(); -#if ARCH(x86_64) - asm volatile( - "movq %%rsp, %%rcx;" - "movq %[load_sp], %%rsp;" - "int %[yield];" - "movq %%rcx, %%rsp;" - // NOTE: This is offset by 2 pointers since interrupt without PL change - // does not push SP and SS. This allows accessing "whole" interrupt stack. - :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), - [yield]"i"(static_cast(IRQ_YIELD)) // WTF GCC 15 - : "memory", "rcx" - ); -#elif ARCH(i686) - asm volatile( - "movl %%esp, %%ecx;" - "movl %[load_sp], %%esp;" - "int %[yield];" - "movl %%ecx, %%esp;" - // NOTE: This is offset by 2 pointers since interrupt without PL change - // does not push SP and SS. This allows accessing "whole" interrupt stack. - :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), - [yield]"i"(static_cast(IRQ_YIELD)) // WTF GCC 15 - : "memory", "ecx" - ); -#else - #error -#endif + asm_yield_trampoline(Processor::current_stack_top()); processor_info.m_start_ns = SystemTimer::get().ns_since_boot(); diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 83f826a1..6cb7953e 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -207,7 +207,7 @@ namespace Kernel m_most_loaded_threads.back().queue = nullptr; } - void Scheduler::reschedule(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) + void Scheduler::reschedule(YieldRegisters* yield_registers) { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); @@ -232,8 +232,7 @@ namespace Kernel case Thread::State::Executing: { const uint64_t current_ns = SystemTimer::get().ns_since_boot(); - m_current->thread->interrupt_stack() = *interrupt_stack; - m_current->thread->interrupt_registers() = *interrupt_registers; + m_current->thread->yield_registers() = *yield_registers; m_current->time_used_ns += current_ns - m_current->last_start_ns; add_current_to_most_loaded(m_current->blocked ? &m_block_queue : &m_run_queue); if (!m_current->blocked) @@ -267,8 +266,7 @@ namespace Kernel { if (&PageTable::current() != &PageTable::kernel()) PageTable::kernel().load(); - *interrupt_stack = m_idle_thread->interrupt_stack(); - *interrupt_registers = m_idle_thread->interrupt_registers(); + *yield_registers = m_idle_thread->yield_registers(); m_idle_thread->m_state = Thread::State::Executing; m_idle_start_ns = SystemTimer::get().ns_since_boot(); return; @@ -296,8 +294,7 @@ namespace Kernel Processor::load_segments(); } - *interrupt_stack = thread->interrupt_stack(); - *interrupt_registers = thread->interrupt_registers(); + *yield_registers = thread->yield_registers(); m_current->last_start_ns = SystemTimer::get().ns_since_boot(); } @@ -333,6 +330,11 @@ namespace Kernel Processor::yield(); } + extern "C" void scheduler_on_yield(YieldRegisters* yield_registers) + { + Processor::scheduler().reschedule(yield_registers); + } + void Scheduler::timer_interrupt() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 73d4c770..9f690493 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -26,7 +26,7 @@ namespace Kernel extern "C" uintptr_t get_thread_start_sp() { - return Thread::current().interrupt_stack().sp; + return Thread::current().yield_registers().sp; } extern "C" void load_thread_sse() @@ -179,13 +179,9 @@ namespace Kernel write_to_stack(sp, data); write_to_stack(sp, entry); - thread->m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); - thread->m_interrupt_stack.cs = 0x08; - thread->m_interrupt_stack.flags = 0x002; - thread->m_interrupt_stack.sp = sp; - thread->m_interrupt_stack.ss = 0x10; - - memset(&thread->m_interrupt_registers, 0, sizeof(InterruptRegisters)); + thread->m_yield_registers = {}; + thread->m_yield_registers.ip = reinterpret_cast(start_kernel_thread); + thread->m_yield_registers.sp = sp; thread_deleter.disable(); @@ -347,20 +343,13 @@ namespace Kernel thread->m_state = State::NotStarted; - thread->m_interrupt_stack.ip = ip; - thread->m_interrupt_stack.cs = 0x08; - thread->m_interrupt_stack.flags = 0x002; - thread->m_interrupt_stack.sp = sp; - thread->m_interrupt_stack.ss = 0x10; - save_sse(); memcpy(thread->m_sse_storage, m_sse_storage, sizeof(m_sse_storage)); -#if ARCH(x86_64) - thread->m_interrupt_registers.rax = 0; -#elif ARCH(i686) - thread->m_interrupt_registers.eax = 0; -#endif + thread->m_yield_registers = {}; + thread->m_yield_registers.ip = ip; + thread->m_yield_registers.sp = sp; + thread->m_yield_registers.ret = 0; thread_deleter.disable(); @@ -498,13 +487,9 @@ namespace Kernel write_to_stack(cur_sp, ip); }); - m_interrupt_stack.ip = reinterpret_cast(start_userspace_thread); - m_interrupt_stack.cs = 0x08; - m_interrupt_stack.flags = 0x002; - m_interrupt_stack.sp = kernel_stack_top() - 5 * sizeof(uintptr_t); - m_interrupt_stack.ss = 0x10; - - memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); + m_yield_registers = {}; + m_yield_registers.ip = reinterpret_cast(start_userspace_thread); + m_yield_registers.sp = kernel_stack_top() - 5 * sizeof(uintptr_t); } void Thread::setup_process_cleanup() @@ -539,13 +524,9 @@ namespace Kernel write_to_stack(sp, entry); }); - m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); - m_interrupt_stack.cs = 0x08; - m_interrupt_stack.flags = 0x002; - m_interrupt_stack.sp = kernel_stack_top() - 4 * sizeof(uintptr_t); - m_interrupt_stack.ss = 0x10; - - memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); + m_yield_registers = {}; + m_yield_registers.ip = reinterpret_cast(start_kernel_thread); + m_yield_registers.sp = kernel_stack_top() - 4 * sizeof(uintptr_t); } bool Thread::is_interrupted_by_signal(bool skip_stop_and_cont) const @@ -811,7 +792,7 @@ namespace Kernel const vaddr_t stack_bottom = reinterpret_cast(m_signal_alt_stack.ss_sp); const vaddr_t stack_top = stack_bottom + m_signal_alt_stack.ss_size; - const vaddr_t sp = m_interrupt_stack.sp; + const vaddr_t sp = m_yield_registers.sp; return stack_bottom <= sp && sp <= stack_top; }