From 35c97e2ff871497d0651e4278336813f07380c8a Mon Sep 17 00:00:00 2001 From: Bananymous Date: Sun, 11 Jan 2026 01:31:09 +0200 Subject: [PATCH] Kernel: optimize yielding Doing a yield no longer raises a software interrupt. Instead it just saves all the callee saved registers, ip, sp and return value. Because yield is only called in the kernel, it can just restore registers and jump to the target address. There is never a need to use iret :) --- kernel/CMakeLists.txt | 2 ++ kernel/arch/i686/Yield.S | 25 +++++++++++++ kernel/arch/i686/interrupts.S | 22 ------------ kernel/arch/x86_64/Yield.S | 29 +++++++++++++++ kernel/arch/x86_64/interrupts.S | 10 ------ kernel/include/kernel/IDT.h | 5 ++- kernel/include/kernel/InterruptStack.h | 23 +++++++++++- kernel/include/kernel/Scheduler.h | 2 +- kernel/include/kernel/Thread.h | 6 ++-- kernel/kernel/IDT.cpp | 9 ----- kernel/kernel/Processor.cpp | 29 ++------------- kernel/kernel/Scheduler.cpp | 16 +++++---- kernel/kernel/Thread.cpp | 49 ++++++++------------------ 13 files changed, 109 insertions(+), 118 deletions(-) create mode 100644 kernel/arch/i686/Yield.S create mode 100644 kernel/arch/x86_64/Yield.S diff --git a/kernel/CMakeLists.txt b/kernel/CMakeLists.txt index ee038c24..46e306fa 100644 --- a/kernel/CMakeLists.txt +++ b/kernel/CMakeLists.txt @@ -137,6 +137,7 @@ if("${BANAN_ARCH}" STREQUAL "x86_64") arch/x86_64/Signal.S arch/x86_64/Syscall.S arch/x86_64/Thread.S + arch/x86_64/Yield.S ) elseif("${BANAN_ARCH}" STREQUAL "i686") set(KERNEL_SOURCES @@ -147,6 +148,7 @@ elseif("${BANAN_ARCH}" STREQUAL "i686") arch/i686/Signal.S arch/i686/Syscall.S arch/i686/Thread.S + arch/i686/Yield.S ) else() message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}") diff --git a/kernel/arch/i686/Yield.S b/kernel/arch/i686/Yield.S new file mode 100644 index 00000000..68b352be --- /dev/null +++ b/kernel/arch/i686/Yield.S @@ -0,0 +1,25 @@ +.global asm_yield_trampoline +asm_yield_trampoline: + movl %esp, %ecx + movl 4(%esp), %esp + + pushl (%ecx) + pushl %ecx + pushl %eax + pushl %ebx + pushl %esi + pushl %edi + pushl %ebp + + pushl %esp + call scheduler_on_yield + addl $4, %esp + + popl %ebp + popl %edi + popl %esi + popl %ebx + popl %eax + movl 4(%esp), %ecx + movl 0(%esp), %esp + jmp *%ecx diff --git a/kernel/arch/i686/interrupts.S b/kernel/arch/i686/interrupts.S index 7bae605f..c355dbe7 100644 --- a/kernel/arch/i686/interrupts.S +++ b/kernel/arch/i686/interrupts.S @@ -83,28 +83,6 @@ irq_stub: addl $8, %esp iret -.global asm_yield_handler -asm_yield_handler: - # This can only be called from kernel, so no segment saving is needed - pushal - cld - - leal 32(%esp), %edi # interrupt stack ptr - movl %esp, %esi # interrupt registers ptr - - movl %esp, %ebp - andl $-16, %esp - - subl $8, %esp - pushl %esi - pushl %edi - call cpp_yield_handler - - movl %ebp, %esp - - popal - iret - .global asm_ipi_handler asm_ipi_handler: pushal diff --git a/kernel/arch/x86_64/Yield.S b/kernel/arch/x86_64/Yield.S new file mode 100644 index 00000000..bec190c4 --- /dev/null +++ b/kernel/arch/x86_64/Yield.S @@ -0,0 +1,29 @@ +.global asm_yield_trampoline +asm_yield_trampoline: + movq %rsp, %rcx + movq %rdi, %rsp + + subq $8, %rsp + pushq (%rcx) + pushq %rcx + pushq %rax + pushq %rbx + pushq %rbp + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + movq %rsp, %rdi + call scheduler_on_yield + + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbp + popq %rbx + popq %rax + movq 8(%rsp), %rcx + movq 0(%rsp), %rsp + jmp *%rcx diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index 0995b353..401951fe 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -71,16 +71,6 @@ irq_stub: addq $16, %rsp iretq -.global asm_yield_handler -asm_yield_handler: - pushaq 8 - cld - leaq 120(%rsp), %rdi # interrupt stack ptr - movq %rsp, %rsi # interrupt register ptr - call cpp_yield_handler - popaq 8 - iretq - .global asm_ipi_handler asm_ipi_handler: pushaq 8 diff --git a/kernel/include/kernel/IDT.h b/kernel/include/kernel/IDT.h index f051dfcb..3da72577 100644 --- a/kernel/include/kernel/IDT.h +++ b/kernel/include/kernel/IDT.h @@ -22,9 +22,8 @@ namespace Kernel #if ARCH(i686) constexpr uint8_t IRQ_SYSCALL = 0xF0; #endif - constexpr uint8_t IRQ_YIELD = 0xF1; - constexpr uint8_t IRQ_IPI = 0xF2; - constexpr uint8_t IRQ_TIMER = 0xF3; + constexpr uint8_t IRQ_IPI = 0xF1; + constexpr uint8_t IRQ_TIMER = 0xF2; #if ARCH(x86_64) struct GateDescriptor diff --git a/kernel/include/kernel/InterruptStack.h b/kernel/include/kernel/InterruptStack.h index 4828bbf3..03006252 100644 --- a/kernel/include/kernel/InterruptStack.h +++ b/kernel/include/kernel/InterruptStack.h @@ -27,7 +27,6 @@ namespace Kernel uintptr_t r10; uintptr_t r9; uintptr_t r8; - uintptr_t rdi; uintptr_t rsi; uintptr_t rbp; @@ -36,6 +35,18 @@ namespace Kernel uintptr_t rcx; uintptr_t rax; }; + struct YieldRegisters + { + uintptr_t r15; + uintptr_t r14; + uintptr_t r13; + uintptr_t r12; + uintptr_t rbp; + uintptr_t rbx; + uintptr_t ret; + uintptr_t sp; + uintptr_t ip; + }; #elif ARCH(i686) struct InterruptRegisters { @@ -48,6 +59,16 @@ namespace Kernel uintptr_t ecx; uintptr_t eax; }; + struct YieldRegisters + { + uintptr_t ebp; + uintptr_t edi; + uintptr_t esi; + uintptr_t ebx; + uintptr_t ret; + uintptr_t sp; + uintptr_t ip; + }; #endif } diff --git a/kernel/include/kernel/Scheduler.h b/kernel/include/kernel/Scheduler.h index 5b3686ee..5b91a648 100644 --- a/kernel/include/kernel/Scheduler.h +++ b/kernel/include/kernel/Scheduler.h @@ -57,7 +57,7 @@ namespace Kernel static BAN::ErrorOr create(); BAN::ErrorOr initialize(); - void reschedule(InterruptStack*, InterruptRegisters*); + void reschedule(YieldRegisters*); void reschedule_if_idle(); void timer_interrupt(); diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index 454c6345..2f163dbe 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -132,8 +132,7 @@ namespace Kernel size_t virtual_page_count() const { return (m_kernel_stack ? (m_kernel_stack->size() / PAGE_SIZE) : 0) + (m_userspace_stack ? (m_userspace_stack->size() / PAGE_SIZE) : 0); } size_t physical_page_count() const { return virtual_page_count(); } - InterruptStack& interrupt_stack() { return m_interrupt_stack; } - InterruptRegisters& interrupt_registers() { return m_interrupt_registers; } + YieldRegisters& yield_registers() { return m_yield_registers; } void save_sse(); void load_sse(); @@ -173,8 +172,7 @@ namespace Kernel SchedulerQueue::Node* m_scheduler_node { nullptr }; - InterruptStack m_interrupt_stack { }; - InterruptRegisters m_interrupt_registers { }; + YieldRegisters m_yield_registers { }; siginfo_t m_signal_infos[_SIGMAX + 1] { }; uint64_t m_signal_pending_mask { 0 }; diff --git a/kernel/kernel/IDT.cpp b/kernel/kernel/IDT.cpp index ebdc5090..03b532e1 100644 --- a/kernel/kernel/IDT.cpp +++ b/kernel/kernel/IDT.cpp @@ -325,14 +325,6 @@ namespace Kernel Thread::current().load_sse(); } - extern "C" void cpp_yield_handler(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) - { - // yield is raised through kernel software interrupt - ASSERT(!InterruptController::get().is_in_service(IRQ_YIELD - IRQ_VECTOR_BASE)); - ASSERT(!GDT::is_user_segment(interrupt_stack->cs)); - Processor::scheduler().reschedule(interrupt_stack, interrupt_registers); - } - extern "C" void cpp_ipi_handler() { ASSERT(InterruptController::get().is_in_service(IRQ_IPI - IRQ_VECTOR_BASE)); @@ -477,7 +469,6 @@ namespace Kernel static_assert(DoubleFault == 8); #endif - idt->register_interrupt_handler(IRQ_YIELD, asm_yield_handler); idt->register_interrupt_handler(IRQ_IPI, asm_ipi_handler); idt->register_interrupt_handler(IRQ_TIMER, asm_timer_handler); #if ARCH(i686) diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 5513e9bb..1ff9d8fd 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -36,6 +36,7 @@ namespace Kernel static BAN::Array s_processor_ids { PROCESSOR_NONE }; extern "C" void asm_syscall_handler(); + extern "C" void asm_yield_trampoline(uintptr_t); ProcessorID Processor::read_processor_id() { @@ -556,33 +557,7 @@ namespace Kernel if (!scheduler().is_idle()) Thread::current().set_cpu_time_stop(); -#if ARCH(x86_64) - asm volatile( - "movq %%rsp, %%rcx;" - "movq %[load_sp], %%rsp;" - "int %[yield];" - "movq %%rcx, %%rsp;" - // NOTE: This is offset by 2 pointers since interrupt without PL change - // does not push SP and SS. This allows accessing "whole" interrupt stack. - :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), - [yield]"i"(static_cast(IRQ_YIELD)) // WTF GCC 15 - : "memory", "rcx" - ); -#elif ARCH(i686) - asm volatile( - "movl %%esp, %%ecx;" - "movl %[load_sp], %%esp;" - "int %[yield];" - "movl %%ecx, %%esp;" - // NOTE: This is offset by 2 pointers since interrupt without PL change - // does not push SP and SS. This allows accessing "whole" interrupt stack. - :: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)), - [yield]"i"(static_cast(IRQ_YIELD)) // WTF GCC 15 - : "memory", "ecx" - ); -#else - #error -#endif + asm_yield_trampoline(Processor::current_stack_top()); processor_info.m_start_ns = SystemTimer::get().ns_since_boot(); diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index 83f826a1..6cb7953e 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -207,7 +207,7 @@ namespace Kernel m_most_loaded_threads.back().queue = nullptr; } - void Scheduler::reschedule(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers) + void Scheduler::reschedule(YieldRegisters* yield_registers) { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); @@ -232,8 +232,7 @@ namespace Kernel case Thread::State::Executing: { const uint64_t current_ns = SystemTimer::get().ns_since_boot(); - m_current->thread->interrupt_stack() = *interrupt_stack; - m_current->thread->interrupt_registers() = *interrupt_registers; + m_current->thread->yield_registers() = *yield_registers; m_current->time_used_ns += current_ns - m_current->last_start_ns; add_current_to_most_loaded(m_current->blocked ? &m_block_queue : &m_run_queue); if (!m_current->blocked) @@ -267,8 +266,7 @@ namespace Kernel { if (&PageTable::current() != &PageTable::kernel()) PageTable::kernel().load(); - *interrupt_stack = m_idle_thread->interrupt_stack(); - *interrupt_registers = m_idle_thread->interrupt_registers(); + *yield_registers = m_idle_thread->yield_registers(); m_idle_thread->m_state = Thread::State::Executing; m_idle_start_ns = SystemTimer::get().ns_since_boot(); return; @@ -296,8 +294,7 @@ namespace Kernel Processor::load_segments(); } - *interrupt_stack = thread->interrupt_stack(); - *interrupt_registers = thread->interrupt_registers(); + *yield_registers = thread->yield_registers(); m_current->last_start_ns = SystemTimer::get().ns_since_boot(); } @@ -333,6 +330,11 @@ namespace Kernel Processor::yield(); } + extern "C" void scheduler_on_yield(YieldRegisters* yield_registers) + { + Processor::scheduler().reschedule(yield_registers); + } + void Scheduler::timer_interrupt() { ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 73d4c770..9f690493 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -26,7 +26,7 @@ namespace Kernel extern "C" uintptr_t get_thread_start_sp() { - return Thread::current().interrupt_stack().sp; + return Thread::current().yield_registers().sp; } extern "C" void load_thread_sse() @@ -179,13 +179,9 @@ namespace Kernel write_to_stack(sp, data); write_to_stack(sp, entry); - thread->m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); - thread->m_interrupt_stack.cs = 0x08; - thread->m_interrupt_stack.flags = 0x002; - thread->m_interrupt_stack.sp = sp; - thread->m_interrupt_stack.ss = 0x10; - - memset(&thread->m_interrupt_registers, 0, sizeof(InterruptRegisters)); + thread->m_yield_registers = {}; + thread->m_yield_registers.ip = reinterpret_cast(start_kernel_thread); + thread->m_yield_registers.sp = sp; thread_deleter.disable(); @@ -347,20 +343,13 @@ namespace Kernel thread->m_state = State::NotStarted; - thread->m_interrupt_stack.ip = ip; - thread->m_interrupt_stack.cs = 0x08; - thread->m_interrupt_stack.flags = 0x002; - thread->m_interrupt_stack.sp = sp; - thread->m_interrupt_stack.ss = 0x10; - save_sse(); memcpy(thread->m_sse_storage, m_sse_storage, sizeof(m_sse_storage)); -#if ARCH(x86_64) - thread->m_interrupt_registers.rax = 0; -#elif ARCH(i686) - thread->m_interrupt_registers.eax = 0; -#endif + thread->m_yield_registers = {}; + thread->m_yield_registers.ip = ip; + thread->m_yield_registers.sp = sp; + thread->m_yield_registers.ret = 0; thread_deleter.disable(); @@ -498,13 +487,9 @@ namespace Kernel write_to_stack(cur_sp, ip); }); - m_interrupt_stack.ip = reinterpret_cast(start_userspace_thread); - m_interrupt_stack.cs = 0x08; - m_interrupt_stack.flags = 0x002; - m_interrupt_stack.sp = kernel_stack_top() - 5 * sizeof(uintptr_t); - m_interrupt_stack.ss = 0x10; - - memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); + m_yield_registers = {}; + m_yield_registers.ip = reinterpret_cast(start_userspace_thread); + m_yield_registers.sp = kernel_stack_top() - 5 * sizeof(uintptr_t); } void Thread::setup_process_cleanup() @@ -539,13 +524,9 @@ namespace Kernel write_to_stack(sp, entry); }); - m_interrupt_stack.ip = reinterpret_cast(start_kernel_thread); - m_interrupt_stack.cs = 0x08; - m_interrupt_stack.flags = 0x002; - m_interrupt_stack.sp = kernel_stack_top() - 4 * sizeof(uintptr_t); - m_interrupt_stack.ss = 0x10; - - memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters)); + m_yield_registers = {}; + m_yield_registers.ip = reinterpret_cast(start_kernel_thread); + m_yield_registers.sp = kernel_stack_top() - 4 * sizeof(uintptr_t); } bool Thread::is_interrupted_by_signal(bool skip_stop_and_cont) const @@ -811,7 +792,7 @@ namespace Kernel const vaddr_t stack_bottom = reinterpret_cast(m_signal_alt_stack.ss_sp); const vaddr_t stack_top = stack_bottom + m_signal_alt_stack.ss_size; - const vaddr_t sp = m_interrupt_stack.sp; + const vaddr_t sp = m_yield_registers.sp; return stack_bottom <= sp && sp <= stack_top; }