kernel/CMakeLists.txt

@@ -137,7 +137,6 @@ if("${BANAN_ARCH}" STREQUAL "x86_64")
 		arch/x86_64/Signal.S
 		arch/x86_64/Syscall.S
 		arch/x86_64/Thread.S
-		arch/x86_64/Yield.S
 	)
 elseif("${BANAN_ARCH}" STREQUAL "i686")
 	set(KERNEL_SOURCES
@@ -148,7 +147,6 @@ elseif("${BANAN_ARCH}" STREQUAL "i686")
 		arch/i686/Signal.S
 		arch/i686/Syscall.S
 		arch/i686/Thread.S
-		arch/i686/Yield.S
 	)
 else()
 	message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}")

kernel/arch/i686/Thread.S

@@ -34,6 +34,8 @@ start_kernel_thread:
 
 .global start_userspace_thread
 start_userspace_thread:
+	call load_thread_sse
+
 	call get_thread_start_sp
 	movl %eax, %esp
 

kernel/arch/i686/Yield.S

@@ -1,25 +0,0 @@
-.global asm_yield_trampoline
-asm_yield_trampoline:
-	movl %esp, %ecx
-	movl 4(%esp), %esp
-
-	pushl (%ecx)
-	pushl %ecx
-	pushl %eax
-	pushl %ebx
-	pushl %esi
-	pushl %edi
-	pushl %ebp
-
-	pushl %esp
-	call scheduler_on_yield
-	addl $4, %esp
-
-	popl %ebp
-	popl %edi
-	popl %esi
-	popl %ebx
-	popl %eax
-	movl 4(%esp), %ecx
-	movl 0(%esp), %esp
-	jmp *%ecx

kernel/arch/i686/interrupts.S

@@ -83,6 +83,28 @@ irq_stub:
 	addl $8, %esp
 	iret
 
+.global asm_yield_handler
+asm_yield_handler:
+	# This can only be called from kernel, so no segment saving is needed
+	pushal
+	cld
+
+	leal 32(%esp), %edi # interrupt stack ptr
+	movl %esp,     %esi # interrupt registers ptr
+
+	movl %esp, %ebp
+	andl $-16, %esp
+
+	subl $8, %esp
+	pushl %esi
+	pushl %edi
+	call cpp_yield_handler
+
+	movl %ebp, %esp
+
+	popal
+	iret
+
 .global asm_ipi_handler
 asm_ipi_handler:
 	pushal

kernel/arch/x86_64/Thread.S

@@ -27,6 +27,8 @@ start_kernel_thread:
 
 .global start_userspace_thread
 start_userspace_thread:
+	call load_thread_sse
+
 	call get_thread_start_sp
 	movq %rax, %rsp
 

kernel/arch/x86_64/Yield.S

@@ -1,29 +0,0 @@
-.global asm_yield_trampoline
-asm_yield_trampoline:
-	movq %rsp, %rcx
-	movq %rdi, %rsp
-
-	subq $8, %rsp
-	pushq (%rcx)
-	pushq %rcx
-	pushq %rax
-	pushq %rbx
-	pushq %rbp
-	pushq %r12
-	pushq %r13
-	pushq %r14
-	pushq %r15
-
-	movq %rsp, %rdi
-	call scheduler_on_yield
-
-	popq %r15
-	popq %r14
-	popq %r13
-	popq %r12
-	popq %rbp
-	popq %rbx
-	popq %rax
-	movq 8(%rsp), %rcx
-	movq 0(%rsp), %rsp
-	jmp *%rcx

kernel/arch/x86_64/interrupts.S

@@ -71,6 +71,16 @@ irq_stub:
 	addq $16, %rsp
 	iretq
 
+.global asm_yield_handler
+asm_yield_handler:
+	pushaq 8
+	cld
+	leaq 120(%rsp), %rdi # interrupt stack ptr
+	movq     %rsp,  %rsi # interrupt register ptr
+	call cpp_yield_handler
+	popaq 8
+	iretq
+
 .global asm_ipi_handler
 asm_ipi_handler:
 	pushaq 8

kernel/include/kernel/IDT.h

@@ -22,8 +22,9 @@ namespace Kernel
 #if ARCH(i686)
 	constexpr uint8_t IRQ_SYSCALL       = 0xF0;
 #endif
-	constexpr uint8_t IRQ_IPI           = 0xF1;
+	constexpr uint8_t IRQ_YIELD         = 0xF1;
-	constexpr uint8_t IRQ_TIMER         = 0xF2;
+	constexpr uint8_t IRQ_IPI           = 0xF2;
+	constexpr uint8_t IRQ_TIMER         = 0xF3;
 
 #if ARCH(x86_64)
 	struct GateDescriptor

kernel/include/kernel/InterruptStack.h

@@ -27,6 +27,7 @@ namespace Kernel
 		uintptr_t r10;
 		uintptr_t r9;
 		uintptr_t r8;
+
 		uintptr_t rdi;
 		uintptr_t rsi;
 		uintptr_t rbp;
@@ -35,18 +36,6 @@ namespace Kernel
 		uintptr_t rcx;
 		uintptr_t rax;
 	};
-	struct YieldRegisters
-	{
-		uintptr_t r15;
-		uintptr_t r14;
-		uintptr_t r13;
-		uintptr_t r12;
-		uintptr_t rbp;
-		uintptr_t rbx;
-		uintptr_t ret;
-		uintptr_t sp;
-		uintptr_t ip;
-	};
 #elif ARCH(i686)
 	struct InterruptRegisters
 	{
@@ -59,16 +48,6 @@ namespace Kernel
 		uintptr_t ecx;
 		uintptr_t eax;
 	};
-	struct YieldRegisters
-	{
-		uintptr_t ebp;
-		uintptr_t edi;
-		uintptr_t esi;
-		uintptr_t ebx;
-		uintptr_t ret;
-		uintptr_t sp;
-		uintptr_t ip;
-	};
 #endif
 
 }

kernel/include/kernel/Processor.h

@@ -120,9 +120,6 @@ namespace Kernel
 		static void update_tsc();
 		static uint64_t ns_since_boot_tsc();
 
-		static Thread* get_current_sse_thread() { return read_gs_sized<Thread*>(offsetof(Processor, m_sse_thread)); };
-		static void set_current_sse_thread(Thread* thread) { write_gs_sized<Thread*>(offsetof(Processor, m_sse_thread), thread); };
-
 		static paddr_t shared_page_paddr() { return s_shared_page_paddr; }
 		static volatile API::SharedPage& shared_page() { return *reinterpret_cast<API::SharedPage*>(s_shared_page_vaddr); }
 
@@ -136,21 +133,6 @@ namespace Kernel
 		static void load_fsbase();
 		static void load_gsbase();
 
-		static void disable_sse()
-		{
-			uintptr_t dummy;
-#if ARCH(x86_64)
-			asm volatile("movq %%cr0, %0; orq $0x08, %0; movq %0, %%cr0" : "=r"(dummy));
-#elif ARCH(i686)
-			asm volatile("movl %%cr0, %0; orl $0x08, %0; movl %0, %%cr0" : "=r"(dummy));
-#endif
-		}
-
-		static void enable_sse()
-		{
-			asm volatile("clts");
-		}
-
 	private:
 		Processor() = default;
 		~Processor() { ASSERT_NOT_REACHED(); }
@@ -212,8 +194,6 @@ namespace Kernel
 
 		vaddr_t m_thread_syscall_stack;
 
-		Thread* m_sse_thread { nullptr };
-
 		static constexpr size_t s_stack_size { 4096 };
 		void* m_stack { nullptr };
 

kernel/include/kernel/Scheduler.h

@@ -57,7 +57,7 @@ namespace Kernel
 		static BAN::ErrorOr<Scheduler*> create();
 		BAN::ErrorOr<void> initialize();
 
-		void reschedule(YieldRegisters*);
+		void reschedule(InterruptStack*, InterruptRegisters*);
 		void reschedule_if_idle();
 
 		void timer_interrupt();

kernel/include/kernel/Thread.h

@@ -132,7 +132,8 @@ namespace Kernel
 		size_t virtual_page_count() const { return (m_kernel_stack ? (m_kernel_stack->size() / PAGE_SIZE) : 0) + (m_userspace_stack ? (m_userspace_stack->size() / PAGE_SIZE) : 0); }
 		size_t physical_page_count() const { return virtual_page_count(); }
 
-		YieldRegisters& yield_registers() { return m_yield_registers; }
+		InterruptStack& interrupt_stack() { return m_interrupt_stack; }
+		InterruptRegisters& interrupt_registers() { return m_interrupt_registers; }
 
 		void save_sse();
 		void load_sse();
@@ -172,7 +173,8 @@ namespace Kernel
 
 		SchedulerQueue::Node*      m_scheduler_node       { nullptr };
 
-		YieldRegisters             m_yield_registers      { };
+		InterruptStack             m_interrupt_stack      { };
+		InterruptRegisters         m_interrupt_registers  { };
 
 		siginfo_t                  m_signal_infos[_SIGMAX + 1] { };
 		uint64_t                   m_signal_pending_mask  { 0 };

kernel/kernel/IDT.cpp

@@ -177,13 +177,17 @@ namespace Kernel
 		const pid_t tid = Thread::current_tid();
 		const pid_t pid = (tid && Thread::current().has_process()) ? Process::current().pid() : 0;
 
-		switch (isr)
+		const char* process_name = "";
-		{
-			case ISR::PageFault:
-			{
-				if (pid == 0 || !Thread::current().is_userspace())
-					break;
 
+		if (tid)
+		{
+			auto& thread = Thread::current();
+			thread.save_sse();
+
+			if (isr == ISR::PageFault && Thread::current().is_userspace())
+			{
+				if (pid)
+				{
 					PageFaultError page_fault_error;
 					page_fault_error.raw = error;
 
@@ -191,33 +195,16 @@ namespace Kernel
 					auto result = Process::current().allocate_page_for_demand_paging(regs->cr2, page_fault_error.write, page_fault_error.instruction);
 					Processor::set_interrupt_state(InterruptState::Disabled);
 
+					if (!result.is_error() && result.value())
+						goto done;
+
 					if (result.is_error())
 					{
 						dwarnln("Demand paging: {}", result.error());
 						Thread::current().handle_signal(SIGKILL, {});
-					return;
+						goto done;
 					}
-
-				if (result.value())
-					return;
-
-				break;
 				}
-			case ISR::DeviceNotAvailable:
-			{
-				if (pid == 0 || !Thread::current().is_userspace())
-					break;
-
-				Processor::enable_sse();
-
-				if (auto* sse_thread = Processor::get_current_sse_thread())
-					sse_thread->save_sse();
-
-				auto* current_thread = &Thread::current();
-				current_thread->load_sse();
-				Processor::set_current_sse_thread(current_thread);
-
-				return;
 			}
 		}
 
@@ -238,9 +225,8 @@ namespace Kernel
 			);
 		}
 
-		const char* process_name = (tid && Thread::current().has_process())
+		if (Thread::current().has_process())
-			? Process::current().name()
+			process_name = Process::current().name();
-			: nullptr;
 
 #if ARCH(x86_64)
 		dwarnln(
@@ -334,6 +320,17 @@ namespace Kernel
 		}
 
 		ASSERT(Thread::current().state() != Thread::State::Terminated);
+
+	done:
+		Thread::current().load_sse();
+	}
+
+	extern "C" void cpp_yield_handler(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers)
+	{
+		// yield is raised through kernel software interrupt
+		ASSERT(!InterruptController::get().is_in_service(IRQ_YIELD - IRQ_VECTOR_BASE));
+		ASSERT(!GDT::is_user_segment(interrupt_stack->cs));
+		Processor::scheduler().reschedule(interrupt_stack, interrupt_registers);
 	}
 
 	extern "C" void cpp_ipi_handler()
@@ -354,6 +351,8 @@ namespace Kernel
 			asm volatile("cli; 1: hlt; jmp 1b");
 		}
 
+		Thread::current().save_sse();
+
 		ASSERT(InterruptController::get().is_in_service(IRQ_TIMER - IRQ_VECTOR_BASE));
 		InterruptController::get().eoi(IRQ_TIMER - IRQ_VECTOR_BASE);
 
@@ -365,6 +364,8 @@ namespace Kernel
 		auto& current_thread = Thread::current();
 		if (current_thread.can_add_signal_to_execute())
 			current_thread.handle_signal();
+
+		Thread::current().load_sse();
 	}
 
 	extern "C" void cpp_irq_handler(uint32_t irq)
@@ -382,6 +383,8 @@ namespace Kernel
 		if (!InterruptController::get().is_in_service(irq))
 			return;
 
+		Thread::current().save_sse();
+
 		InterruptController::get().eoi(irq);
 		if (auto* handler = s_interruptables[irq])
 			handler->handle_irq();
@@ -395,6 +398,8 @@ namespace Kernel
 		Processor::scheduler().reschedule_if_idle();
 
 		ASSERT(Thread::current().state() != Thread::State::Terminated);
+
+		Thread::current().load_sse();
 	}
 
 	void IDT::register_interrupt_handler(uint8_t index, void (*handler)(), uint8_t ist)
@@ -472,6 +477,7 @@ namespace Kernel
 		static_assert(DoubleFault == 8);
 #endif
 
+		idt->register_interrupt_handler(IRQ_YIELD, asm_yield_handler);
 		idt->register_interrupt_handler(IRQ_IPI,   asm_ipi_handler);
 		idt->register_interrupt_handler(IRQ_TIMER, asm_timer_handler);
 #if ARCH(i686)

kernel/kernel/Processor.cpp

@@ -36,7 +36,6 @@ namespace Kernel
 	static BAN::Array<ProcessorID, 0xFF> s_processor_ids { PROCESSOR_NONE };
 
 	extern "C" void asm_syscall_handler();
-	extern "C" void asm_yield_trampoline(uintptr_t);
 
 	ProcessorID Processor::read_processor_id()
 	{
@@ -147,8 +146,6 @@ namespace Kernel
 		ASSERT(processor.m_idt);
 		processor.idt().load();
 
-		disable_sse();
-
 		return processor;
 	}
 
@@ -559,7 +556,33 @@ namespace Kernel
 		if (!scheduler().is_idle())
 			Thread::current().set_cpu_time_stop();
 
-		asm_yield_trampoline(Processor::current_stack_top());
+#if ARCH(x86_64)
+		asm volatile(
+			"movq %%rsp, %%rcx;"
+			"movq %[load_sp], %%rsp;"
+			"int %[yield];"
+			"movq %%rcx, %%rsp;"
+			// NOTE: This is offset by 2 pointers since interrupt without PL change
+			//       does not push SP and SS. This allows accessing "whole" interrupt stack.
+			:: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)),
+			   [yield]"i"(static_cast<int>(IRQ_YIELD)) // WTF GCC 15
+			:  "memory", "rcx"
+		);
+#elif ARCH(i686)
+		asm volatile(
+			"movl %%esp, %%ecx;"
+			"movl %[load_sp], %%esp;"
+			"int %[yield];"
+			"movl %%ecx, %%esp;"
+			// NOTE: This is offset by 2 pointers since interrupt without PL change
+			//       does not push SP and SS. This allows accessing "whole" interrupt stack.
+			:: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)),
+			   [yield]"i"(static_cast<int>(IRQ_YIELD)) // WTF GCC 15
+			:  "memory", "ecx"
+		);
+#else
+		#error
+#endif
 
 		processor_info.m_start_ns = SystemTimer::get().ns_since_boot();
 

kernel/kernel/Scheduler.cpp

@@ -207,7 +207,7 @@ namespace Kernel
 		m_most_loaded_threads.back().queue = nullptr;
 	}
 
-	void Scheduler::reschedule(YieldRegisters* yield_registers)
+	void Scheduler::reschedule(InterruptStack* interrupt_stack, InterruptRegisters* interrupt_registers)
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
 
@@ -232,7 +232,8 @@ namespace Kernel
 				case Thread::State::Executing:
 				{
 					const uint64_t current_ns = SystemTimer::get().ns_since_boot();
-					m_current->thread->yield_registers() = *yield_registers;
+					m_current->thread->interrupt_stack()     = *interrupt_stack;
+					m_current->thread->interrupt_registers() = *interrupt_registers;
 					m_current->time_used_ns += current_ns - m_current->last_start_ns;
 					add_current_to_most_loaded(m_current->blocked ? &m_block_queue : &m_run_queue);
 					if (!m_current->blocked)
@@ -266,7 +267,8 @@ namespace Kernel
 		{
 			if (&PageTable::current() != &PageTable::kernel())
 				PageTable::kernel().load();
-			*yield_registers = m_idle_thread->yield_registers();
+			*interrupt_stack       = m_idle_thread->interrupt_stack();
+			*interrupt_registers   = m_idle_thread->interrupt_registers();
 			m_idle_thread->m_state = Thread::State::Executing;
 			m_idle_start_ns        = SystemTimer::get().ns_since_boot();
 			return;
@@ -294,11 +296,8 @@ namespace Kernel
 			Processor::load_segments();
 		}
 
-		(Processor::get_current_sse_thread() == thread)
+		*interrupt_stack     = thread->interrupt_stack();
-			? Processor::enable_sse()
+		*interrupt_registers = thread->interrupt_registers();
-			: Processor::disable_sse();
-
-		*yield_registers = thread->yield_registers();
 
 		m_current->last_start_ns = SystemTimer::get().ns_since_boot();
 	}
@@ -334,11 +333,6 @@ namespace Kernel
 			Processor::yield();
 	}
 
-	extern "C" void scheduler_on_yield(YieldRegisters* yield_registers)
-	{
-		Processor::scheduler().reschedule(yield_registers);
-	}
-
 	void Scheduler::timer_interrupt()
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
@@ -572,14 +566,6 @@ namespace Kernel
 				dprintln_if(DEBUG_SCHEDULER, "CPU {}: sending tid {} to CPU {}", Processor::current_id(), thread_info.node->thread->tid(), least_loaded_id);
 			}
 
-			if (auto* thread = thread_info.node->thread; thread == Processor::get_current_sse_thread())
-			{
-				Processor::enable_sse();
-				thread->save_sse();
-				Processor::set_current_sse_thread(nullptr);
-				Processor::disable_sse();
-			}
-
 			thread_info.node->time_used_ns = 0;
 
 			{

kernel/kernel/Thread.cpp

@@ -26,7 +26,12 @@ namespace Kernel
 
 	extern "C" uintptr_t get_thread_start_sp()
 	{
-		return Thread::current().yield_registers().sp;
+		return Thread::current().interrupt_stack().sp;
+	}
+
+	extern "C" void load_thread_sse()
+	{
+		Thread::current().load_sse();
 	}
 
 	static pid_t s_next_tid = 1;
@@ -46,11 +51,6 @@ namespace Kernel
 			return;
 		}
 
-		const auto state = Processor::get_interrupt_state();
-		Processor::set_interrupt_state(InterruptState::Disabled);
-
-		Processor::enable_sse();
-
 		const uint32_t mxcsr = 0x1F80;
 		asm volatile(
 			"finit;"
@@ -66,10 +66,6 @@ namespace Kernel
 			: [mxcsr]"m"(mxcsr)
 		);
 
-		Processor::disable_sse();
-
-		Processor::set_interrupt_state(state);
-
 		s_default_sse_storage_initialized = true;
 	}
 
@@ -183,9 +179,13 @@ namespace Kernel
 		write_to_stack(sp, data);
 		write_to_stack(sp, entry);
 
-		thread->m_yield_registers = {};
+		thread->m_interrupt_stack.ip = reinterpret_cast<vaddr_t>(start_kernel_thread);
-		thread->m_yield_registers.ip = reinterpret_cast<vaddr_t>(start_kernel_thread);
+		thread->m_interrupt_stack.cs = 0x08;
-		thread->m_yield_registers.sp = sp;
+		thread->m_interrupt_stack.flags = 0x002;
+		thread->m_interrupt_stack.sp = sp;
+		thread->m_interrupt_stack.ss = 0x10;
+
+		memset(&thread->m_interrupt_registers, 0, sizeof(InterruptRegisters));
 
 		thread_deleter.disable();
 
@@ -265,12 +265,6 @@ namespace Kernel
 
 	Thread::~Thread()
 	{
-		if (Processor::get_current_sse_thread() == this)
-		{
-			Processor::set_current_sse_thread(nullptr);
-			Processor::disable_sse();
-		}
-
 		if (m_delete_process)
 		{
 			ASSERT(m_process);
@@ -353,13 +347,20 @@ namespace Kernel
 
 		thread->m_state = State::NotStarted;
 
+		thread->m_interrupt_stack.ip = ip;
+		thread->m_interrupt_stack.cs = 0x08;
+		thread->m_interrupt_stack.flags = 0x002;
+		thread->m_interrupt_stack.sp = sp;
+		thread->m_interrupt_stack.ss = 0x10;
+
 		save_sse();
 		memcpy(thread->m_sse_storage, m_sse_storage, sizeof(m_sse_storage));
 
-		thread->m_yield_registers = {};
+#if ARCH(x86_64)
-		thread->m_yield_registers.ip = ip;
+		thread->m_interrupt_registers.rax = 0;
-		thread->m_yield_registers.sp = sp;
+#elif ARCH(i686)
-		thread->m_yield_registers.ret = 0;
+		thread->m_interrupt_registers.eax = 0;
+#endif
 
 		thread_deleter.disable();
 
@@ -497,9 +498,13 @@ namespace Kernel
 			write_to_stack(cur_sp, ip);
 		});
 
-		m_yield_registers = {};
+		m_interrupt_stack.ip = reinterpret_cast<vaddr_t>(start_userspace_thread);
-		m_yield_registers.ip = reinterpret_cast<vaddr_t>(start_userspace_thread);
+		m_interrupt_stack.cs = 0x08;
-		m_yield_registers.sp = kernel_stack_top() - 5 * sizeof(uintptr_t);
+		m_interrupt_stack.flags = 0x002;
+		m_interrupt_stack.sp = kernel_stack_top() - 5 * sizeof(uintptr_t);
+		m_interrupt_stack.ss = 0x10;
+
+		memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters));
 	}
 
 	void Thread::setup_process_cleanup()
@@ -534,9 +539,13 @@ namespace Kernel
 			write_to_stack(sp, entry);
 		});
 
-		m_yield_registers = {};
+		m_interrupt_stack.ip = reinterpret_cast<vaddr_t>(start_kernel_thread);
-		m_yield_registers.ip = reinterpret_cast<vaddr_t>(start_kernel_thread);
+		m_interrupt_stack.cs = 0x08;
-		m_yield_registers.sp = kernel_stack_top() - 4 * sizeof(uintptr_t);
+		m_interrupt_stack.flags = 0x002;
+		m_interrupt_stack.sp = kernel_stack_top() - 4 * sizeof(uintptr_t);
+		m_interrupt_stack.ss = 0x10;
+
+		memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters));
 	}
 
 	bool Thread::is_interrupted_by_signal(bool skip_stop_and_cont) const
@@ -802,7 +811,7 @@ namespace Kernel
 
 		const vaddr_t stack_bottom = reinterpret_cast<vaddr_t>(m_signal_alt_stack.ss_sp);
 		const vaddr_t stack_top = stack_bottom + m_signal_alt_stack.ss_size;
-		const vaddr_t sp = m_yield_registers.sp;
+		const vaddr_t sp = m_interrupt_stack.sp;
 		return stack_bottom <= sp && sp <= stack_top;
 	}