diff --git a/kernel/arch/i686/Thread.S b/kernel/arch/i686/Thread.S index 15960e9b..8da4d975 100644 --- a/kernel/arch/i686/Thread.S +++ b/kernel/arch/i686/Thread.S @@ -45,11 +45,11 @@ start_userspace_thread: movw $(0x20 | 3), %bx movw %bx, %ds movw %bx, %es - # gs = thread local + # fs movw $(0x30 | 3), %bx - movw %bx, %gs - # fs = 0 - xorw %bx, %bx movw %bx, %fs + # gs + movw $(0x38 | 3), %bx + movw %bx, %gs iret diff --git a/kernel/arch/x86_64/Syscall.S b/kernel/arch/x86_64/Syscall.S index ac447607..15757dfb 100644 --- a/kernel/arch/x86_64/Syscall.S +++ b/kernel/arch/x86_64/Syscall.S @@ -2,6 +2,7 @@ // System V ABI: RDI, RSI, RDX, RCX, R8, R9 .global asm_syscall_handler asm_syscall_handler: + swapgs pushq %rbx pushq %rcx pushq %rdx @@ -42,9 +43,9 @@ asm_syscall_handler: popq %rdx popq %rcx popq %rbx + swapgs iretq - .global sys_fork_trampoline sys_fork_trampoline: pushq %rbx diff --git a/kernel/arch/x86_64/Thread.S b/kernel/arch/x86_64/Thread.S index ddc07b92..1f58e228 100644 --- a/kernel/arch/x86_64/Thread.S +++ b/kernel/arch/x86_64/Thread.S @@ -32,4 +32,6 @@ start_userspace_thread: call get_thread_start_sp movq %rax, %rsp + swapgs + iretq diff --git a/kernel/arch/x86_64/interrupts.S b/kernel/arch/x86_64/interrupts.S index 8f6effb2..32e4a41c 100644 --- a/kernel/arch/x86_64/interrupts.S +++ b/kernel/arch/x86_64/interrupts.S @@ -1,4 +1,12 @@ -.macro pushaq +.macro swapgs_if_necessary, n + cmpb $0x08, \n(%rsp) + je 1f + swapgs +1: +.endm + +.macro pushaq, n + swapgs_if_necessary \n pushq %rax pushq %rcx pushq %rdx @@ -16,7 +24,7 @@ pushq %r15 .endm -.macro popaq +.macro popaq, n popq %r15 popq %r14 popq %r13 @@ -32,10 +40,11 @@ popq %rdx popq %rcx popq %rax + swapgs_if_necessary \n .endm isr_stub: - pushaq + pushaq 24 cld movq %cr0, %rax; pushq %rax movq %cr2, %rax; pushq %rax @@ -49,43 +58,43 @@ isr_stub: call cpp_isr_handler addq $32, %rsp - popaq + popaq 24 addq $16, %rsp iretq irq_stub: - pushaq + pushaq 24 cld movq 120(%rsp), %rdi # irq number call cpp_irq_handler - popaq + popaq 24 addq $16, %rsp iretq .global asm_yield_handler asm_yield_handler: - pushaq + pushaq 8 cld leaq 120(%rsp), %rdi # interrupt stack ptr movq %rsp, %rsi # interrupt register ptr call cpp_yield_handler - popaq + popaq 8 iretq .global asm_ipi_handler asm_ipi_handler: - pushaq + pushaq 8 cld call cpp_ipi_handler - popaq + popaq 8 iretq .global asm_timer_handler asm_timer_handler: - pushaq + pushaq 8 cld call cpp_timer_handler - popaq + popaq 8 iretq .macro isr n diff --git a/kernel/include/kernel/GDT.h b/kernel/include/kernel/GDT.h index 989b3a72..4e19d10c 100644 --- a/kernel/include/kernel/GDT.h +++ b/kernel/include/kernel/GDT.h @@ -129,7 +129,8 @@ namespace Kernel } #if ARCH(i686) - void set_tls(uintptr_t addr); + void set_fsbase(uintptr_t addr); + void set_gsbase(uintptr_t addr); #endif private: @@ -153,8 +154,8 @@ namespace Kernel BAN::Array m_gdt; // null, kernel code, kernel data, user code, user data, tss low, tss high static constexpr uint16_t m_tss_offset = 0x28; #elif ARCH(i686) - BAN::Array m_gdt; // null, kernel code, kernel data, user code, user data, processor data, tls, tss - static constexpr uint16_t m_tss_offset = 0x38; + BAN::Array m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, tss + static constexpr uint16_t m_tss_offset = 0x40; #endif TaskStateSegment m_tss; const GDTR m_gdtr { diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index 41c85638..e71ea8ab 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -203,8 +203,12 @@ namespace Kernel BAN::ErrorOr sys_futex(int op, const uint32_t* addr, uint32_t val, const timespec* abstime); BAN::ErrorOr sys_yield(); - BAN::ErrorOr sys_set_tls(void*); - BAN::ErrorOr sys_get_tls(); + + BAN::ErrorOr sys_set_fsbase(void*); + BAN::ErrorOr sys_get_fsbase(); + BAN::ErrorOr sys_set_gsbase(void*); + BAN::ErrorOr sys_get_gsbase(); + BAN::ErrorOr sys_pthread_create(const pthread_attr_t* attr, void (*entry)(void*), void* arg); BAN::ErrorOr sys_pthread_exit(void* value); BAN::ErrorOr sys_pthread_join(pthread_t thread, void** value); diff --git a/kernel/include/kernel/Processor.h b/kernel/include/kernel/Processor.h index e0937b5d..afb717a5 100644 --- a/kernel/include/kernel/Processor.h +++ b/kernel/include/kernel/Processor.h @@ -112,7 +112,9 @@ namespace Kernel static void send_smp_message(ProcessorID, const SMPMessage&, bool send_ipi = true); static void broadcast_smp_message(const SMPMessage&); - static void load_tls(); + static void load_segments(); + static void load_fsbase(); + static void load_gsbase(); private: Processor() = default; diff --git a/kernel/include/kernel/Thread.h b/kernel/include/kernel/Thread.h index 05364498..60c869b1 100644 --- a/kernel/include/kernel/Thread.h +++ b/kernel/include/kernel/Thread.h @@ -122,8 +122,10 @@ namespace Kernel void set_cpu_time_start(); void set_cpu_time_stop(); - void set_tls(vaddr_t tls) { m_tls = tls; } - vaddr_t get_tls() const { return m_tls; } + void set_fsbase(vaddr_t base) { m_fsbase = base; } + vaddr_t get_fsbase() const { return m_fsbase; } + void set_gsbase(vaddr_t base) { m_gsbase = base; } + vaddr_t get_gsbase() const { return m_gsbase; } size_t virtual_page_count() const { return (m_kernel_stack ? (m_kernel_stack->size() / PAGE_SIZE) : 0) + (m_userspace_stack ? (m_userspace_stack->size() / PAGE_SIZE) : 0); } size_t physical_page_count() const { return virtual_page_count(); } @@ -164,7 +166,8 @@ namespace Kernel bool m_is_userspace { false }; bool m_delete_process { false }; - vaddr_t m_tls { 0 }; + vaddr_t m_fsbase { 0 }; + vaddr_t m_gsbase { 0 }; SchedulerQueue::Node* m_scheduler_node { nullptr }; diff --git a/kernel/kernel/GDT.cpp b/kernel/kernel/GDT.cpp index 50d4649b..d431a7a9 100644 --- a/kernel/kernel/GDT.cpp +++ b/kernel/kernel/GDT.cpp @@ -29,7 +29,8 @@ namespace Kernel gdt->write_entry(0x20, 0x00000000, 0xFFFFF, 0xF2, data_flags); // user data #if ARCH(i686) gdt->write_entry(0x28, reinterpret_cast(processor), sizeof(Processor), 0x92, 0x4); // processor data - gdt->write_entry(0x30, 0x00000000, 0x00000, 0x00, 0x0); // tls + gdt->write_entry(0x30, 0x00000000, 0x00000, 0x00, 0x0); // fsbase + gdt->write_entry(0x38, 0x00000000, 0x00000, 0x00, 0x0); // gsbase #endif gdt->write_tss(); @@ -37,10 +38,14 @@ namespace Kernel } #if ARCH(i686) - void GDT::set_tls(uintptr_t addr) + void GDT::set_fsbase(uintptr_t addr) { write_entry(0x30, addr, 0xFFFF, 0xF2, 0xC); } + void GDT::set_gsbase(uintptr_t addr) + { + write_entry(0x38, addr, 0xFFFF, 0xF2, 0xC); + } #endif void GDT::write_entry(uint8_t offset, uint32_t base, uint32_t limit, uint8_t access, uint8_t flags) diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 633b627a..3cb15f8c 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -173,7 +173,15 @@ namespace Kernel auxiliary_vector.span() )); if (tls_addr.has_value()) - thread->set_tls(*tls_addr); + { +#if ARCH(x86_64) + thread->set_fsbase(*tls_addr); +#elif ARCH(i686) + thread->set_gsbase(*tls_addr); +#else +#error +#endif + } process->add_thread(thread); process->register_to_scheduler(); @@ -730,7 +738,13 @@ namespace Kernel { auto tls_result = TRY(initialize_thread_local_storage(*new_page_table, *executable.master_tls)); TRY(new_mapped_regions.emplace_back(BAN::move(tls_result.region))); - new_thread->set_tls(tls_result.addr); +#if ARCH(x86_64) + new_thread->set_fsbase(tls_result.addr); +#elif ARCH(i686) + new_thread->set_gsbase(tls_result.addr); +#else +#error +#endif } // NOTE: this is done before disabling interrupts and moving the threads as @@ -3016,16 +3030,28 @@ namespace Kernel return 0; } - BAN::ErrorOr Process::sys_set_tls(void* addr) + BAN::ErrorOr Process::sys_set_fsbase(void* addr) { - Thread::current().set_tls(reinterpret_cast(addr)); - Processor::load_tls(); + Thread::current().set_fsbase(reinterpret_cast(addr)); + Processor::load_fsbase(); return 0; } - BAN::ErrorOr Process::sys_get_tls() + BAN::ErrorOr Process::sys_get_fsbase() { - return Thread::current().get_tls(); + return Thread::current().get_fsbase(); + } + + BAN::ErrorOr Process::sys_set_gsbase(void* addr) + { + Thread::current().set_gsbase(reinterpret_cast(addr)); + Processor::load_gsbase(); + return 0; + } + + BAN::ErrorOr Process::sys_get_gsbase() + { + return Thread::current().get_gsbase(); } BAN::ErrorOr Process::sys_pthread_create(const pthread_attr_t* attr, void (*entry)(void*), void* arg) diff --git a/kernel/kernel/Processor.cpp b/kernel/kernel/Processor.cpp index 879fa121..1a98a45f 100644 --- a/kernel/kernel/Processor.cpp +++ b/kernel/kernel/Processor.cpp @@ -12,6 +12,7 @@ namespace Kernel #if ARCH(x86_64) static constexpr uint32_t MSR_IA32_FS_BASE = 0xC0000100; static constexpr uint32_t MSR_IA32_GS_BASE = 0xC0000101; + static constexpr uint32_t MSR_IA32_KERNEL_GS_BASE = 0xC0000102; #endif ProcessorID Processor::s_bsp_id { PROCESSOR_NONE }; @@ -265,15 +266,52 @@ namespace Kernel set_interrupt_state(state); } - void Processor::load_tls() + void Processor::load_segments() { - const auto addr = scheduler().current_thread().get_tls(); + { + const auto addr = scheduler().current_thread().get_fsbase(); +#if ARCH(x86_64) + uint32_t ptr_hi = addr >> 32; + uint32_t ptr_lo = addr & 0xFFFFFFFF; + asm volatile("wrmsr" :: "d"(ptr_hi), "a"(ptr_lo), "c"(MSR_IA32_FS_BASE)); +#elif ARCH(i686) + gdt().set_fsbase(addr); +#endif + } + + { + const auto addr = scheduler().current_thread().get_gsbase(); +#if ARCH(x86_64) + uint32_t ptr_hi = addr >> 32; + uint32_t ptr_lo = addr & 0xFFFFFFFF; + asm volatile("wrmsr" :: "d"(ptr_hi), "a"(ptr_lo), "c"(MSR_IA32_KERNEL_GS_BASE)); +#elif ARCH(i686) + gdt().set_gsbase(addr); +#endif + } + } + + void Processor::load_fsbase() + { + const auto addr = scheduler().current_thread().get_fsbase(); #if ARCH(x86_64) uint32_t ptr_hi = addr >> 32; uint32_t ptr_lo = addr & 0xFFFFFFFF; asm volatile("wrmsr" :: "d"(ptr_hi), "a"(ptr_lo), "c"(MSR_IA32_FS_BASE)); #elif ARCH(i686) - gdt().set_tls(addr); + gdt().set_fsbase(addr); +#endif + } + + void Processor::load_gsbase() + { + const auto addr = scheduler().current_thread().get_gsbase(); +#if ARCH(x86_64) + uint32_t ptr_hi = addr >> 32; + uint32_t ptr_lo = addr & 0xFFFFFFFF; + asm volatile("wrmsr" :: "d"(ptr_hi), "a"(ptr_lo), "c"(MSR_IA32_KERNEL_GS_BASE)); +#elif ARCH(i686) + gdt().set_gsbase(addr); #endif } diff --git a/kernel/kernel/Scheduler.cpp b/kernel/kernel/Scheduler.cpp index ba261dc6..dcbbe62d 100644 --- a/kernel/kernel/Scheduler.cpp +++ b/kernel/kernel/Scheduler.cpp @@ -282,7 +282,8 @@ namespace Kernel } Processor::gdt().set_tss_stack(thread->kernel_stack_top()); - Processor::load_tls(); + if (thread->is_userspace()) + Processor::load_segments(); *interrupt_stack = thread->interrupt_stack(); *interrupt_registers = thread->interrupt_registers(); diff --git a/kernel/kernel/Thread.cpp b/kernel/kernel/Thread.cpp index 7b310a74..e0e5fc7a 100644 --- a/kernel/kernel/Thread.cpp +++ b/kernel/kernel/Thread.cpp @@ -328,7 +328,8 @@ namespace Kernel thread->m_kernel_stack = TRY(m_kernel_stack->clone(new_process->page_table())); thread->m_userspace_stack = TRY(m_userspace_stack->clone(new_process->page_table())); - thread->m_tls = m_tls; + thread->m_fsbase = m_fsbase; + thread->m_gsbase = m_gsbase; thread->m_state = State::NotStarted; diff --git a/userspace/libraries/LibC/include/sys/syscall.h b/userspace/libraries/LibC/include/sys/syscall.h index 7d236e52..a917dd26 100644 --- a/userspace/libraries/LibC/include/sys/syscall.h +++ b/userspace/libraries/LibC/include/sys/syscall.h @@ -101,8 +101,10 @@ __BEGIN_DECLS O(SYS_HARDLINKAT, hardlinkat) \ O(SYS_UTIMENSAT, utimensat) \ O(SYS_YIELD, yield) \ - O(SYS_SET_TLS, set_tls) \ - O(SYS_GET_TLS, get_tls) \ + O(SYS_SET_FSBASE, set_fsbase) \ + O(SYS_GET_FSBASE, get_fsbase) \ + O(SYS_SET_GSBASE, set_gsbase) \ + O(SYS_GET_GSBASE, get_gsbase) \ O(SYS_PTHREAD_CREATE, pthread_create) \ O(SYS_PTHREAD_EXIT, pthread_exit) \ O(SYS_PTHREAD_JOIN, pthread_join) \ diff --git a/userspace/libraries/LibC/pthread.cpp b/userspace/libraries/LibC/pthread.cpp index 3a37ad96..3befd45c 100644 --- a/userspace/libraries/LibC/pthread.cpp +++ b/userspace/libraries/LibC/pthread.cpp @@ -49,7 +49,13 @@ extern "C" void _pthread_trampoline_cpp(void* arg) { auto info = *reinterpret_cast(arg); info.uthread->id = syscall(SYS_PTHREAD_SELF); - syscall(SYS_SET_TLS, info.uthread); +#if defined(__x86_64__) + syscall(SYS_SET_FSBASE, info.uthread); +#elif defined(__i686__) + syscall(SYS_SET_GSBASE, info.uthread); +#else +#error +#endif free(arg); pthread_exit(info.start_routine(info.arg)); ASSERT_NOT_REACHED(); diff --git a/userspace/libraries/LibC/unistd.cpp b/userspace/libraries/LibC/unistd.cpp index d2abfc4e..f8200651 100644 --- a/userspace/libraries/LibC/unistd.cpp +++ b/userspace/libraries/LibC/unistd.cpp @@ -34,7 +34,13 @@ extern "C" void _init_libc(char** environ, init_funcs_t init_funcs, init_funcs_t if (::environ == nullptr) ::environ = environ; - if (uthread* self = reinterpret_cast(syscall(SYS_GET_TLS))) +#if defined(__x86_64__) + if (uthread* self = reinterpret_cast(syscall(SYS_GET_FSBASE))) +#elif defined(__i686__) + if (uthread* self = reinterpret_cast(syscall(SYS_GET_GSBASE))) +#else +#error +#endif { self->cleanup_stack = nullptr; self->id = syscall(SYS_PTHREAD_SELF); @@ -61,7 +67,13 @@ extern "C" void _init_libc(char** environ, init_funcs_t init_funcs, init_funcs_t }; uthread.dtv[0] = 0; - syscall(SYS_SET_TLS, &uthread); +#if defined(__x86_64__) + syscall(SYS_SET_FSBASE, &uthread); +#elif defined(__i686__) + syscall(SYS_SET_GSBASE, &uthread); +#else +#error +#endif } // call global constructors diff --git a/userspace/programs/DynamicLoader/main.cpp b/userspace/programs/DynamicLoader/main.cpp index cffbc6d3..b2d90bd7 100644 --- a/userspace/programs/DynamicLoader/main.cpp +++ b/userspace/programs/DynamicLoader/main.cpp @@ -1366,7 +1366,13 @@ static void initialize_tls(MasterTLS master_tls) uthread.dtv[elf.tls_module] = reinterpret_cast(tls_addr) + uthread.master_tls_size - elf.tls_offset; } - syscall(SYS_SET_TLS, &uthread); +#if defined(__x86_64__) + syscall(SYS_SET_FSBASE, &uthread); +#elif defined(__i686__) + syscall(SYS_SET_GSBASE, &uthread); +#else +#error +#endif } static void initialize_environ(char** envp)