From ac90800c3c3225ecbc693a4a3c6f7cd4e08f5cc5 Mon Sep 17 00:00:00 2001 From: Bananymous Date: Tue, 15 Apr 2025 23:33:39 +0300 Subject: [PATCH] Kernel/LibC/DynamicLoader: Implement thread local storage For some reason this does not work on 32 bit version, so it is disabled on that platform. I'll have to look into it later to find the bug :) --- kernel/include/kernel/ELF.h | 9 +- kernel/include/kernel/Process.h | 8 + kernel/kernel/ELF.cpp | 62 +++++++- kernel/kernel/Process.cpp | 79 +++++++++- userspace/libraries/LibC/CMakeLists.txt | 5 + userspace/libraries/LibC/errno.cpp | 4 + userspace/libraries/LibC/include/pthread.h | 9 ++ userspace/libraries/LibC/pthread.cpp | 108 +++++++++++++- userspace/libraries/LibC/unistd.cpp | 17 +++ userspace/programs/DynamicLoader/main.cpp | 161 +++++++++++++++++++++ 10 files changed, 447 insertions(+), 15 deletions(-) diff --git a/kernel/include/kernel/ELF.h b/kernel/include/kernel/ELF.h index b777a939..cb6c13ec 100644 --- a/kernel/include/kernel/ELF.h +++ b/kernel/include/kernel/ELF.h @@ -8,8 +8,15 @@ namespace Kernel::ELF struct LoadResult { - bool has_interpreter; + struct TLS + { + vaddr_t addr; + size_t size; + }; + + bool open_execfd; vaddr_t entry_point; + BAN::Optional master_tls; BAN::Vector> regions; }; diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index 7ff5da7e..ee2b1adc 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -219,6 +220,13 @@ namespace Kernel Process(const Credentials&, pid_t pid, pid_t parent, pid_t sid, pid_t pgrp); static Process* create_process(const Credentials&, pid_t parent, pid_t sid = 0, pid_t pgrp = 0); + struct TLSResult + { + BAN::UniqPtr region; + vaddr_t addr; + }; + static BAN::ErrorOr initialize_thread_local_storage(PageTable&, ELF::LoadResult::TLS master_tls); + struct FileParent { VirtualFileSystem::File parent; diff --git a/kernel/kernel/ELF.cpp b/kernel/kernel/ELF.cpp index 372ffea3..b491950d 100644 --- a/kernel/kernel/ELF.cpp +++ b/kernel/kernel/ELF.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace Kernel::ELF { @@ -108,13 +109,13 @@ namespace Kernel::ELF auto file_header = TRY(read_and_validate_file_header(inode)); auto program_headers = TRY(read_program_headers(inode, file_header)); - vaddr_t executable_end = 0; + size_t exec_max_offset { 0 }; BAN::String interpreter; for (const auto& program_header : program_headers) { if (program_header.p_type == PT_LOAD) - executable_end = BAN::Math::max(executable_end, program_header.p_vaddr + program_header.p_memsz); + exec_max_offset = BAN::Math::max(exec_max_offset, program_header.p_vaddr + program_header.p_memsz); else if (program_header.p_type == PT_INTERP) { BAN::Vector interp_buffer; @@ -140,9 +141,6 @@ namespace Kernel::ELF } } - if (file_header.e_type == ET_DYN) - executable_end = 0x400000; - if (!interpreter.empty()) { auto interpreter_inode = TRY(VirtualFileSystem::get().file_from_absolute_path(credentials, interpreter, O_EXEC)).inode; @@ -164,15 +162,16 @@ namespace Kernel::ELF } const vaddr_t load_base_vaddr = - [&file_header, executable_end]() -> vaddr_t + [&file_header, exec_max_offset]() -> vaddr_t { if (file_header.e_type == ET_EXEC) return 0; if (file_header.e_type == ET_DYN) - return (executable_end + PAGE_SIZE - 1) & PAGE_ADDR_MASK; + return (exec_max_offset + PAGE_SIZE - 1) & PAGE_ADDR_MASK; ASSERT_NOT_REACHED(); }(); + vaddr_t last_loaded_address = 0; BAN::Vector> memory_regions; for (const auto& program_header : program_headers) { @@ -241,10 +240,57 @@ namespace Kernel::ELF TRY(memory_regions.emplace_back(BAN::move(region))); } + + last_loaded_address = BAN::Math::max(last_loaded_address, pheader_base + program_header.p_memsz); } LoadResult result; - result.has_interpreter = !interpreter.empty(); + + for (const auto& program_header : program_headers) + { + if (program_header.p_type != PT_TLS) + continue; + + if (!BAN::Math::is_power_of_two(program_header.p_align)) + return BAN::Error::from_errno(EINVAL); + + size_t region_size = program_header.p_memsz; + if (auto rem = region_size % program_header.p_align) + region_size += program_header.p_align - rem; + + size_t offset = 0; + if (auto rem = region_size % alignof(uthread)) + offset = alignof(uthread) - rem; + + auto region = TRY(MemoryBackedRegion::create( + page_table, + offset + region_size, + { .start = last_loaded_address, .end = USERSPACE_END }, + MemoryRegion::Type::PRIVATE, + PageTable::Flags::UserSupervisor | PageTable::Flags::Present + )); + + for (vaddr_t vaddr = region->vaddr(); vaddr < region->vaddr() + offset + region->size(); vaddr += PAGE_SIZE) + TRY(region->allocate_page_containing(vaddr, false)); + + if (program_header.p_filesz > 0) + { + BAN::Vector file_data_buffer; + TRY(file_data_buffer.resize(program_header.p_filesz)); + if (TRY(inode->read(program_header.p_offset, file_data_buffer.span())) != file_data_buffer.size()) + return BAN::Error::from_errno(EFAULT); + TRY(region->copy_data_to_region(offset, file_data_buffer.data(), file_data_buffer.size())); + } + + result.master_tls = LoadResult::TLS { + .addr = region->vaddr(), + .size = region->size(), + }; + + TRY(memory_regions.emplace_back(BAN::move(region))); + } + + result.open_execfd = !interpreter.empty(); result.entry_point = load_base_vaddr + file_header.e_entry; result.regions = BAN::move(memory_regions); return BAN::move(result); diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 64a8a1c6..638f84da 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -142,7 +143,7 @@ namespace Kernel BAN::Vector auxiliary_vector; TRY(auxiliary_vector.reserve(1 + executable.open_execfd)); - if (executable.has_interpreter) + if (executable.open_execfd) { const int execfd = TRY(process->m_open_file_descriptors.open(BAN::move(executable_file), O_RDONLY)); TRY(auxiliary_vector.push_back({ @@ -156,6 +157,14 @@ namespace Kernel .a_un = { .a_val = 0 }, })); + BAN::Optional tls_addr; + if (executable.master_tls.has_value()) + { + auto tls_result = TRY(process->initialize_thread_local_storage(process->page_table(), *executable.master_tls)); + TRY(process->m_mapped_regions.emplace_back(BAN::move(tls_result.region))); + tls_addr = tls_result.addr; + } + auto* thread = MUST(Thread::create_userspace(process, process->page_table())); MUST(thread->initialize_userspace( executable.entry_point, @@ -163,6 +172,8 @@ namespace Kernel process->m_environ.span(), auxiliary_vector.span() )); + if (tls_addr.has_value()) + thread->set_tls(*tls_addr); process->add_thread(thread); process->register_to_scheduler(); @@ -295,6 +306,63 @@ namespace Kernel ASSERT_NOT_REACHED(); } + BAN::ErrorOr Process::initialize_thread_local_storage(PageTable& page_table, ELF::LoadResult::TLS master_tls) + { + const auto [master_addr, master_size] = master_tls; + ASSERT(master_size % alignof(uthread) == 0); + + const size_t tls_size = master_size + PAGE_SIZE; + + auto region = TRY(MemoryBackedRegion::create( + page_table, + tls_size, + { .start = master_addr, .end = USERSPACE_END }, + MemoryRegion::Type::PRIVATE, + PageTable::Flags::UserSupervisor | PageTable::Flags::ReadWrite | PageTable::Flags::Present + )); + + BAN::Vector temp_buffer; + TRY(temp_buffer.resize(BAN::Math::min(master_size, PAGE_SIZE))); + + size_t bytes_copied = 0; + while (bytes_copied < master_size) + { + const size_t to_copy = BAN::Math::min(master_size - bytes_copied, temp_buffer.size()); + + const vaddr_t vaddr = master_addr + bytes_copied; + const paddr_t paddr = page_table.physical_address_of(vaddr & PAGE_ADDR_MASK); + PageTable::with_fast_page(paddr, [&] { + memcpy(temp_buffer.data(), PageTable::fast_page_as_ptr(vaddr % PAGE_SIZE), to_copy); + }); + + TRY(region->copy_data_to_region(bytes_copied, temp_buffer.data(), to_copy)); + bytes_copied += to_copy; + } + + const uthread uthread { + .self = reinterpret_cast(region->vaddr() + master_size), + .master_tls_addr = reinterpret_cast(master_addr), + .master_tls_size = master_size, + }; + const uintptr_t dtv[2] { 1, region->vaddr() }; + + TRY(region->copy_data_to_region( + master_size, + reinterpret_cast(&uthread), + sizeof(uthread) + )); + TRY(region->copy_data_to_region( + master_size + sizeof(uthread), + reinterpret_cast(&dtv), + sizeof(dtv) + )); + + TLSResult result; + result.addr = region->vaddr() + master_size;; + result.region = BAN::move(region); + return result; + } + size_t Process::proc_meminfo(off_t offset, BAN::ByteSpan buffer) const { ASSERT(offset >= 0); @@ -572,7 +640,7 @@ namespace Kernel MUST(m_open_file_descriptors.close(auxiliary_vector.front().a_un.a_val)); }); - if (executable.has_interpreter) + if (executable.open_execfd) { const int execfd = TRY(m_open_file_descriptors.open(BAN::move(executable_file), O_RDONLY)); TRY(auxiliary_vector.push_back({ @@ -594,6 +662,13 @@ namespace Kernel auxiliary_vector.span() )); + if (executable.master_tls.has_value()) + { + auto tls_result = TRY(initialize_thread_local_storage(*new_page_table, *executable.master_tls)); + TRY(new_mapped_regions.emplace_back(BAN::move(tls_result.region))); + new_thread->set_tls(tls_result.addr); + } + ASSERT(Processor::get_interrupt_state() == InterruptState::Enabled); Processor::set_interrupt_state(InterruptState::Disabled); diff --git a/userspace/libraries/LibC/CMakeLists.txt b/userspace/libraries/LibC/CMakeLists.txt index 2ad05ad6..549cf39b 100644 --- a/userspace/libraries/LibC/CMakeLists.txt +++ b/userspace/libraries/LibC/CMakeLists.txt @@ -58,6 +58,10 @@ target_compile_definitions(objlibc PRIVATE __arch=${BANAN_ARCH}) target_compile_options(objlibc PRIVATE -O2 -g -Wstack-usage=512 -fno-exceptions -fpic -nolibc) target_compile_options(objlibc PUBLIC -Wall -Wextra -Werror -Wno-error=stack-usage=) +if("${BANAN_ARCH}" STREQUAL "i686") + target_compile_definitions(objlibc PRIVATE __disable_thread_local_storage) +endif() + function(add_crtx crtx) add_custom_target(${crtx} COMMAND ${CMAKE_CXX_COMPILER} -c -o ${CMAKE_INSTALL_LIBDIR}/${crtx}.o ${CMAKE_CURRENT_SOURCE_DIR}/arch/${BANAN_ARCH}/${crtx}.S @@ -71,6 +75,7 @@ add_crtx(crtn) banan_include_headers(objlibc ban) banan_include_headers(objlibc kernel) +banan_include_headers(objlibc libelf) banan_install_headers(objlibc) diff --git a/userspace/libraries/LibC/errno.cpp b/userspace/libraries/LibC/errno.cpp index e60ef5c3..b5682b6e 100644 --- a/userspace/libraries/LibC/errno.cpp +++ b/userspace/libraries/LibC/errno.cpp @@ -1,6 +1,10 @@ #include +#if __disable_thread_local_storage static int s_errno = 0; +#else +static thread_local int s_errno = 0; +#endif int* __errno_location() { diff --git a/userspace/libraries/LibC/include/pthread.h b/userspace/libraries/LibC/include/pthread.h index 72f7784b..273ffbcd 100644 --- a/userspace/libraries/LibC/include/pthread.h +++ b/userspace/libraries/LibC/include/pthread.h @@ -8,6 +8,7 @@ __BEGIN_DECLS #include +#include #include #define __need_size_t @@ -27,6 +28,14 @@ __BEGIN_DECLS #define __need_pthread_t #include +struct uthread +{ + struct uthread* self; + void* master_tls_addr; + size_t master_tls_size; + uintptr_t dtv[]; +}; + #define PTHREAD_BARRIER_SERIAL_THREAD 1 #define PTHREAD_CANCEL_ASYNCHRONOUS 2 #define PTHREAD_CANCEL_ENABLE 3 diff --git a/userspace/libraries/LibC/pthread.cpp b/userspace/libraries/LibC/pthread.cpp index 6318e20d..36442a04 100644 --- a/userspace/libraries/LibC/pthread.cpp +++ b/userspace/libraries/LibC/pthread.cpp @@ -8,11 +8,13 @@ #include #include #include +#include #include #include struct pthread_trampoline_info_t { + struct uthread* uthread; void* (*start_routine)(void*); void* arg; }; @@ -41,11 +43,36 @@ asm( extern "C" void _pthread_trampoline_cpp(void* arg) { auto info = *reinterpret_cast(arg); + syscall(SYS_SET_TLS, info.uthread); free(arg); pthread_exit(info.start_routine(info.arg)); ASSERT_NOT_REACHED(); } +static uthread* get_uthread() +{ + uthread* result; +#if ARCH(x86_64) + asm volatile("movq %%fs:0, %0" : "=r"(result)); +#elif ARCH(i686) + asm volatile("movl %%gs:0, %0" : "=r"(result)); +#endif + return result; +} + +static void free_uthread(uthread* uthread) +{ + if (uthread->dtv[0] == 0) + return free(uthread); + + uint8_t* tls_addr = reinterpret_cast(uthread) - uthread->master_tls_size; + const size_t tls_size = uthread->master_tls_size + + sizeof(struct uthread) + + (uthread->dtv[0] + 1) * sizeof(uintptr_t); + munmap(tls_addr, tls_size); +} + +#if not __disable_thread_local_storage struct pthread_cleanup_t { void (*routine)(void*); @@ -79,6 +106,7 @@ void pthread_cleanup_push(void (*routine)(void*), void* arg) s_cleanup_stack = cleanup; } +#endif int pthread_attr_init(pthread_attr_t* attr) { @@ -93,29 +121,76 @@ int pthread_create(pthread_t* __restrict thread_id, const pthread_attr_t* __rest return errno; *info = { + .uthread = nullptr, .start_routine = start_routine, .arg = arg, }; - const auto ret = syscall(SYS_PTHREAD_CREATE, attr, pthread_trampoline, info); - if (ret == -1) + long syscall_ret = 0; + + if (uthread* self = get_uthread(); self->master_tls_addr == nullptr) + { + uthread* uthread = static_cast(malloc(sizeof(struct uthread) + sizeof(uintptr_t))); + if (uthread == nullptr) + goto pthread_create_error; + uthread->self = uthread; + uthread->master_tls_addr = nullptr; + uthread->master_tls_size = 0; + uthread->dtv[0] = 0; + + info->uthread = uthread; + } + else + { + const size_t module_count = self->dtv[0]; + + const size_t tls_size = self->master_tls_size + + sizeof(uthread) + + (module_count + 1) * sizeof(uintptr_t); + + uint8_t* tls_addr = static_cast(mmap(nullptr, tls_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); + if (tls_addr == MAP_FAILED) + goto pthread_create_error; + memcpy(tls_addr, self->master_tls_addr, self->master_tls_size); + + uthread* uthread = reinterpret_cast(tls_addr + self->master_tls_size); + uthread->self = uthread; + uthread->master_tls_addr = self->master_tls_addr; + uthread->master_tls_size = self->master_tls_size; + + const uintptr_t self_addr = reinterpret_cast(self); + const uintptr_t uthread_addr = reinterpret_cast(uthread); + + uthread->dtv[0] = module_count; + for (size_t i = 1; i <= module_count; i++) + uthread->dtv[i] = self->dtv[i] - self_addr + uthread_addr; + + info->uthread = uthread; + } + + syscall_ret = syscall(SYS_PTHREAD_CREATE, attr, _pthread_trampoline, info); + if (syscall_ret == -1) goto pthread_create_error; - if (thread_id) - *thread_id = ret; + *thread_id = syscall_ret; return 0; pthread_create_error: const int return_code = errno; + if (info->uthread) + free_uthread(info->uthread); free(info); return return_code; } void pthread_exit(void* value_ptr) { +#if not __disable_thread_local_storage while (s_cleanup_stack) pthread_cleanup_pop(1); +#endif + free_uthread(get_uthread()); syscall(SYS_PTHREAD_EXIT, value_ptr); ASSERT_NOT_REACHED(); } @@ -127,7 +202,14 @@ int pthread_join(pthread_t thread, void** value_ptr) pthread_t pthread_self(void) { +#if __disable_thread_local_storage return syscall(SYS_PTHREAD_SELF); +#else + static thread_local pthread_t s_pthread_self { -1 }; + if (s_pthread_self == -1) [[unlikely]] + s_pthread_self = syscall(SYS_PTHREAD_SELF); + return s_pthread_self; +#endif } static inline BAN::Atomic& pthread_spin_get_atomic(pthread_spinlock_t* lock) @@ -188,3 +270,21 @@ int pthread_spin_unlock(pthread_spinlock_t* lock) atomic.store(0, BAN::MemoryOrder::memory_order_release); return 0; } + +struct tls_index +{ + unsigned long int ti_module; + unsigned long int ti_offset; +}; + +extern "C" void* __tls_get_addr(tls_index* ti) +{ + return reinterpret_cast(get_uthread()->dtv[ti->ti_module] + ti->ti_offset); +} + +#if ARCH(i686) +extern "C" void* __attribute__((__regparm__(1))) ___tls_get_addr(tls_index* ti) +{ + return reinterpret_cast(get_uthread()->dtv[ti->ti_module] + ti->ti_offset); +} +#endif diff --git a/userspace/libraries/LibC/unistd.cpp b/userspace/libraries/LibC/unistd.cpp index 404388df..83237371 100644 --- a/userspace/libraries/LibC/unistd.cpp +++ b/userspace/libraries/LibC/unistd.cpp @@ -1,10 +1,12 @@ #include #include + #include #include #include #include +#include #include #include #include @@ -30,6 +32,21 @@ extern "C" void _init_libc(char** environ, init_funcs_t init_funcs, init_funcs_t if (::environ == nullptr) ::environ = environ; + if (syscall(SYS_GET_TLS) == 0) + { + alignas(uthread) static uint8_t storage[sizeof(uthread) + sizeof(uintptr_t)]; + + uthread& uthread = *reinterpret_cast(storage); + uthread = { + .self = &uthread, + .master_tls_addr = nullptr, + .master_tls_size = 0, + }; + uthread.dtv[0] = 0; + + syscall(SYS_SET_TLS, &uthread); + } + // call global constructors if (init_funcs.func) init_funcs.func(); diff --git a/userspace/programs/DynamicLoader/main.cpp b/userspace/programs/DynamicLoader/main.cpp index 3296aed2..5f495316 100644 --- a/userspace/programs/DynamicLoader/main.cpp +++ b/userspace/programs/DynamicLoader/main.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -145,8 +146,13 @@ static void resolve_symbol_trampoline() struct LoadedElf { ElfNativeFileHeader file_header; + ElfNativeProgramHeader tls_header; ElfNativeDynamic* dynamics; + uint8_t* tls_addr; + size_t tls_module; + size_t tls_offset; + int fd; uintptr_t base; @@ -326,9 +332,19 @@ static void handle_tls_relocation(const LoadedElf& elf, const RelocT& reloc) if (!is_tls_relocation(reloc)) return; + if (ELF_R_SYM(reloc.r_info)) + print_error_and_exit("tls relocation with symbol index", 0); + + if (elf.tls_addr == nullptr) + print_error_and_exit("tls relocation without tls", 0); + + #if defined(__x86_64__) switch (ELF64_R_TYPE(reloc.r_info)) { + case R_X86_64_DTPMOD64: + *reinterpret_cast(elf.base + reloc.r_offset) = elf.tls_module; + break; default: print(STDERR_FILENO, "unsupported tls reloc type "); print_uint(STDERR_FILENO, ELF64_R_TYPE(reloc.r_info)); @@ -339,6 +355,9 @@ static void handle_tls_relocation(const LoadedElf& elf, const RelocT& reloc) #elif defined(__i686__) switch (ELF32_R_TYPE(reloc.r_info)) { + case R_386_TLS_DTPMOD32: + *reinterpret_cast(elf.base + reloc.r_offset) = elf.tls_module; + break; default: print(STDERR_FILENO, "unsupported tls reloc type "); print_uint(STDERR_FILENO, ELF64_R_TYPE(reloc.r_info)); @@ -890,6 +909,9 @@ static LoadedElf& load_elf(const char* path, int fd) break; } + ElfNativeProgramHeader tls_header {}; + tls_header.p_type = PT_NULL; + for (size_t i = 0; i < file_header.e_phnum; i++) { ElfNativeProgramHeader program_header; @@ -909,6 +931,9 @@ static LoadedElf& load_elf(const char* path, int fd) case PT_GNU_RELRO: print(STDDBG_FILENO, "TODO: PT_GNU_*\n"); break; + case PT_TLS: + tls_header = program_header; + break; case PT_LOAD: program_header.p_vaddr += base; load_program_header(program_header, fd, needs_writable); @@ -921,6 +946,7 @@ static LoadedElf& load_elf(const char* path, int fd) } auto& elf = s_loaded_files[s_loaded_file_count++]; + elf.tls_header = tls_header; elf.base = base; elf.fd = fd; elf.dynamics = nullptr; @@ -950,6 +976,139 @@ static LoadedElf& load_elf(const char* path, int fd) return elf; } +struct MasterTLS +{ + uint8_t* addr; + size_t size; + size_t module_count; +}; + +static MasterTLS initialize_master_tls() +{ + constexpr auto round = + [](size_t a, size_t b) -> size_t + { + return b * ((a + b - 1) / b); + }; + + size_t max_align = alignof(uthread); + size_t tls_m_offset = 0; + size_t tls_m_size = 0; + size_t module_count = 0; + for (size_t i = 0; i < s_loaded_file_count; i++) + { + const auto& tls_header = s_loaded_files[i].tls_header; + if (tls_header.p_type != PT_TLS) + continue; + if (tls_header.p_align == 0) + print_error_and_exit("TLS alignment is 0", 0); + + max_align = max(max_align, tls_header.p_align); + tls_m_offset = round(tls_m_offset + tls_header.p_memsz, tls_header.p_align); + tls_m_size = tls_header.p_memsz; + + module_count++; + } + + if (module_count == 0) + return { .addr = nullptr, .size = 0, .module_count = 0 }; + + size_t master_tls_size = tls_m_offset + tls_m_size; + if (auto rem = master_tls_size % max_align) + master_tls_size += max_align - rem; + + uint8_t* master_tls_addr; + + { + const sys_mmap_t mmap_args { + .addr = nullptr, + .len = master_tls_size, + .prot = PROT_READ | PROT_WRITE, + .flags = MAP_ANONYMOUS | MAP_PRIVATE, + .fildes = -1, + .off = 0, + }; + + const auto ret = syscall(SYS_MMAP, &mmap_args); + if (ret < 0) + print_error_and_exit("failed to allocate master TLS", ret); + master_tls_addr = reinterpret_cast(ret); + } + + for (size_t i = 0, tls_offset = 0, tls_module = 1; i < s_loaded_file_count; i++) + { + const auto& tls_header = s_loaded_files[i].tls_header; + if (tls_header.p_type != PT_TLS) + continue; + + tls_offset = round(tls_offset + tls_header.p_memsz, tls_header.p_align); + + uint8_t* tls_buffer = master_tls_addr + master_tls_size - tls_offset; + + if (tls_header.p_filesz > 0) + { + const int fd = s_loaded_files[i].fd; + if (auto ret = syscall(SYS_PREAD, fd, tls_buffer, tls_header.p_filesz, tls_header.p_offset); ret != static_cast(tls_header.p_filesz)) + print_error_and_exit("failed to read TLS data", ret); + } + + memset(tls_buffer + tls_header.p_filesz, 0, tls_header.p_memsz - tls_header.p_filesz); + + auto& elf = s_loaded_files[i]; + elf.tls_addr = tls_buffer; + elf.tls_module = tls_module++; + elf.tls_offset = master_tls_size - tls_offset; + } + + return { .addr = master_tls_addr, .size = master_tls_size, .module_count = module_count }; +} + +static void initialize_tls(MasterTLS master_tls) +{ + if (master_tls.addr == nullptr) + return; + + const size_t tls_size = master_tls.size + + sizeof(uthread) + + (master_tls.module_count + 1) * sizeof(uintptr_t); + + uint8_t* tls_addr; + + { + const sys_mmap_t mmap_args { + .addr = nullptr, + .len = tls_size, + .prot = PROT_READ | PROT_WRITE, + .flags = MAP_ANONYMOUS | MAP_PRIVATE, + .fildes = -1, + .off = 0, + }; + + const auto ret = syscall(SYS_MMAP, &mmap_args); + if (ret < 0) + print_error_and_exit("failed to allocate master TLS", ret); + tls_addr = reinterpret_cast(ret); + } + + memcpy(tls_addr, master_tls.addr, master_tls.size); + + uthread* uthread = reinterpret_cast(tls_addr + master_tls.size); + uthread->self = uthread; + uthread->master_tls_addr = master_tls.addr; + uthread->master_tls_size = master_tls.size; + + uthread->dtv[0] = master_tls.module_count; + for (size_t i = 0; i < s_loaded_file_count; i++) + { + const auto& elf = s_loaded_files[i]; + if (elf.tls_addr == nullptr) + continue; + uthread->dtv[elf.tls_module] = reinterpret_cast(tls_addr) + elf.tls_offset; + } + + syscall(SYS_SET_TLS, uthread); +} + static void initialize_environ(char** envp) { uintptr_t environ = SYM_NOT_FOUND; @@ -1040,7 +1199,9 @@ uintptr_t _entry(int argc, char* argv[], char* envp[]) auto& elf = load_elf(argv[0], execfd); fini_random(); + const auto master_tls = initialize_master_tls(); relocate_elf(elf, true); + initialize_tls(master_tls); initialize_environ(envp); call_init_funcs(elf, true);