diff --git a/kernel/include/kernel/ELF.h b/kernel/include/kernel/ELF.h index b777a939..cb6c13ec 100644 --- a/kernel/include/kernel/ELF.h +++ b/kernel/include/kernel/ELF.h @@ -8,8 +8,15 @@ namespace Kernel::ELF struct LoadResult { - bool has_interpreter; + struct TLS + { + vaddr_t addr; + size_t size; + }; + + bool open_execfd; vaddr_t entry_point; + BAN::Optional master_tls; BAN::Vector> regions; }; diff --git a/kernel/include/kernel/Process.h b/kernel/include/kernel/Process.h index 7ff5da7e..ee2b1adc 100644 --- a/kernel/include/kernel/Process.h +++ b/kernel/include/kernel/Process.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -219,6 +220,13 @@ namespace Kernel Process(const Credentials&, pid_t pid, pid_t parent, pid_t sid, pid_t pgrp); static Process* create_process(const Credentials&, pid_t parent, pid_t sid = 0, pid_t pgrp = 0); + struct TLSResult + { + BAN::UniqPtr region; + vaddr_t addr; + }; + static BAN::ErrorOr initialize_thread_local_storage(PageTable&, ELF::LoadResult::TLS master_tls); + struct FileParent { VirtualFileSystem::File parent; diff --git a/kernel/kernel/ELF.cpp b/kernel/kernel/ELF.cpp index 372ffea3..b491950d 100644 --- a/kernel/kernel/ELF.cpp +++ b/kernel/kernel/ELF.cpp @@ -8,6 +8,7 @@ #include #include +#include namespace Kernel::ELF { @@ -108,13 +109,13 @@ namespace Kernel::ELF auto file_header = TRY(read_and_validate_file_header(inode)); auto program_headers = TRY(read_program_headers(inode, file_header)); - vaddr_t executable_end = 0; + size_t exec_max_offset { 0 }; BAN::String interpreter; for (const auto& program_header : program_headers) { if (program_header.p_type == PT_LOAD) - executable_end = BAN::Math::max(executable_end, program_header.p_vaddr + program_header.p_memsz); + exec_max_offset = BAN::Math::max(exec_max_offset, program_header.p_vaddr + program_header.p_memsz); else if (program_header.p_type == PT_INTERP) { BAN::Vector interp_buffer; @@ -140,9 +141,6 @@ namespace Kernel::ELF } } - if (file_header.e_type == ET_DYN) - executable_end = 0x400000; - if (!interpreter.empty()) { auto interpreter_inode = TRY(VirtualFileSystem::get().file_from_absolute_path(credentials, interpreter, O_EXEC)).inode; @@ -164,15 +162,16 @@ namespace Kernel::ELF } const vaddr_t load_base_vaddr = - [&file_header, executable_end]() -> vaddr_t + [&file_header, exec_max_offset]() -> vaddr_t { if (file_header.e_type == ET_EXEC) return 0; if (file_header.e_type == ET_DYN) - return (executable_end + PAGE_SIZE - 1) & PAGE_ADDR_MASK; + return (exec_max_offset + PAGE_SIZE - 1) & PAGE_ADDR_MASK; ASSERT_NOT_REACHED(); }(); + vaddr_t last_loaded_address = 0; BAN::Vector> memory_regions; for (const auto& program_header : program_headers) { @@ -241,10 +240,57 @@ namespace Kernel::ELF TRY(memory_regions.emplace_back(BAN::move(region))); } + + last_loaded_address = BAN::Math::max(last_loaded_address, pheader_base + program_header.p_memsz); } LoadResult result; - result.has_interpreter = !interpreter.empty(); + + for (const auto& program_header : program_headers) + { + if (program_header.p_type != PT_TLS) + continue; + + if (!BAN::Math::is_power_of_two(program_header.p_align)) + return BAN::Error::from_errno(EINVAL); + + size_t region_size = program_header.p_memsz; + if (auto rem = region_size % program_header.p_align) + region_size += program_header.p_align - rem; + + size_t offset = 0; + if (auto rem = region_size % alignof(uthread)) + offset = alignof(uthread) - rem; + + auto region = TRY(MemoryBackedRegion::create( + page_table, + offset + region_size, + { .start = last_loaded_address, .end = USERSPACE_END }, + MemoryRegion::Type::PRIVATE, + PageTable::Flags::UserSupervisor | PageTable::Flags::Present + )); + + for (vaddr_t vaddr = region->vaddr(); vaddr < region->vaddr() + offset + region->size(); vaddr += PAGE_SIZE) + TRY(region->allocate_page_containing(vaddr, false)); + + if (program_header.p_filesz > 0) + { + BAN::Vector file_data_buffer; + TRY(file_data_buffer.resize(program_header.p_filesz)); + if (TRY(inode->read(program_header.p_offset, file_data_buffer.span())) != file_data_buffer.size()) + return BAN::Error::from_errno(EFAULT); + TRY(region->copy_data_to_region(offset, file_data_buffer.data(), file_data_buffer.size())); + } + + result.master_tls = LoadResult::TLS { + .addr = region->vaddr(), + .size = region->size(), + }; + + TRY(memory_regions.emplace_back(BAN::move(region))); + } + + result.open_execfd = !interpreter.empty(); result.entry_point = load_base_vaddr + file_header.e_entry; result.regions = BAN::move(memory_regions); return BAN::move(result); diff --git a/kernel/kernel/Process.cpp b/kernel/kernel/Process.cpp index 64a8a1c6..638f84da 100644 --- a/kernel/kernel/Process.cpp +++ b/kernel/kernel/Process.cpp @@ -22,6 +22,7 @@ #include #include +#include #include #include #include @@ -142,7 +143,7 @@ namespace Kernel BAN::Vector auxiliary_vector; TRY(auxiliary_vector.reserve(1 + executable.open_execfd)); - if (executable.has_interpreter) + if (executable.open_execfd) { const int execfd = TRY(process->m_open_file_descriptors.open(BAN::move(executable_file), O_RDONLY)); TRY(auxiliary_vector.push_back({ @@ -156,6 +157,14 @@ namespace Kernel .a_un = { .a_val = 0 }, })); + BAN::Optional tls_addr; + if (executable.master_tls.has_value()) + { + auto tls_result = TRY(process->initialize_thread_local_storage(process->page_table(), *executable.master_tls)); + TRY(process->m_mapped_regions.emplace_back(BAN::move(tls_result.region))); + tls_addr = tls_result.addr; + } + auto* thread = MUST(Thread::create_userspace(process, process->page_table())); MUST(thread->initialize_userspace( executable.entry_point, @@ -163,6 +172,8 @@ namespace Kernel process->m_environ.span(), auxiliary_vector.span() )); + if (tls_addr.has_value()) + thread->set_tls(*tls_addr); process->add_thread(thread); process->register_to_scheduler(); @@ -295,6 +306,63 @@ namespace Kernel ASSERT_NOT_REACHED(); } + BAN::ErrorOr Process::initialize_thread_local_storage(PageTable& page_table, ELF::LoadResult::TLS master_tls) + { + const auto [master_addr, master_size] = master_tls; + ASSERT(master_size % alignof(uthread) == 0); + + const size_t tls_size = master_size + PAGE_SIZE; + + auto region = TRY(MemoryBackedRegion::create( + page_table, + tls_size, + { .start = master_addr, .end = USERSPACE_END }, + MemoryRegion::Type::PRIVATE, + PageTable::Flags::UserSupervisor | PageTable::Flags::ReadWrite | PageTable::Flags::Present + )); + + BAN::Vector temp_buffer; + TRY(temp_buffer.resize(BAN::Math::min(master_size, PAGE_SIZE))); + + size_t bytes_copied = 0; + while (bytes_copied < master_size) + { + const size_t to_copy = BAN::Math::min(master_size - bytes_copied, temp_buffer.size()); + + const vaddr_t vaddr = master_addr + bytes_copied; + const paddr_t paddr = page_table.physical_address_of(vaddr & PAGE_ADDR_MASK); + PageTable::with_fast_page(paddr, [&] { + memcpy(temp_buffer.data(), PageTable::fast_page_as_ptr(vaddr % PAGE_SIZE), to_copy); + }); + + TRY(region->copy_data_to_region(bytes_copied, temp_buffer.data(), to_copy)); + bytes_copied += to_copy; + } + + const uthread uthread { + .self = reinterpret_cast(region->vaddr() + master_size), + .master_tls_addr = reinterpret_cast(master_addr), + .master_tls_size = master_size, + }; + const uintptr_t dtv[2] { 1, region->vaddr() }; + + TRY(region->copy_data_to_region( + master_size, + reinterpret_cast(&uthread), + sizeof(uthread) + )); + TRY(region->copy_data_to_region( + master_size + sizeof(uthread), + reinterpret_cast(&dtv), + sizeof(dtv) + )); + + TLSResult result; + result.addr = region->vaddr() + master_size;; + result.region = BAN::move(region); + return result; + } + size_t Process::proc_meminfo(off_t offset, BAN::ByteSpan buffer) const { ASSERT(offset >= 0); @@ -572,7 +640,7 @@ namespace Kernel MUST(m_open_file_descriptors.close(auxiliary_vector.front().a_un.a_val)); }); - if (executable.has_interpreter) + if (executable.open_execfd) { const int execfd = TRY(m_open_file_descriptors.open(BAN::move(executable_file), O_RDONLY)); TRY(auxiliary_vector.push_back({ @@ -594,6 +662,13 @@ namespace Kernel auxiliary_vector.span() )); + if (executable.master_tls.has_value()) + { + auto tls_result = TRY(initialize_thread_local_storage(*new_page_table, *executable.master_tls)); + TRY(new_mapped_regions.emplace_back(BAN::move(tls_result.region))); + new_thread->set_tls(tls_result.addr); + } + ASSERT(Processor::get_interrupt_state() == InterruptState::Enabled); Processor::set_interrupt_state(InterruptState::Disabled); diff --git a/userspace/libraries/LibC/CMakeLists.txt b/userspace/libraries/LibC/CMakeLists.txt index 2ad05ad6..549cf39b 100644 --- a/userspace/libraries/LibC/CMakeLists.txt +++ b/userspace/libraries/LibC/CMakeLists.txt @@ -58,6 +58,10 @@ target_compile_definitions(objlibc PRIVATE __arch=${BANAN_ARCH}) target_compile_options(objlibc PRIVATE -O2 -g -Wstack-usage=512 -fno-exceptions -fpic -nolibc) target_compile_options(objlibc PUBLIC -Wall -Wextra -Werror -Wno-error=stack-usage=) +if("${BANAN_ARCH}" STREQUAL "i686") + target_compile_definitions(objlibc PRIVATE __disable_thread_local_storage) +endif() + function(add_crtx crtx) add_custom_target(${crtx} COMMAND ${CMAKE_CXX_COMPILER} -c -o ${CMAKE_INSTALL_LIBDIR}/${crtx}.o ${CMAKE_CURRENT_SOURCE_DIR}/arch/${BANAN_ARCH}/${crtx}.S @@ -71,6 +75,7 @@ add_crtx(crtn) banan_include_headers(objlibc ban) banan_include_headers(objlibc kernel) +banan_include_headers(objlibc libelf) banan_install_headers(objlibc) diff --git a/userspace/libraries/LibC/errno.cpp b/userspace/libraries/LibC/errno.cpp index e60ef5c3..b5682b6e 100644 --- a/userspace/libraries/LibC/errno.cpp +++ b/userspace/libraries/LibC/errno.cpp @@ -1,6 +1,10 @@ #include +#if __disable_thread_local_storage static int s_errno = 0; +#else +static thread_local int s_errno = 0; +#endif int* __errno_location() { diff --git a/userspace/libraries/LibC/include/pthread.h b/userspace/libraries/LibC/include/pthread.h index 72f7784b..273ffbcd 100644 --- a/userspace/libraries/LibC/include/pthread.h +++ b/userspace/libraries/LibC/include/pthread.h @@ -8,6 +8,7 @@ __BEGIN_DECLS #include +#include #include #define __need_size_t @@ -27,6 +28,14 @@ __BEGIN_DECLS #define __need_pthread_t #include +struct uthread +{ + struct uthread* self; + void* master_tls_addr; + size_t master_tls_size; + uintptr_t dtv[]; +}; + #define PTHREAD_BARRIER_SERIAL_THREAD 1 #define PTHREAD_CANCEL_ASYNCHRONOUS 2 #define PTHREAD_CANCEL_ENABLE 3 diff --git a/userspace/libraries/LibC/pthread.cpp b/userspace/libraries/LibC/pthread.cpp index 6318e20d..36442a04 100644 --- a/userspace/libraries/LibC/pthread.cpp +++ b/userspace/libraries/LibC/pthread.cpp @@ -8,11 +8,13 @@ #include #include #include +#include #include #include struct pthread_trampoline_info_t { + struct uthread* uthread; void* (*start_routine)(void*); void* arg; }; @@ -41,11 +43,36 @@ asm( extern "C" void _pthread_trampoline_cpp(void* arg) { auto info = *reinterpret_cast(arg); + syscall(SYS_SET_TLS, info.uthread); free(arg); pthread_exit(info.start_routine(info.arg)); ASSERT_NOT_REACHED(); } +static uthread* get_uthread() +{ + uthread* result; +#if ARCH(x86_64) + asm volatile("movq %%fs:0, %0" : "=r"(result)); +#elif ARCH(i686) + asm volatile("movl %%gs:0, %0" : "=r"(result)); +#endif + return result; +} + +static void free_uthread(uthread* uthread) +{ + if (uthread->dtv[0] == 0) + return free(uthread); + + uint8_t* tls_addr = reinterpret_cast(uthread) - uthread->master_tls_size; + const size_t tls_size = uthread->master_tls_size + + sizeof(struct uthread) + + (uthread->dtv[0] + 1) * sizeof(uintptr_t); + munmap(tls_addr, tls_size); +} + +#if not __disable_thread_local_storage struct pthread_cleanup_t { void (*routine)(void*); @@ -79,6 +106,7 @@ void pthread_cleanup_push(void (*routine)(void*), void* arg) s_cleanup_stack = cleanup; } +#endif int pthread_attr_init(pthread_attr_t* attr) { @@ -93,29 +121,76 @@ int pthread_create(pthread_t* __restrict thread_id, const pthread_attr_t* __rest return errno; *info = { + .uthread = nullptr, .start_routine = start_routine, .arg = arg, }; - const auto ret = syscall(SYS_PTHREAD_CREATE, attr, pthread_trampoline, info); - if (ret == -1) + long syscall_ret = 0; + + if (uthread* self = get_uthread(); self->master_tls_addr == nullptr) + { + uthread* uthread = static_cast(malloc(sizeof(struct uthread) + sizeof(uintptr_t))); + if (uthread == nullptr) + goto pthread_create_error; + uthread->self = uthread; + uthread->master_tls_addr = nullptr; + uthread->master_tls_size = 0; + uthread->dtv[0] = 0; + + info->uthread = uthread; + } + else + { + const size_t module_count = self->dtv[0]; + + const size_t tls_size = self->master_tls_size + + sizeof(uthread) + + (module_count + 1) * sizeof(uintptr_t); + + uint8_t* tls_addr = static_cast(mmap(nullptr, tls_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0)); + if (tls_addr == MAP_FAILED) + goto pthread_create_error; + memcpy(tls_addr, self->master_tls_addr, self->master_tls_size); + + uthread* uthread = reinterpret_cast(tls_addr + self->master_tls_size); + uthread->self = uthread; + uthread->master_tls_addr = self->master_tls_addr; + uthread->master_tls_size = self->master_tls_size; + + const uintptr_t self_addr = reinterpret_cast(self); + const uintptr_t uthread_addr = reinterpret_cast(uthread); + + uthread->dtv[0] = module_count; + for (size_t i = 1; i <= module_count; i++) + uthread->dtv[i] = self->dtv[i] - self_addr + uthread_addr; + + info->uthread = uthread; + } + + syscall_ret = syscall(SYS_PTHREAD_CREATE, attr, _pthread_trampoline, info); + if (syscall_ret == -1) goto pthread_create_error; - if (thread_id) - *thread_id = ret; + *thread_id = syscall_ret; return 0; pthread_create_error: const int return_code = errno; + if (info->uthread) + free_uthread(info->uthread); free(info); return return_code; } void pthread_exit(void* value_ptr) { +#if not __disable_thread_local_storage while (s_cleanup_stack) pthread_cleanup_pop(1); +#endif + free_uthread(get_uthread()); syscall(SYS_PTHREAD_EXIT, value_ptr); ASSERT_NOT_REACHED(); } @@ -127,7 +202,14 @@ int pthread_join(pthread_t thread, void** value_ptr) pthread_t pthread_self(void) { +#if __disable_thread_local_storage return syscall(SYS_PTHREAD_SELF); +#else + static thread_local pthread_t s_pthread_self { -1 }; + if (s_pthread_self == -1) [[unlikely]] + s_pthread_self = syscall(SYS_PTHREAD_SELF); + return s_pthread_self; +#endif } static inline BAN::Atomic& pthread_spin_get_atomic(pthread_spinlock_t* lock) @@ -188,3 +270,21 @@ int pthread_spin_unlock(pthread_spinlock_t* lock) atomic.store(0, BAN::MemoryOrder::memory_order_release); return 0; } + +struct tls_index +{ + unsigned long int ti_module; + unsigned long int ti_offset; +}; + +extern "C" void* __tls_get_addr(tls_index* ti) +{ + return reinterpret_cast(get_uthread()->dtv[ti->ti_module] + ti->ti_offset); +} + +#if ARCH(i686) +extern "C" void* __attribute__((__regparm__(1))) ___tls_get_addr(tls_index* ti) +{ + return reinterpret_cast(get_uthread()->dtv[ti->ti_module] + ti->ti_offset); +} +#endif diff --git a/userspace/libraries/LibC/unistd.cpp b/userspace/libraries/LibC/unistd.cpp index 404388df..83237371 100644 --- a/userspace/libraries/LibC/unistd.cpp +++ b/userspace/libraries/LibC/unistd.cpp @@ -1,10 +1,12 @@ #include #include + #include #include #include #include +#include #include #include #include @@ -30,6 +32,21 @@ extern "C" void _init_libc(char** environ, init_funcs_t init_funcs, init_funcs_t if (::environ == nullptr) ::environ = environ; + if (syscall(SYS_GET_TLS) == 0) + { + alignas(uthread) static uint8_t storage[sizeof(uthread) + sizeof(uintptr_t)]; + + uthread& uthread = *reinterpret_cast(storage); + uthread = { + .self = &uthread, + .master_tls_addr = nullptr, + .master_tls_size = 0, + }; + uthread.dtv[0] = 0; + + syscall(SYS_SET_TLS, &uthread); + } + // call global constructors if (init_funcs.func) init_funcs.func(); diff --git a/userspace/programs/DynamicLoader/main.cpp b/userspace/programs/DynamicLoader/main.cpp index 3296aed2..5f495316 100644 --- a/userspace/programs/DynamicLoader/main.cpp +++ b/userspace/programs/DynamicLoader/main.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -145,8 +146,13 @@ static void resolve_symbol_trampoline() struct LoadedElf { ElfNativeFileHeader file_header; + ElfNativeProgramHeader tls_header; ElfNativeDynamic* dynamics; + uint8_t* tls_addr; + size_t tls_module; + size_t tls_offset; + int fd; uintptr_t base; @@ -326,9 +332,19 @@ static void handle_tls_relocation(const LoadedElf& elf, const RelocT& reloc) if (!is_tls_relocation(reloc)) return; + if (ELF_R_SYM(reloc.r_info)) + print_error_and_exit("tls relocation with symbol index", 0); + + if (elf.tls_addr == nullptr) + print_error_and_exit("tls relocation without tls", 0); + + #if defined(__x86_64__) switch (ELF64_R_TYPE(reloc.r_info)) { + case R_X86_64_DTPMOD64: + *reinterpret_cast(elf.base + reloc.r_offset) = elf.tls_module; + break; default: print(STDERR_FILENO, "unsupported tls reloc type "); print_uint(STDERR_FILENO, ELF64_R_TYPE(reloc.r_info)); @@ -339,6 +355,9 @@ static void handle_tls_relocation(const LoadedElf& elf, const RelocT& reloc) #elif defined(__i686__) switch (ELF32_R_TYPE(reloc.r_info)) { + case R_386_TLS_DTPMOD32: + *reinterpret_cast(elf.base + reloc.r_offset) = elf.tls_module; + break; default: print(STDERR_FILENO, "unsupported tls reloc type "); print_uint(STDERR_FILENO, ELF64_R_TYPE(reloc.r_info)); @@ -890,6 +909,9 @@ static LoadedElf& load_elf(const char* path, int fd) break; } + ElfNativeProgramHeader tls_header {}; + tls_header.p_type = PT_NULL; + for (size_t i = 0; i < file_header.e_phnum; i++) { ElfNativeProgramHeader program_header; @@ -909,6 +931,9 @@ static LoadedElf& load_elf(const char* path, int fd) case PT_GNU_RELRO: print(STDDBG_FILENO, "TODO: PT_GNU_*\n"); break; + case PT_TLS: + tls_header = program_header; + break; case PT_LOAD: program_header.p_vaddr += base; load_program_header(program_header, fd, needs_writable); @@ -921,6 +946,7 @@ static LoadedElf& load_elf(const char* path, int fd) } auto& elf = s_loaded_files[s_loaded_file_count++]; + elf.tls_header = tls_header; elf.base = base; elf.fd = fd; elf.dynamics = nullptr; @@ -950,6 +976,139 @@ static LoadedElf& load_elf(const char* path, int fd) return elf; } +struct MasterTLS +{ + uint8_t* addr; + size_t size; + size_t module_count; +}; + +static MasterTLS initialize_master_tls() +{ + constexpr auto round = + [](size_t a, size_t b) -> size_t + { + return b * ((a + b - 1) / b); + }; + + size_t max_align = alignof(uthread); + size_t tls_m_offset = 0; + size_t tls_m_size = 0; + size_t module_count = 0; + for (size_t i = 0; i < s_loaded_file_count; i++) + { + const auto& tls_header = s_loaded_files[i].tls_header; + if (tls_header.p_type != PT_TLS) + continue; + if (tls_header.p_align == 0) + print_error_and_exit("TLS alignment is 0", 0); + + max_align = max(max_align, tls_header.p_align); + tls_m_offset = round(tls_m_offset + tls_header.p_memsz, tls_header.p_align); + tls_m_size = tls_header.p_memsz; + + module_count++; + } + + if (module_count == 0) + return { .addr = nullptr, .size = 0, .module_count = 0 }; + + size_t master_tls_size = tls_m_offset + tls_m_size; + if (auto rem = master_tls_size % max_align) + master_tls_size += max_align - rem; + + uint8_t* master_tls_addr; + + { + const sys_mmap_t mmap_args { + .addr = nullptr, + .len = master_tls_size, + .prot = PROT_READ | PROT_WRITE, + .flags = MAP_ANONYMOUS | MAP_PRIVATE, + .fildes = -1, + .off = 0, + }; + + const auto ret = syscall(SYS_MMAP, &mmap_args); + if (ret < 0) + print_error_and_exit("failed to allocate master TLS", ret); + master_tls_addr = reinterpret_cast(ret); + } + + for (size_t i = 0, tls_offset = 0, tls_module = 1; i < s_loaded_file_count; i++) + { + const auto& tls_header = s_loaded_files[i].tls_header; + if (tls_header.p_type != PT_TLS) + continue; + + tls_offset = round(tls_offset + tls_header.p_memsz, tls_header.p_align); + + uint8_t* tls_buffer = master_tls_addr + master_tls_size - tls_offset; + + if (tls_header.p_filesz > 0) + { + const int fd = s_loaded_files[i].fd; + if (auto ret = syscall(SYS_PREAD, fd, tls_buffer, tls_header.p_filesz, tls_header.p_offset); ret != static_cast(tls_header.p_filesz)) + print_error_and_exit("failed to read TLS data", ret); + } + + memset(tls_buffer + tls_header.p_filesz, 0, tls_header.p_memsz - tls_header.p_filesz); + + auto& elf = s_loaded_files[i]; + elf.tls_addr = tls_buffer; + elf.tls_module = tls_module++; + elf.tls_offset = master_tls_size - tls_offset; + } + + return { .addr = master_tls_addr, .size = master_tls_size, .module_count = module_count }; +} + +static void initialize_tls(MasterTLS master_tls) +{ + if (master_tls.addr == nullptr) + return; + + const size_t tls_size = master_tls.size + + sizeof(uthread) + + (master_tls.module_count + 1) * sizeof(uintptr_t); + + uint8_t* tls_addr; + + { + const sys_mmap_t mmap_args { + .addr = nullptr, + .len = tls_size, + .prot = PROT_READ | PROT_WRITE, + .flags = MAP_ANONYMOUS | MAP_PRIVATE, + .fildes = -1, + .off = 0, + }; + + const auto ret = syscall(SYS_MMAP, &mmap_args); + if (ret < 0) + print_error_and_exit("failed to allocate master TLS", ret); + tls_addr = reinterpret_cast(ret); + } + + memcpy(tls_addr, master_tls.addr, master_tls.size); + + uthread* uthread = reinterpret_cast(tls_addr + master_tls.size); + uthread->self = uthread; + uthread->master_tls_addr = master_tls.addr; + uthread->master_tls_size = master_tls.size; + + uthread->dtv[0] = master_tls.module_count; + for (size_t i = 0; i < s_loaded_file_count; i++) + { + const auto& elf = s_loaded_files[i]; + if (elf.tls_addr == nullptr) + continue; + uthread->dtv[elf.tls_module] = reinterpret_cast(tls_addr) + elf.tls_offset; + } + + syscall(SYS_SET_TLS, uthread); +} + static void initialize_environ(char** envp) { uintptr_t environ = SYM_NOT_FOUND; @@ -1040,7 +1199,9 @@ uintptr_t _entry(int argc, char* argv[], char* envp[]) auto& elf = load_elf(argv[0], execfd); fini_random(); + const auto master_tls = initialize_master_tls(); relocate_elf(elf, true); + initialize_tls(master_tls); initialize_environ(envp); call_init_funcs(elf, true);