Kernel/LibC/DynamicLoader: Implement thread local storage

For some reason this does not work on 32 bit version, so it is disabled
on that platform. I'll have to look into it later to find the bug :)

kernel/include/kernel/ELF.h

@@ -8,8 +8,15 @@ namespace Kernel::ELF
 
 	struct LoadResult
 	{
-		bool has_interpreter;
+		struct TLS
+		{
+			vaddr_t addr;
+			size_t size;
+		};
+
+		bool open_execfd;
 		vaddr_t entry_point;
+		BAN::Optional<TLS> master_tls;
 		BAN::Vector<BAN::UniqPtr<MemoryRegion>> regions;
 	};
 

kernel/include/kernel/Process.h

@@ -6,6 +6,7 @@
 #include <BAN/StringView.h>
 #include <BAN/Vector.h>
 #include <kernel/Credentials.h>
+#include <kernel/ELF.h>
 #include <kernel/FS/Inode.h>
 #include <kernel/Lock/Mutex.h>
 #include <kernel/Memory/Heap.h>
@@ -219,6 +220,13 @@ namespace Kernel
 		Process(const Credentials&, pid_t pid, pid_t parent, pid_t sid, pid_t pgrp);
 		static Process* create_process(const Credentials&, pid_t parent, pid_t sid = 0, pid_t pgrp = 0);
 
+		struct TLSResult
+		{
+			BAN::UniqPtr<MemoryRegion> region;
+			vaddr_t addr;
+		};
+		static BAN::ErrorOr<TLSResult> initialize_thread_local_storage(PageTable&, ELF::LoadResult::TLS master_tls);
+
 		struct FileParent
 		{
 			VirtualFileSystem::File parent;

kernel/kernel/ELF.cpp

@@ -8,6 +8,7 @@
 
 #include <ctype.h>
 #include <fcntl.h>
+#include <pthread.h>
 
 namespace Kernel::ELF
 {
@@ -108,13 +109,13 @@ namespace Kernel::ELF
 		auto file_header = TRY(read_and_validate_file_header(inode));
 		auto program_headers = TRY(read_program_headers(inode, file_header));
 
-		vaddr_t executable_end = 0;
+		size_t exec_max_offset { 0 };
 		BAN::String interpreter;
 
 		for (const auto& program_header : program_headers)
 		{
 			if (program_header.p_type == PT_LOAD)
-				executable_end = BAN::Math::max<vaddr_t>(executable_end, program_header.p_vaddr + program_header.p_memsz);
+				exec_max_offset = BAN::Math::max<vaddr_t>(exec_max_offset, program_header.p_vaddr + program_header.p_memsz);
 			else if (program_header.p_type == PT_INTERP)
 			{
 				BAN::Vector<uint8_t> interp_buffer;
@@ -140,9 +141,6 @@ namespace Kernel::ELF
 			}
 		}
 
-		if (file_header.e_type == ET_DYN)
-			executable_end = 0x400000;
-
 		if (!interpreter.empty())
 		{
 			auto interpreter_inode = TRY(VirtualFileSystem::get().file_from_absolute_path(credentials, interpreter, O_EXEC)).inode;
@@ -164,15 +162,16 @@ namespace Kernel::ELF
 		}
 
 		const vaddr_t load_base_vaddr =
-			[&file_header, executable_end]() -> vaddr_t
+			[&file_header, exec_max_offset]() -> vaddr_t
 			{
 				if (file_header.e_type == ET_EXEC)
 					return 0;
 				if (file_header.e_type == ET_DYN)
-					return (executable_end + PAGE_SIZE - 1) & PAGE_ADDR_MASK;
+					return (exec_max_offset + PAGE_SIZE - 1) & PAGE_ADDR_MASK;
 				ASSERT_NOT_REACHED();
 			}();
 
+		vaddr_t last_loaded_address = 0;
 		BAN::Vector<BAN::UniqPtr<MemoryRegion>> memory_regions;
 		for (const auto& program_header : program_headers)
 		{
@@ -241,10 +240,57 @@ namespace Kernel::ELF
 
 				TRY(memory_regions.emplace_back(BAN::move(region)));
 			}
+
+			last_loaded_address = BAN::Math::max(last_loaded_address, pheader_base + program_header.p_memsz);
 		}
 
 		LoadResult result;
-		result.has_interpreter = !interpreter.empty();
+
+		for (const auto& program_header : program_headers)
+		{
+			if (program_header.p_type != PT_TLS)
+				continue;
+
+			if (!BAN::Math::is_power_of_two(program_header.p_align))
+				return BAN::Error::from_errno(EINVAL);
+
+			size_t region_size = program_header.p_memsz;
+			if (auto rem = region_size % program_header.p_align)
+				region_size += program_header.p_align - rem;
+
+			size_t offset = 0;
+			if (auto rem = region_size % alignof(uthread))
+				offset = alignof(uthread) - rem;
+
+			auto region = TRY(MemoryBackedRegion::create(
+				page_table,
+				offset + region_size,
+				{ .start = last_loaded_address, .end = USERSPACE_END },
+				MemoryRegion::Type::PRIVATE,
+				PageTable::Flags::UserSupervisor | PageTable::Flags::Present
+			));
+
+			for (vaddr_t vaddr = region->vaddr(); vaddr < region->vaddr() + offset + region->size(); vaddr += PAGE_SIZE)
+				TRY(region->allocate_page_containing(vaddr, false));
+
+			if (program_header.p_filesz > 0)
+			{
+				BAN::Vector<uint8_t> file_data_buffer;
+				TRY(file_data_buffer.resize(program_header.p_filesz));
+				if (TRY(inode->read(program_header.p_offset, file_data_buffer.span())) != file_data_buffer.size())
+					return BAN::Error::from_errno(EFAULT);
+				TRY(region->copy_data_to_region(offset, file_data_buffer.data(), file_data_buffer.size()));
+			}
+
+			result.master_tls = LoadResult::TLS {
+				.addr = region->vaddr(),
+				.size = region->size(),
+			};
+
+			TRY(memory_regions.emplace_back(BAN::move(region)));
+		}
+
+		result.open_execfd = !interpreter.empty();
 		result.entry_point = load_base_vaddr + file_header.e_entry;
 		result.regions = BAN::move(memory_regions);
 		return BAN::move(result);

kernel/kernel/Process.cpp

@@ -22,6 +22,7 @@
 #include <LibInput/KeyboardLayout.h>
 
 #include <fcntl.h>
+#include <pthread.h>
 #include <stdio.h>
 #include <sys/banan-os.h>
 #include <sys/sysmacros.h>
@@ -142,7 +143,7 @@ namespace Kernel
 		BAN::Vector<LibELF::AuxiliaryVector> auxiliary_vector;
 		TRY(auxiliary_vector.reserve(1 + executable.open_execfd));
 
-		if (executable.has_interpreter)
+		if (executable.open_execfd)
 		{
 			const int execfd = TRY(process->m_open_file_descriptors.open(BAN::move(executable_file), O_RDONLY));
 			TRY(auxiliary_vector.push_back({
@@ -156,6 +157,14 @@ namespace Kernel
 			.a_un = { .a_val = 0 },
 		}));
 
+		BAN::Optional<vaddr_t> tls_addr;
+		if (executable.master_tls.has_value())
+		{
+			auto tls_result = TRY(process->initialize_thread_local_storage(process->page_table(), *executable.master_tls));
+			TRY(process->m_mapped_regions.emplace_back(BAN::move(tls_result.region)));
+			tls_addr = tls_result.addr;
+		}
+
 		auto* thread = MUST(Thread::create_userspace(process, process->page_table()));
 		MUST(thread->initialize_userspace(
 			executable.entry_point,
@@ -163,6 +172,8 @@ namespace Kernel
 			process->m_environ.span(),
 			auxiliary_vector.span()
 		));
+		if (tls_addr.has_value())
+			thread->set_tls(*tls_addr);
 
 		process->add_thread(thread);
 		process->register_to_scheduler();
@@ -295,6 +306,63 @@ namespace Kernel
 		ASSERT_NOT_REACHED();
 	}
 
+	BAN::ErrorOr<Process::TLSResult> Process::initialize_thread_local_storage(PageTable& page_table, ELF::LoadResult::TLS master_tls)
+	{
+		const auto [master_addr, master_size] = master_tls;
+		ASSERT(master_size % alignof(uthread) == 0);
+
+		const size_t tls_size = master_size + PAGE_SIZE;
+
+		auto region = TRY(MemoryBackedRegion::create(
+			page_table,
+			tls_size,
+			{ .start = master_addr, .end = USERSPACE_END },
+			MemoryRegion::Type::PRIVATE,
+			PageTable::Flags::UserSupervisor | PageTable::Flags::ReadWrite | PageTable::Flags::Present
+		));
+
+		BAN::Vector<uint8_t> temp_buffer;
+		TRY(temp_buffer.resize(BAN::Math::min<size_t>(master_size, PAGE_SIZE)));
+
+		size_t bytes_copied = 0;
+		while (bytes_copied < master_size)
+		{
+			const size_t to_copy = BAN::Math::min(master_size - bytes_copied, temp_buffer.size());
+
+			const vaddr_t vaddr = master_addr + bytes_copied;
+			const paddr_t paddr = page_table.physical_address_of(vaddr & PAGE_ADDR_MASK);
+			PageTable::with_fast_page(paddr, [&] {
+				memcpy(temp_buffer.data(), PageTable::fast_page_as_ptr(vaddr % PAGE_SIZE), to_copy);
+			});
+
+			TRY(region->copy_data_to_region(bytes_copied, temp_buffer.data(), to_copy));
+			bytes_copied += to_copy;
+		}
+
+		const uthread uthread {
+			.self = reinterpret_cast<struct uthread*>(region->vaddr() + master_size),
+			.master_tls_addr = reinterpret_cast<void*>(master_addr),
+			.master_tls_size = master_size,
+		};
+		const uintptr_t dtv[2] { 1, region->vaddr() };
+
+		TRY(region->copy_data_to_region(
+			master_size,
+			reinterpret_cast<const uint8_t*>(&uthread),
+			sizeof(uthread)
+		));
+		TRY(region->copy_data_to_region(
+			master_size + sizeof(uthread),
+			reinterpret_cast<const uint8_t*>(&dtv),
+			sizeof(dtv)
+		));
+
+		TLSResult result;
+		result.addr = region->vaddr() + master_size;;
+		result.region = BAN::move(region);
+		return result;
+	}
+
 	size_t Process::proc_meminfo(off_t offset, BAN::ByteSpan buffer) const
 	{
 		ASSERT(offset >= 0);
@@ -572,7 +640,7 @@ namespace Kernel
 				MUST(m_open_file_descriptors.close(auxiliary_vector.front().a_un.a_val));
 			});
 
-			if (executable.has_interpreter)
+			if (executable.open_execfd)
 			{
 				const int execfd = TRY(m_open_file_descriptors.open(BAN::move(executable_file), O_RDONLY));
 				TRY(auxiliary_vector.push_back({
@@ -594,6 +662,13 @@ namespace Kernel
 				auxiliary_vector.span()
 			));
 
+			if (executable.master_tls.has_value())
+			{
+				auto tls_result = TRY(initialize_thread_local_storage(*new_page_table, *executable.master_tls));
+				TRY(new_mapped_regions.emplace_back(BAN::move(tls_result.region)));
+				new_thread->set_tls(tls_result.addr);
+			}
+
 			ASSERT(Processor::get_interrupt_state() == InterruptState::Enabled);
 			Processor::set_interrupt_state(InterruptState::Disabled);
 

userspace/libraries/LibC/CMakeLists.txt

@@ -58,6 +58,10 @@ target_compile_definitions(objlibc PRIVATE __arch=${BANAN_ARCH})
 target_compile_options(objlibc PRIVATE -O2 -g -Wstack-usage=512 -fno-exceptions -fpic -nolibc)
 target_compile_options(objlibc PUBLIC -Wall -Wextra -Werror -Wno-error=stack-usage=)
 
+if("${BANAN_ARCH}" STREQUAL "i686")
+	target_compile_definitions(objlibc PRIVATE __disable_thread_local_storage)
+endif()
+
 function(add_crtx crtx)
 	add_custom_target(${crtx}
 		COMMAND ${CMAKE_CXX_COMPILER} -c -o ${CMAKE_INSTALL_LIBDIR}/${crtx}.o ${CMAKE_CURRENT_SOURCE_DIR}/arch/${BANAN_ARCH}/${crtx}.S
@@ -71,6 +75,7 @@ add_crtx(crtn)
 
 banan_include_headers(objlibc ban)
 banan_include_headers(objlibc kernel)
+banan_include_headers(objlibc libelf)
 
 banan_install_headers(objlibc)
 

userspace/libraries/LibC/errno.cpp

@@ -1,6 +1,10 @@
 #include <errno.h>
 
+#if __disable_thread_local_storage
 static int s_errno = 0;
+#else
+static thread_local int s_errno = 0;
+#endif
 
 int* __errno_location()
 {

userspace/libraries/LibC/include/pthread.h

@@ -8,6 +8,7 @@
 __BEGIN_DECLS
 
 #include <sched.h>
+#include <stdint.h>
 #include <time.h>
 
 #define __need_size_t
@@ -27,6 +28,14 @@ __BEGIN_DECLS
 #define __need_pthread_t
 #include <sys/types.h>
 
+struct uthread
+{
+	struct uthread* self;
+	void* master_tls_addr;
+	size_t master_tls_size;
+	uintptr_t dtv[];
+};
+
 #define PTHREAD_BARRIER_SERIAL_THREAD	1
 #define PTHREAD_CANCEL_ASYNCHRONOUS		2
 #define PTHREAD_CANCEL_ENABLE			3

userspace/libraries/LibC/pthread.cpp

@@ -8,11 +8,13 @@
 #include <pthread.h>
 #include <stdlib.h>
 #include <string.h>
+#include <sys/mman.h>
 #include <sys/syscall.h>
 #include <unistd.h>
 
 struct pthread_trampoline_info_t
 {
+	struct uthread* uthread;
 	void* (*start_routine)(void*);
 	void* arg;
 };
@@ -41,11 +43,36 @@ asm(
 extern "C" void _pthread_trampoline_cpp(void* arg)
 {
 	auto info = *reinterpret_cast<pthread_trampoline_info_t*>(arg);
+	syscall(SYS_SET_TLS, info.uthread);
 	free(arg);
 	pthread_exit(info.start_routine(info.arg));
 	ASSERT_NOT_REACHED();
 }
 
+static uthread* get_uthread()
+{
+	uthread* result;
+#if ARCH(x86_64)
+	asm volatile("movq %%fs:0, %0" : "=r"(result));
+#elif ARCH(i686)
+	asm volatile("movl %%gs:0, %0" : "=r"(result));
+#endif
+	return result;
+}
+
+static void free_uthread(uthread* uthread)
+{
+	if (uthread->dtv[0] == 0)
+		return free(uthread);
+
+	uint8_t* tls_addr = reinterpret_cast<uint8_t*>(uthread) - uthread->master_tls_size;
+	const size_t tls_size = uthread->master_tls_size
+		+ sizeof(struct uthread)
+		+ (uthread->dtv[0] + 1) * sizeof(uintptr_t);
+	munmap(tls_addr, tls_size);
+}
+
+#if not __disable_thread_local_storage
 struct pthread_cleanup_t
 {
 	void (*routine)(void*);
@@ -79,6 +106,7 @@ void pthread_cleanup_push(void (*routine)(void*), void* arg)
 
 	s_cleanup_stack = cleanup;
 }
+#endif
 
 int pthread_attr_init(pthread_attr_t* attr)
 {
@@ -93,29 +121,76 @@ int pthread_create(pthread_t* __restrict thread_id, const pthread_attr_t* __rest
 		return errno;
 
 	*info = {
+		.uthread = nullptr,
 		.start_routine = start_routine,
 		.arg = arg,
 	};
 
-	const auto ret = syscall(SYS_PTHREAD_CREATE, attr, pthread_trampoline, info);
+	long syscall_ret = 0;
-	if (ret == -1)
+
+	if (uthread* self = get_uthread(); self->master_tls_addr == nullptr)
+	{
+		uthread* uthread = static_cast<struct uthread*>(malloc(sizeof(struct uthread) + sizeof(uintptr_t)));
+		if (uthread == nullptr)
+			goto pthread_create_error;
+		uthread->self = uthread;
+		uthread->master_tls_addr = nullptr;
+		uthread->master_tls_size = 0;
+		uthread->dtv[0] = 0;
+
+		info->uthread = uthread;
+	}
+	else
+	{
+		const size_t module_count = self->dtv[0];
+
+		const size_t tls_size = self->master_tls_size
+			+ sizeof(uthread)
+			+ (module_count + 1) * sizeof(uintptr_t);
+
+		uint8_t* tls_addr = static_cast<uint8_t*>(mmap(nullptr, tls_size, PROT_READ | PROT_WRITE, MAP_ANONYMOUS | MAP_PRIVATE, -1, 0));
+		if (tls_addr == MAP_FAILED)
+			goto pthread_create_error;
+		memcpy(tls_addr, self->master_tls_addr, self->master_tls_size);
+
+		uthread* uthread = reinterpret_cast<struct uthread*>(tls_addr + self->master_tls_size);
+		uthread->self = uthread;
+		uthread->master_tls_addr = self->master_tls_addr;
+		uthread->master_tls_size = self->master_tls_size;
+
+		const uintptr_t self_addr = reinterpret_cast<uintptr_t>(self);
+		const uintptr_t uthread_addr = reinterpret_cast<uintptr_t>(uthread);
+
+		uthread->dtv[0] = module_count;
+		for (size_t i = 1; i <= module_count; i++)
+			uthread->dtv[i] = self->dtv[i] - self_addr + uthread_addr;
+
+		info->uthread = uthread;
+	}
+
+	syscall_ret = syscall(SYS_PTHREAD_CREATE, attr, _pthread_trampoline, info);
+	if (syscall_ret == -1)
 		goto pthread_create_error;
 
-
 	if (thread_id)
-		*thread_id = ret;
+		*thread_id = syscall_ret;
 	return 0;
 
 pthread_create_error:
 	const int return_code = errno;
+	if (info->uthread)
+		free_uthread(info->uthread);
 	free(info);
 	return return_code;
 }
 
 void pthread_exit(void* value_ptr)
 {
+#if not __disable_thread_local_storage
 	while (s_cleanup_stack)
 		pthread_cleanup_pop(1);
+#endif
+	free_uthread(get_uthread());
 	syscall(SYS_PTHREAD_EXIT, value_ptr);
 	ASSERT_NOT_REACHED();
 }
@@ -127,7 +202,14 @@ int pthread_join(pthread_t thread, void** value_ptr)
 
 pthread_t pthread_self(void)
 {
+#if __disable_thread_local_storage
 	return syscall(SYS_PTHREAD_SELF);
+#else
+	static thread_local pthread_t s_pthread_self { -1 };
+	if (s_pthread_self == -1) [[unlikely]]
+		s_pthread_self = syscall(SYS_PTHREAD_SELF);
+	return s_pthread_self;
+#endif
 }
 
 static inline BAN::Atomic<pthread_t>& pthread_spin_get_atomic(pthread_spinlock_t* lock)
@@ -188,3 +270,21 @@ int pthread_spin_unlock(pthread_spinlock_t* lock)
 	atomic.store(0, BAN::MemoryOrder::memory_order_release);
 	return 0;
 }
+
+struct tls_index
+{
+	unsigned long int ti_module;
+	unsigned long int ti_offset;
+};
+
+extern "C" void* __tls_get_addr(tls_index* ti)
+{
+	return reinterpret_cast<void*>(get_uthread()->dtv[ti->ti_module] + ti->ti_offset);
+}
+
+#if ARCH(i686)
+extern "C" void* __attribute__((__regparm__(1))) ___tls_get_addr(tls_index* ti)
+{
+	return reinterpret_cast<void*>(get_uthread()->dtv[ti->ti_module] + ti->ti_offset);
+}
+#endif

userspace/libraries/LibC/unistd.cpp

@@ -1,10 +1,12 @@
 #include <BAN/Assert.h>
 #include <BAN/Debug.h>
+
 #include <kernel/Memory/Types.h>
 #include <kernel/Syscall.h>
 
 #include <errno.h>
 #include <fcntl.h>
+#include <pthread.h>
 #include <pwd.h>
 #include <stdarg.h>
 #include <stdio.h>
@@ -30,6 +32,21 @@ extern "C" void _init_libc(char** environ, init_funcs_t init_funcs, init_funcs_t
 	if (::environ == nullptr)
 		::environ = environ;
 
+	if (syscall(SYS_GET_TLS) == 0)
+	{
+		alignas(uthread) static uint8_t storage[sizeof(uthread) + sizeof(uintptr_t)];
+
+		uthread& uthread = *reinterpret_cast<struct uthread*>(storage);
+		uthread = {
+			.self = &uthread,
+			.master_tls_addr = nullptr,
+			.master_tls_size = 0,
+		};
+		uthread.dtv[0] = 0;
+
+		syscall(SYS_SET_TLS, &uthread);
+	}
+
 	// call global constructors
 	if (init_funcs.func)
 		init_funcs.func();

userspace/programs/DynamicLoader/main.cpp

@@ -6,6 +6,7 @@
 
 #include <fcntl.h>
 #include <limits.h>
+#include <pthread.h>
 #include <sys/mman.h>
 #include <sys/syscall.h>
 #include <unistd.h>
@@ -145,8 +146,13 @@ static void resolve_symbol_trampoline()
 struct LoadedElf
 {
 	ElfNativeFileHeader file_header;
+	ElfNativeProgramHeader tls_header;
 	ElfNativeDynamic* dynamics;
 
+	uint8_t* tls_addr;
+	size_t tls_module;
+	size_t tls_offset;
+
 	int fd;
 
 	uintptr_t base;
@@ -326,9 +332,19 @@ static void handle_tls_relocation(const LoadedElf& elf, const RelocT& reloc)
 	if (!is_tls_relocation(reloc))
 		return;
 
+	if (ELF_R_SYM(reloc.r_info))
+		print_error_and_exit("tls relocation with symbol index", 0);
+
+	if (elf.tls_addr == nullptr)
+		print_error_and_exit("tls relocation without tls", 0);
+
+
 #if defined(__x86_64__)
 	switch (ELF64_R_TYPE(reloc.r_info))
 	{
+		case R_X86_64_DTPMOD64:
+			*reinterpret_cast<uint64_t*>(elf.base + reloc.r_offset) = elf.tls_module;
+			break;
 		default:
 			print(STDERR_FILENO, "unsupported tls reloc type ");
 			print_uint(STDERR_FILENO, ELF64_R_TYPE(reloc.r_info));
@@ -339,6 +355,9 @@ static void handle_tls_relocation(const LoadedElf& elf, const RelocT& reloc)
 #elif defined(__i686__)
 	switch (ELF32_R_TYPE(reloc.r_info))
 	{
+		case R_386_TLS_DTPMOD32:
+			*reinterpret_cast<uint32_t*>(elf.base + reloc.r_offset) = elf.tls_module;
+			break;
 		default:
 			print(STDERR_FILENO, "unsupported tls reloc type ");
 			print_uint(STDERR_FILENO, ELF64_R_TYPE(reloc.r_info));
@@ -890,6 +909,9 @@ static LoadedElf& load_elf(const char* path, int fd)
 		break;
 	}
 
+	ElfNativeProgramHeader tls_header {};
+	tls_header.p_type = PT_NULL;
+
 	for (size_t i = 0; i < file_header.e_phnum; i++)
 	{
 		ElfNativeProgramHeader program_header;
@@ -909,6 +931,9 @@ static LoadedElf& load_elf(const char* path, int fd)
 			case PT_GNU_RELRO:
 				print(STDDBG_FILENO, "TODO: PT_GNU_*\n");
 				break;
+			case PT_TLS:
+				tls_header = program_header;
+				break;
 			case PT_LOAD:
 				program_header.p_vaddr += base;
 				load_program_header(program_header, fd, needs_writable);
@@ -921,6 +946,7 @@ static LoadedElf& load_elf(const char* path, int fd)
 	}
 
 	auto& elf = s_loaded_files[s_loaded_file_count++];
+	elf.tls_header = tls_header;
 	elf.base = base;
 	elf.fd = fd;
 	elf.dynamics = nullptr;
@@ -950,6 +976,139 @@ static LoadedElf& load_elf(const char* path, int fd)
 	return elf;
 }
 
+struct MasterTLS
+{
+	uint8_t* addr;
+	size_t size;
+	size_t module_count;
+};
+
+static MasterTLS initialize_master_tls()
+{
+	constexpr auto round =
+		[](size_t a, size_t b) -> size_t
+		{
+			return b * ((a + b - 1) / b);
+		};
+
+	size_t max_align = alignof(uthread);
+	size_t tls_m_offset = 0;
+	size_t tls_m_size = 0;
+	size_t module_count = 0;
+	for (size_t i = 0; i < s_loaded_file_count; i++)
+	{
+		const auto& tls_header = s_loaded_files[i].tls_header;
+		if (tls_header.p_type != PT_TLS)
+			continue;
+		if (tls_header.p_align == 0)
+			print_error_and_exit("TLS alignment is 0", 0);
+
+		max_align = max<size_t>(max_align, tls_header.p_align);
+		tls_m_offset = round(tls_m_offset + tls_header.p_memsz, tls_header.p_align);
+		tls_m_size = tls_header.p_memsz;
+
+		module_count++;
+	}
+
+	if (module_count == 0)
+		return { .addr = nullptr, .size = 0, .module_count = 0 };
+
+	size_t master_tls_size = tls_m_offset + tls_m_size;
+	if (auto rem = master_tls_size % max_align)
+		master_tls_size += max_align - rem;
+
+	uint8_t* master_tls_addr;
+
+	{
+		const sys_mmap_t mmap_args {
+			.addr = nullptr,
+			.len = master_tls_size,
+			.prot = PROT_READ | PROT_WRITE,
+			.flags = MAP_ANONYMOUS | MAP_PRIVATE,
+			.fildes = -1,
+			.off = 0,
+		};
+
+		const auto ret = syscall(SYS_MMAP, &mmap_args);
+		if (ret < 0)
+			print_error_and_exit("failed to allocate master TLS", ret);
+		master_tls_addr = reinterpret_cast<uint8_t*>(ret);
+	}
+
+	for (size_t i = 0, tls_offset = 0, tls_module = 1; i < s_loaded_file_count; i++)
+	{
+		const auto& tls_header = s_loaded_files[i].tls_header;
+		if (tls_header.p_type != PT_TLS)
+			continue;
+
+		tls_offset = round(tls_offset + tls_header.p_memsz, tls_header.p_align);
+
+		uint8_t* tls_buffer = master_tls_addr + master_tls_size - tls_offset;
+
+		if (tls_header.p_filesz > 0)
+		{
+			const int fd = s_loaded_files[i].fd;
+			if (auto ret = syscall(SYS_PREAD, fd, tls_buffer, tls_header.p_filesz, tls_header.p_offset); ret != static_cast<long>(tls_header.p_filesz))
+				print_error_and_exit("failed to read TLS data", ret);
+		}
+
+		memset(tls_buffer + tls_header.p_filesz, 0, tls_header.p_memsz - tls_header.p_filesz);
+
+		auto& elf = s_loaded_files[i];
+		elf.tls_addr = tls_buffer;
+		elf.tls_module = tls_module++;
+		elf.tls_offset = master_tls_size - tls_offset;
+	}
+
+	return { .addr = master_tls_addr, .size = master_tls_size, .module_count = module_count };
+}
+
+static void initialize_tls(MasterTLS master_tls)
+{
+	if (master_tls.addr == nullptr)
+		return;
+
+	const size_t tls_size = master_tls.size
+		+ sizeof(uthread)
+		+ (master_tls.module_count + 1) * sizeof(uintptr_t);
+
+	uint8_t* tls_addr;
+
+	{
+		const sys_mmap_t mmap_args {
+			.addr = nullptr,
+			.len = tls_size,
+			.prot = PROT_READ | PROT_WRITE,
+			.flags = MAP_ANONYMOUS | MAP_PRIVATE,
+			.fildes = -1,
+			.off = 0,
+		};
+
+		const auto ret = syscall(SYS_MMAP, &mmap_args);
+		if (ret < 0)
+			print_error_and_exit("failed to allocate master TLS", ret);
+		tls_addr = reinterpret_cast<uint8_t*>(ret);
+	}
+
+	memcpy(tls_addr, master_tls.addr, master_tls.size);
+
+	uthread* uthread = reinterpret_cast<struct uthread*>(tls_addr + master_tls.size);
+	uthread->self = uthread;
+	uthread->master_tls_addr = master_tls.addr;
+	uthread->master_tls_size = master_tls.size;
+
+	uthread->dtv[0] = master_tls.module_count;
+	for (size_t i = 0; i < s_loaded_file_count; i++)
+	{
+		const auto& elf = s_loaded_files[i];
+		if (elf.tls_addr == nullptr)
+			continue;
+		uthread->dtv[elf.tls_module] = reinterpret_cast<uintptr_t>(tls_addr) + elf.tls_offset;
+	}
+
+	syscall(SYS_SET_TLS, uthread);
+}
+
 static void initialize_environ(char** envp)
 {
 	uintptr_t environ = SYM_NOT_FOUND;
@@ -1040,7 +1199,9 @@ uintptr_t _entry(int argc, char* argv[], char* envp[])
 	auto& elf = load_elf(argv[0], execfd);
 	fini_random();
 
+	const auto master_tls = initialize_master_tls();
 	relocate_elf(elf, true);
+	initialize_tls(master_tls);
 	initialize_environ(envp);
 	call_init_funcs(elf, true);