Kernel: Fix ByteRingBuffer->back()

No need to swap bytes of every 16 bit word in the packet, we can just do
one swap at the return

.gitignore

@@ -3,3 +3,4 @@
 build/
 base/
 script/fakeroot-context
+tools/update-image-perms

BAN/CMakeLists.txt

@@ -11,5 +11,8 @@ banan_link_library(ban libc)
 
 set_target_properties(ban PROPERTIES OUTPUT_NAME libban)
 
+# set SONAME as cmake doesn't set it for some reason??
+set_target_properties(ban PROPERTIES LINK_FLAGS "-Wl,-soname,libban.so")
+
 banan_install_headers(ban)
 install(TARGETS ban OPTIONAL)

BAN/include/BAN/Array.h

@@ -18,78 +18,78 @@ namespace BAN
 		using const_iterator = ConstIteratorSimple<T, Array>;
 
 	public:
-		Array() = default;
-		Array(const T&);
+		constexpr Array() = default;
+		constexpr Array(const T&);
 
 		iterator begin() { return iterator(m_data); }
 		iterator end() { return iterator(m_data + size()); }
 		const_iterator begin() const { return const_iterator(m_data); }
 		const_iterator end() const { return const_iterator(m_data + size()); }
 
-		const T& operator[](size_type) const;
-		T& operator[](size_type);
+		constexpr const T& operator[](size_type) const;
+		constexpr T& operator[](size_type);
 
-		const T& back() const;
-		T& back();
-		const T& front() const;
-		T& front();
+		constexpr const T& back() const;
+		constexpr T& back();
+		constexpr const T& front() const;
+		constexpr T& front();
 
 		Span<T> span() { return Span(m_data, size()); }
-		const Span<T> span() const { return Span(m_data, size()); }
+		Span<const T> span() const { return Span(m_data, size()); }
 
 		constexpr size_type size() const;
 
-		const T* data() const { return m_data; }
-		T* data() { return m_data; }
+		constexpr const T* data() const { return m_data; }
+		constexpr T* data() { return m_data; }
 
 	private:
 		T m_data[S] {};
 	};
 
 	template<typename T, size_t S>
-	Array<T, S>::Array(const T& value)
+	constexpr Array<T, S>::Array(const T& value)
 	{
 		for (size_type i = 0; i < S; i++)
 			m_data[i] = value;
 	}
 
 	template<typename T, size_t S>
-	const T& Array<T, S>::operator[](size_type index) const
+	constexpr const T& Array<T, S>::operator[](size_type index) const
 	{
 		ASSERT(index < S);
 		return m_data[index];
 	}
 
 	template<typename T, size_t S>
-	T& Array<T, S>::operator[](size_type index)
+	constexpr T& Array<T, S>::operator[](size_type index)
 	{
 		ASSERT(index < S);
 		return m_data[index];
 	}
 
 	template<typename T, size_t S>
-	const T& Array<T, S>::back() const
+	constexpr const T& Array<T, S>::back() const
 	{
 		ASSERT(S != 0);
 		return m_data[S - 1];
 	}
 
 	template<typename T, size_t S>
-	T& Array<T, S>::back()
+	constexpr T& Array<T, S>::back()
 	{
 		ASSERT(S != 0);
 		return m_data[S - 1];
 	}
 
 	template<typename T, size_t S>
-	const T& Array<T, S>::front() const
+	constexpr const T& Array<T, S>::front() const
 	{
 		ASSERT(S != 0);
 		return m_data[0];
 	}
 
 	template<typename T, size_t S>
-	T& Array<T, S>::front()
+	constexpr T& Array<T, S>::front()
 	{
 		ASSERT(S != 0);
 		return m_data[0];

BAN/include/BAN/Atomic.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <BAN/Traits.h>
+
 namespace BAN
 {
 
@@ -13,8 +15,36 @@ namespace BAN
 		memory_order_seq_cst = __ATOMIC_SEQ_CST,
 	};
 
-	template<typename T, MemoryOrder MEM_ORDER = MemoryOrder::memory_order_seq_cst>
-	requires requires { __atomic_always_lock_free(sizeof(T), 0); }
+	template<typename T> concept atomic_c = is_integral_v<T> || is_pointer_v<T>;
+	template<typename T> concept atomic_lockfree_c = (is_integral_v<T> || is_pointer_v<T>) && __atomic_always_lock_free(sizeof(T), 0);
+
+	template<atomic_lockfree_c T, atomic_c U>
+	inline void atomic_store(T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { __atomic_store_n(&obj, value, mem_order); }
+	template<atomic_lockfree_c T>
+	inline T atomic_load(T& obj, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_load_n(&obj, mem_order); }
+
+	template<atomic_lockfree_c T, atomic_c U>
+	inline T atomic_exchange(T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_exchange_n(&obj, value, mem_order); }
+	template<atomic_lockfree_c T, atomic_lockfree_c U, atomic_c V>
+	inline bool atomic_compare_exchange(T& obj, U& expected, V value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_compare_exchange_n(&obj, &expected, value, false, mem_order, mem_order); }
+
+#define DECL_ATOMIC_INLINE template<atomic_lockfree_c T, atomic_c U> inline
+	DECL_ATOMIC_INLINE T atomic_add_fetch (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_add_fetch (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_sub_fetch (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_sub_fetch (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_and_fetch (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_and_fetch (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_xor_fetch (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_xor_fetch (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_or_fetch  (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_or_fetch  (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_nand_fetch(T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_nand_fetch(&obj, value, mem_order); }
+
+	DECL_ATOMIC_INLINE T atomic_fetch_add (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_fetch_add (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_fetch_sub (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_fetch_sub (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_fetch_and (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_fetch_and (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_fetch_xor (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_fetch_xor (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_fetch_or  (T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_fetch_or  (&obj, value, mem_order); }
+	DECL_ATOMIC_INLINE T atomic_fetch_nand(T& obj, U value, MemoryOrder mem_order = MemoryOrder::memory_order_seq_cst) { return __atomic_fetch_nand(&obj, value, mem_order); }
+#undef DECL_ATOMIC_INLINE
+
+	template<atomic_lockfree_c T, MemoryOrder MEM_ORDER = MemoryOrder::memory_order_seq_cst>
 	class Atomic
 	{
 		Atomic(const Atomic&) = delete;
@@ -26,41 +56,41 @@ namespace BAN
 		constexpr Atomic() : m_value(0) {}
 		constexpr Atomic(T val) : m_value(val) {}
 
-		inline T load(MemoryOrder mem_order = MEM_ORDER) const volatile			{ return __atomic_load_n(&m_value, mem_order); }
-		inline void store(T val, MemoryOrder mem_order = MEM_ORDER) volatile	{ __atomic_store_n(&m_value, val, mem_order); }
+		inline T load(MemoryOrder mem_order = MEM_ORDER) const volatile			{ return atomic_load(m_value, mem_order); }
+		inline void store(T val, MemoryOrder mem_order = MEM_ORDER) volatile	{ atomic_store(m_value, val, mem_order); }
 
 		inline T operator=(T val) volatile { store(val); return val; }
 
 		inline operator T() const volatile { return load(); }
 
-		inline T operator+=(T val) volatile { return __atomic_add_fetch(&m_value, val, MEM_ORDER); }
-		inline T operator-=(T val) volatile { return __atomic_sub_fetch(&m_value, val, MEM_ORDER); }
-		inline T operator&=(T val) volatile { return __atomic_and_fetch(&m_value, val, MEM_ORDER); }
-		inline T operator^=(T val) volatile { return __atomic_xor_fetch(&m_value, val, MEM_ORDER); }
-		inline T operator|=(T val) volatile { return __atomic_or_fetch(&m_value, val, MEM_ORDER); }
+		inline T operator+=(T val) volatile { return atomic_add_fetch(m_value, val, MEM_ORDER); }
+		inline T operator-=(T val) volatile { return atomic_sub_fetch(m_value, val, MEM_ORDER); }
+		inline T operator&=(T val) volatile { return atomic_and_fetch(m_value, val, MEM_ORDER); }
+		inline T operator^=(T val) volatile { return atomic_xor_fetch(m_value, val, MEM_ORDER); }
+		inline T operator|=(T val) volatile { return atomic_or_fetch(m_value, val, MEM_ORDER); }
 
-		inline T operator--() volatile { return __atomic_sub_fetch(&m_value, 1, MEM_ORDER); }
-		inline T operator++() volatile { return __atomic_add_fetch(&m_value, 1, MEM_ORDER); }
+		inline T operator--() volatile { return atomic_sub_fetch(m_value, 1, MEM_ORDER); }
+		inline T operator++() volatile { return atomic_add_fetch(m_value, 1, MEM_ORDER); }
 
-		inline T operator--(int) volatile { return __atomic_fetch_sub(&m_value, 1, MEM_ORDER); }
-		inline T operator++(int) volatile { return __atomic_fetch_add(&m_value, 1, MEM_ORDER); }
+		inline T operator--(int) volatile { return atomic_fetch_sub(m_value, 1, MEM_ORDER); }
+		inline T operator++(int) volatile { return atomic_fetch_add(m_value, 1, MEM_ORDER); }
 
-		inline bool compare_exchange(T& expected, T desired, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_compare_exchange_n(&m_value, &expected, desired, false, mem_order, mem_order); }
-		inline T exchange(T desired, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_exchange_n(&m_value, desired, mem_order); };
+		inline bool compare_exchange(T& expected, T desired, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_compare_exchange(m_value, expected, desired, mem_order); }
+		inline T exchange(T desired, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_exchange(m_value, desired, mem_order); };
 
-		inline T add_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_add_fetch (&m_value, val, mem_order); }
-		inline T sub_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_sub_fetch (&m_value, val, mem_order); }
-		inline T and_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_and_fetch (&m_value, val, mem_order); }
-		inline T xor_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_xor_fetch (&m_value, val, mem_order); }
-		inline T or_fetch  (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_or_fetch  (&m_value, val, mem_order); }
-		inline T nand_fetch(T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_nand_fetch(&m_value, val, mem_order); }
+		inline T add_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_add_fetch (m_value, val, mem_order); }
+		inline T sub_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_sub_fetch (m_value, val, mem_order); }
+		inline T and_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_and_fetch (m_value, val, mem_order); }
+		inline T xor_fetch (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_xor_fetch (m_value, val, mem_order); }
+		inline T or_fetch  (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_or_fetch  (m_value, val, mem_order); }
+		inline T nand_fetch(T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_nand_fetch(m_value, val, mem_order); }
 
-		inline T fetch_add (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_fetch_add (&m_value, val, mem_order); }
-		inline T fetch_sub (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_fetch_sub (&m_value, val, mem_order); }
-		inline T fetch_and (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_fetch_and (&m_value, val, mem_order); }
-		inline T fetch_xor (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_fetch_xor (&m_value, val, mem_order); }
-		inline T fetch_or  (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_fetch__or  (&m_value, val, mem_order); }
-		inline T fetch_nand(T val, MemoryOrder mem_order = MEM_ORDER) volatile { return __atomic_nfetch_and(&m_value, val, mem_order); }
+		inline T fetch_add (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_fetch_add (m_value, val, mem_order); }
+		inline T fetch_sub (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_fetch_sub (m_value, val, mem_order); }
+		inline T fetch_and (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_fetch_and (m_value, val, mem_order); }
+		inline T fetch_xor (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_fetch_xor (m_value, val, mem_order); }
+		inline T fetch_or  (T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_fetch_or  (m_value, val, mem_order); }
+		inline T fetch_nand(T val, MemoryOrder mem_order = MEM_ORDER) volatile { return atomic_fetch_nand(m_value, val, mem_order); }
 
 	private:
 		T m_value;

BAN/include/BAN/CircularQueue.h

@@ -34,6 +34,11 @@ namespace BAN
 		const T& back() const;
 		T& back();
 
+		const T& operator[](size_t index) const;
+		T& operator[](size_t index);
+
+		void clear();
+
 		size_type size() const { return m_size; }
 		bool empty() const { return size() == 0; }
 		bool full() const { return size() == capacity(); }
@@ -53,8 +58,7 @@ namespace BAN
 	template<typename T, size_t S>
 	CircularQueue<T, S>::~CircularQueue()
 	{
-		for (size_type i = 0; i < m_size; i++)
-			element_at((m_first + i) % capacity())->~T();
+		clear();
 	}
 
 	template<typename T, size_t S>
@@ -115,6 +119,28 @@ namespace BAN
 		return *element_at((m_first + m_size - 1) % capacity());
 	}
 
+	template<typename T, size_t S>
+	const T& CircularQueue<T, S>::operator[](size_t index) const
+	{
+		ASSERT(index < m_size);
+		return *element_at((m_first + index) % capacity());
+	}
+
+	template<typename T, size_t S>
+	T& CircularQueue<T, S>::operator[](size_t index)
+	{
+		ASSERT(index < m_size);
+		return *element_at((m_first + index) % capacity());
+	}
+
+	template<typename T, size_t S>
+	void CircularQueue<T, S>::clear()
+	{
+		for (size_type i = 0; i < m_size; i++)
+			element_at((m_first + i) % capacity())->~T();
+		m_size = 0;
+	}
+
 	template<typename T, size_t S>
 	const T* CircularQueue<T, S>::element_at(size_type index) const
 	{

BAN/include/BAN/Debug.h

@@ -9,29 +9,35 @@
 #include <BAN/Formatter.h>
 #include <stdio.h>
 
-#define __debug_putchar [](int c) { putc(c, stddbg); }
+#define __debug_putchar [](int c) { putc_unlocked(c, stddbg); }
 
 #define dprintln(...)											\
 	do {														\
+		flockfile(stddbg);										\
 		BAN::Formatter::print(__debug_putchar, __VA_ARGS__);	\
-		BAN::Formatter::print(__debug_putchar,"\r\n");			\
+		BAN::Formatter::print(__debug_putchar,"\n");			\
 		fflush(stddbg);											\
+		funlockfile(stddbg);									\
 	} while (false)
 
 #define dwarnln(...)											\
 	do {														\
+		flockfile(stddbg);										\
 		BAN::Formatter::print(__debug_putchar, "\e[33m");		\
 		BAN::Formatter::print(__debug_putchar, __VA_ARGS__);	\
-		BAN::Formatter::print(__debug_putchar, "\e[m\r\n");		\
+		BAN::Formatter::print(__debug_putchar, "\e[m\n");		\
 		fflush(stddbg);											\
+		funlockfile(stddbg);									\
 	} while(false)
 
 #define derrorln(...)											\
 	do {														\
+		flockfile(stddbg);										\
 		BAN::Formatter::print(__debug_putchar, "\e[31m");		\
 		BAN::Formatter::print(__debug_putchar, __VA_ARGS__);	\
-		BAN::Formatter::print(__debug_putchar, "\e[m\r\n");		\
+		BAN::Formatter::print(__debug_putchar, "\e[m\n");		\
 		fflush(stddbg);											\
+		funlockfile(stddbg);									\
 	} while(false)
 
 #define dprintln_if(cond, ...)		\

BAN/include/BAN/Formatter.h

@@ -204,10 +204,14 @@ namespace BAN::Formatter
 		template<typename F, typename T>
 		inline void print_floating(F putc, T value, const ValueFormat& format)
 		{
+			if (value < 0)
+			{
+				putc('-');
+				return print_floating(putc, -value, format);
+			}
+
 			int64_t int_part = (int64_t)value;
 			T frac_part = value - (T)int_part;
-			if (frac_part < 0)
-				frac_part = -frac_part;
 
 			print_integer(putc, int_part, format);
 

BAN/include/BAN/HashMap.h

@@ -1,49 +1,154 @@
 #pragma once
 
-#include <BAN/Hash.h>
-#include <BAN/LinkedList.h>
-#include <BAN/Vector.h>
+#include <BAN/HashSet.h>
 
 namespace BAN
 {
 
-	template<typename Key, typename T, typename HASH = BAN::hash<Key>>
+	template<typename HashSetIt, typename HashMap, typename Entry>
+	class HashMapIterator
+	{
+	public:
+		HashMapIterator() = default;
+
+		Entry& operator*()
+		{
+			return const_cast<Entry&>(m_iterator.operator*());
+		}
+		const Entry& operator*() const
+		{
+			return m_iterator.operator*();
+		}
+
+		Entry* operator->()
+		{
+			return const_cast<Entry*>(m_iterator.operator->());
+		}
+		const Entry* operator->() const
+		{
+			return m_iterator.operator->();
+		}
+
+		HashMapIterator& operator++()
+		{
+			++m_iterator;
+			return *this;
+		}
+		HashMapIterator operator++(int)
+		{
+			auto temp = *this;
+			++(*this);
+			return temp;
+		}
+
+		bool operator==(HashMapIterator other) const
+		{
+			return m_iterator == other.m_iterator;
+		}
+		bool operator!=(HashMapIterator other) const
+		{
+			return m_iterator != other.m_iterator;
+		}
+
+	private:
+		explicit HashMapIterator(HashSetIt it)
+			: m_iterator(it)
+		{ }
+
+	private:
+		HashSetIt m_iterator;
+		friend HashMap;
+	};
+
+	namespace detail
+	{
+		template<typename T, typename Key, typename HASH, typename COMP>
+		concept HashMapFindable = requires(const Key& a, const T& b) { COMP()(a, b); HASH()(b); };
+	}
+
+	template<typename Key, typename T, typename HASH = BAN::hash<Key>, typename COMP = BAN::equal<Key>>
 	class HashMap
 	{
 	public:
 		struct Entry
 		{
-			template<typename... Args>
-			Entry(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-				: key(key)
-				, value(forward<Args>(args)...)
-			{}
-
-			template<typename... Args>
-			Entry(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-				: key(BAN::move(key))
-				, value(forward<Args>(args)...)
-			{}
-
-			Key key;
+			const Key key;
 			T value;
+
+			Entry() = delete;
+			Entry& operator=(const Entry&) = delete;
+			Entry& operator=(Entry&&) = delete;
+
+			Entry(const Entry& other)
+				: key(other.key)
+				, value(other.value)
+			{ }
+
+			Entry(Entry&& other)
+				: key(BAN::move(const_cast<Key&>(other.key)))
+				, value(BAN::move(other.value))
+			{ }
+
+			template<typename... Args>
+			Entry(Key&& key, Args&&... args)
+				: key(BAN::move(key))
+				, value(BAN::forward<Args>(args)...)
+			{ }
+		};
+
+		struct EntryHash
+		{
+			constexpr bool operator()(const Key& a)
+			{
+				return HASH()(a);
+			}
+			constexpr bool operator()(const Entry& a)
+			{
+				return HASH()(a.key);
+			}
+		};
+
+		struct EntryComp
+		{
+			constexpr bool operator()(const Entry& a, const Key& b)
+			{
+				return COMP()(a.key, b);
+			}
+			constexpr bool operator()(const Entry& a, const Entry& b)
+			{
+				return COMP()(a.key, b.key);
+			}
 		};
 
 	public:
-		using size_type = size_t;
-		using key_type = Key;
-		using value_type = T;
-		using iterator = IteratorDouble<Entry, Vector, LinkedList, HashMap>;
-		using const_iterator = ConstIteratorDouble<Entry, Vector, LinkedList, HashMap>;
+		using size_type      = size_t;
+		using key_type       = Key;
+		using value_type     = T;
+		using iterator       = HashMapIterator<typename HashSet<Entry, EntryHash, EntryComp>::iterator,       HashMap, Entry>;
+		using const_iterator = HashMapIterator<typename HashSet<Entry, EntryHash, EntryComp>::const_iterator, HashMap, const Entry>;
 
 	public:
 		HashMap() = default;
-		HashMap(const HashMap<Key, T, HASH>&);
-		HashMap(HashMap<Key, T, HASH>&&);
-		~HashMap();
+		~HashMap() { clear(); }
 
-		HashMap<Key, T, HASH>& operator=(const HashMap<Key, T, HASH>&);
-		HashMap<Key, T, HASH>& operator=(HashMap<Key, T, HASH>&&);
+		HashMap(const HashMap& other) { *this = other; }
+		HashMap& operator=(const HashMap& other)
+		{
+			m_hash_set = other.m_hash_set;
+			return *this;
+		}
+
+		HashMap(HashMap&& other) { *this = BAN::move(other); }
+		HashMap& operator=(HashMap&& other)
+		{
+			m_hash_set = BAN::move(other.m_hash_set);
+			return *this;
+		}
+
+		iterator begin()             { return iterator(m_hash_set.begin()); }
+		iterator end()               { return iterator(m_hash_set.end()); }
+		const_iterator begin() const { return const_iterator(m_hash_set.begin()); }
+		const_iterator end() const   { return const_iterator(m_hash_set.end()); }
 
 		ErrorOr<iterator> insert(const Key& key, const T& value)           { return emplace(key, value); }
 		ErrorOr<iterator> insert(const Key& key, T&& value)                { return emplace(key, move(value)); }
@@ -57,263 +162,100 @@ namespace BAN
 
 		template<typename... Args>
 		ErrorOr<iterator> emplace(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-		{ return emplace(Key(key), forward<Args>(args)...); }
+		{ return emplace(Key(key), BAN::forward<Args>(args)...); }
 		template<typename... Args>
-		ErrorOr<iterator> emplace(Key&&, Args&&...) requires is_constructible_v<T, Args...>;
+		ErrorOr<iterator> emplace(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
+		{
+			ASSERT(!contains(key));
+			auto it = TRY(m_hash_set.insert(Entry { BAN::move(key), T(BAN::forward<Args>(args)...) }));
+			return iterator(it);
+		}
 
 		template<typename... Args>
 		ErrorOr<iterator> emplace_or_assign(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-		{ return emplace_or_assign(Key(key), forward<Args>(args)...); }
+		{ return emplace_or_assign(Key(key), BAN::forward<Args>(args)...); }
 		template<typename... Args>
-		ErrorOr<iterator> emplace_or_assign(Key&&, Args&&...) requires is_constructible_v<T, Args...>;
+		ErrorOr<iterator> emplace_or_assign(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
+		{
+			if (auto it = m_hash_set.find(key); it != m_hash_set.end())
+			{
+				const_cast<T&>(it->value) = T(BAN::forward<Args>(args)...);
+				return iterator(it);
+			}
 
-		iterator begin() { return iterator(m_buckets.end(), m_buckets.begin()); }
-		iterator end()   { return iterator(m_buckets.end(), m_buckets.end()); }
-		const_iterator begin() const { return const_iterator(m_buckets.end(), m_buckets.begin()); }
-		const_iterator end() const   { return const_iterator(m_buckets.end(), m_buckets.end()); }
+			auto it = TRY(m_hash_set.insert(Entry { BAN::move(key), T(BAN::forward<Args>(args)...) }));
+			return iterator(it);
+		}
 
-		ErrorOr<void> reserve(size_type);
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		void remove(const U& key)
+		{
+			if (auto it = find(key); it != end())
+				remove(it);
+		}
 
-		void remove(const Key&);
-		void remove(iterator it);
-		void clear();
+		iterator remove(iterator it)
+		{
+			return iterator(m_hash_set.remove(it.m_iterator));
+		}
 
-		T& operator[](const Key&);
-		const T& operator[](const Key&) const;
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		iterator find(const U& key)
+		{
+			return iterator(m_hash_set.find(key));
+		}
 
-		iterator find(const Key& key);
-		const_iterator find(const Key& key) const;
-		bool contains(const Key&) const;
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		const_iterator find(const U& key) const
+		{
+			return const_iterator(m_hash_set.find(key));
+		}
 
-		bool empty() const;
-		size_type size() const;
+		void clear()
+		{
+			m_hash_set.clear();
+		}
+
+		ErrorOr<void> reserve(size_type size)
+		{
+			return m_hash_set.reserve(size);
+		}
+
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		T& operator[](const U& key)
+		{
+			return find(key)->value;
+		}
+
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		const T& operator[](const U& key) const
+		{
+			return find(key)->value;
+		}
+
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		bool contains(const U& key) const
+		{
+			return find(key) != end();
+		}
+
+		size_type capacity() const
+		{
+			return m_hash_set.capacity();
+		}
+
+		size_type size() const
+		{
+			return m_hash_set.size();
+		}
+
+		bool empty() const
+		{
+			return m_hash_set.empty();
+		}
 
 	private:
-		ErrorOr<void> rebucket(size_type);
-		LinkedList<Entry>& get_bucket(const Key&);
-		const LinkedList<Entry>& get_bucket(const Key&) const;
-		Vector<LinkedList<Entry>>::iterator get_bucket_iterator(const Key&);
-		Vector<LinkedList<Entry>>::const_iterator get_bucket_iterator(const Key&) const;
-
-	private:
-		Vector<LinkedList<Entry>> m_buckets;
-		size_type m_size = 0;
-
-		friend iterator;
+		HashSet<Entry, EntryHash, EntryComp> m_hash_set;
 	};
 
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::HashMap(const HashMap<Key, T, HASH>& other)
-	{
-		*this = other;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::HashMap(HashMap<Key, T, HASH>&& other)
-	{
-		*this = move(other);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::~HashMap()
-	{
-		clear();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>& HashMap<Key, T, HASH>::operator=(const HashMap<Key, T, HASH>& other)
-	{
-		clear();
-		m_buckets = other.m_buckets;
-		m_size = other.m_size;
-		return *this;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>& HashMap<Key, T, HASH>::operator=(HashMap<Key, T, HASH>&& other)
-	{
-		clear();
-		m_buckets = move(other.m_buckets);
-		m_size = other.m_size;
-		other.m_size = 0;
-		return *this;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	template<typename... Args>
-	ErrorOr<typename HashMap<Key, T, HASH>::iterator> HashMap<Key, T, HASH>::emplace(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-	{
-		ASSERT(!contains(key));
-		TRY(rebucket(m_size + 1));
-
-		auto bucket_it = get_bucket_iterator(key);
-		TRY(bucket_it->emplace_back(move(key), forward<Args>(args)...));
-		m_size++;
-
-		return iterator(m_buckets.end(), bucket_it, prev(bucket_it->end(), 1));
-	}
-
-	template<typename Key, typename T, typename HASH>
-	template<typename... Args>
-	ErrorOr<typename HashMap<Key, T, HASH>::iterator> HashMap<Key, T, HASH>::emplace_or_assign(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-	{
-		if (empty())
-			return emplace(move(key), forward<Args>(args)...);
-
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto entry_it = bucket_it->begin(); entry_it != bucket_it->end(); entry_it++)
-		{
-			if (entry_it->key != key)
-				continue;
-			entry_it->value = T(forward<Args>(args)...);
-			return iterator(m_buckets.end(), bucket_it, entry_it);
-		}
-
-		return emplace(move(key), forward<Args>(args)...);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	ErrorOr<void> HashMap<Key, T, HASH>::reserve(size_type size)
-	{
-		TRY(rebucket(size));
-		return {};
-	}
-
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::remove(const Key& key)
-	{
-		auto it = find(key);
-		if (it != end())
-			remove(it);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::remove(iterator it)
-	{
-		it.outer_current()->remove(it.inner_current());
-		m_size--;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::clear()
-	{
-		m_buckets.clear();
-		m_size = 0;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	T& HashMap<Key, T, HASH>::operator[](const Key& key)
-	{
-		ASSERT(!empty());
-		auto& bucket = get_bucket(key);
-		for (Entry& entry : bucket)
-			if (entry.key == key)
-				return entry.value;
-		ASSERT_NOT_REACHED();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	const T& HashMap<Key, T, HASH>::operator[](const Key& key) const
-	{
-		ASSERT(!empty());
-		const auto& bucket = get_bucket(key);
-		for (const Entry& entry : bucket)
-			if (entry.key == key)
-				return entry.value;
-		ASSERT_NOT_REACHED();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	typename HashMap<Key, T, HASH>::iterator HashMap<Key, T, HASH>::find(const Key& key)
-	{
-		if (empty())
-			return end();
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto it = bucket_it->begin(); it != bucket_it->end(); it++)
-			if (it->key == key)
-				return iterator(m_buckets.end(), bucket_it, it);
-		return end();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	typename HashMap<Key, T, HASH>::const_iterator HashMap<Key, T, HASH>::find(const Key& key) const
-	{
-		if (empty())
-			return end();
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto it = bucket_it->begin(); it != bucket_it->end(); it++)
-			if (it->key == key)
-				return const_iterator(m_buckets.end(), bucket_it, it);
-		return end();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	bool HashMap<Key, T, HASH>::contains(const Key& key) const
-	{
-		return find(key) != end();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	bool HashMap<Key, T, HASH>::empty() const
-	{
-		return m_size == 0;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	typename HashMap<Key, T, HASH>::size_type HashMap<Key, T, HASH>::size() const
-	{
-		return m_size;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	ErrorOr<void> HashMap<Key, T, HASH>::rebucket(size_type bucket_count)
-	{
-		if (m_buckets.size() >= bucket_count)
-			return {};
-
-		size_type new_bucket_count = BAN::Math::max<size_type>(bucket_count, m_buckets.size() * 2);
-		Vector<LinkedList<Entry>> new_buckets;
-		TRY(new_buckets.resize(new_bucket_count));
-
-		for (auto& bucket : m_buckets)
-		{
-			for (auto it = bucket.begin(); it != bucket.end();)
-			{
-				size_type new_bucket_index = HASH()(it->key) % new_buckets.size();
-				it = bucket.move_element_to_other_linked_list(new_buckets[new_bucket_index], new_buckets[new_bucket_index].end(), it);
-			}
-		}
-
-		m_buckets = move(new_buckets);
-		return {};
-	}
-
-	template<typename Key, typename T, typename HASH>
-	LinkedList<typename HashMap<Key, T, HASH>::Entry>& HashMap<Key, T, HASH>::get_bucket(const Key& key)
-	{
-		return *get_bucket_iterator(key);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	const LinkedList<typename HashMap<Key, T, HASH>::Entry>& HashMap<Key, T, HASH>::get_bucket(const Key& key) const
-	{
-		return *get_bucket_iterator(key);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	Vector<LinkedList<typename HashMap<Key, T, HASH>::Entry>>::iterator HashMap<Key, T, HASH>::get_bucket_iterator(const Key& key)
-	{
-		ASSERT(!m_buckets.empty());
-		auto index = HASH()(key) % m_buckets.size();
-		return next(m_buckets.begin(), index);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	Vector<LinkedList<typename HashMap<Key, T, HASH>::Entry>>::const_iterator HashMap<Key, T, HASH>::get_bucket_iterator(const Key& key) const
-	{
-		ASSERT(!m_buckets.empty());
-		auto index = HASH()(key) % m_buckets.size();
-		return next(m_buckets.begin(), index);
-	}
-
 }

BAN/include/BAN/HashSet.h

@@ -2,198 +2,358 @@
 
 #include <BAN/Errors.h>
 #include <BAN/Hash.h>
-#include <BAN/Iterators.h>
-#include <BAN/LinkedList.h>
 #include <BAN/Math.h>
 #include <BAN/Move.h>
-#include <BAN/Vector.h>
+#include <BAN/New.h>
 
 namespace BAN
 {
 
-	template<typename T, typename HASH = hash<T>>
-	class HashSet
+	template<typename HashSet, typename Bucket, typename T>
+	class HashSetIterator
 	{
 	public:
-		using value_type = T;
-		using size_type = size_t;
-		using iterator = IteratorDouble<T, Vector, LinkedList, HashSet>;
-		using const_iterator = ConstIteratorDouble<T, Vector, LinkedList, HashSet>;
+		HashSetIterator() = default;
+
+		const T& operator*() const
+		{
+			ASSERT(m_bucket);
+			return *m_bucket->element();
+		}
+
+		const T* operator->() const
+		{
+			ASSERT(m_bucket);
+			return m_bucket->element();
+		}
+
+		HashSetIterator& operator++()
+		{
+			ASSERT(m_bucket);
+			m_bucket++;
+			skip_to_valid_bucket();
+			return *this;
+		}
+		HashSetIterator operator++(int)
+		{
+			auto temp = *this;
+			++(*this);
+			return temp;
+		}
+
+		bool operator==(HashSetIterator other) const
+		{
+			return m_bucket == other.m_bucket;
+		}
+		bool operator!=(HashSetIterator other) const
+		{
+			return m_bucket != other.m_bucket;
+		}
+
+	private:
+		explicit HashSetIterator(Bucket* bucket)
+			: m_bucket(bucket)
+		{
+			if (m_bucket != nullptr)
+				skip_to_valid_bucket();
+		}
+
+		void skip_to_valid_bucket()
+		{
+			while (m_bucket->state != Bucket::USED && !m_bucket->end)
+				m_bucket++;
+			if (m_bucket->end)
+				m_bucket = nullptr;
+		}
+
+	private:
+		Bucket* m_bucket { nullptr };
+		friend HashSet;
+	};
+
+	namespace detail
+	{
+		template<typename T, typename U, typename HASH, typename COMP>
+		concept HashSetFindable = requires(const U& a, const T& b) { COMP()(a, b); HASH()(b); };
+	}
+
+	template<typename T, typename HASH = BAN::hash<T>, typename COMP = BAN::equal<T>>
+	class HashSet
+	{
+	private:
+		struct Bucket
+		{
+			static constexpr uint8_t UNUSED  = 0;
+			static constexpr uint8_t USED    = 1;
+			static constexpr uint8_t REMOVED = 2;
+
+			alignas(T) uint8_t storage[sizeof(T)];
+			hash_t hash;
+			uint8_t state       : 2;
+			uint8_t chain_start : 1;
+			uint8_t end         : 1;
+
+			T* element() { return reinterpret_cast<T*>(storage); }
+			const T* element() const { return reinterpret_cast<const T*>(storage); }
+		};
+
+	public:
+		using value_type     = T;
+		using size_type      = size_t;
+		using iterator       = HashSetIterator<HashSet,       Bucket,       T>;
+		using const_iterator = HashSetIterator<HashSet, const Bucket, const T>;
 
 	public:
 		HashSet() = default;
-		HashSet(const HashSet&);
-		HashSet(HashSet&&);
+		~HashSet() { clear(); }
 
-		HashSet& operator=(const HashSet&);
-		HashSet& operator=(HashSet&&);
+		HashSet(const HashSet& other) { *this = other; }
+		HashSet& operator=(const HashSet& other)
+		{
+			clear();
 
-		ErrorOr<void> insert(const T&);
-		ErrorOr<void> insert(T&&);
-		void remove(const T&);
-		void clear();
+			MUST(reserve(other.size()));
+			for (auto& bucket : other)
+				MUST(insert(bucket));
 
-		ErrorOr<void> reserve(size_type);
+			return *this;
+		}
 
-		iterator begin() { return iterator(m_buckets.end(), m_buckets.begin()); }
-		iterator end()   { return iterator(m_buckets.end(), m_buckets.end()); }
-		const_iterator begin() const { return const_iterator(m_buckets.end(), m_buckets.begin()); }
-		const_iterator end() const   { return const_iterator(m_buckets.end(), m_buckets.end()); }
+		HashSet(HashSet&& other) { *this = BAN::move(other); }
+		HashSet& operator=(HashSet&& other)
+		{
+			clear();
 
-		bool contains(const T&) const;
+			m_buckets  = other.m_buckets;
+			m_capacity = other.m_capacity;
+			m_size     = other.m_size;
+			m_removed  = other.m_removed;
 
-		size_type size() const;
-		bool empty() const;
+			other.m_buckets  = nullptr;
+			other.m_capacity = 0;
+			other.m_size     = 0;
+			other.m_removed  = 0;
+
+			return *this;
+		}
+
+		iterator begin()             { return iterator(m_buckets); }
+		iterator end()               { return iterator(nullptr); }
+		const_iterator begin() const { return const_iterator(m_buckets); }
+		const_iterator end() const   { return const_iterator(nullptr); }
+
+		ErrorOr<iterator> insert(const T& value)
+		{
+			return insert(T(value));
+		}
+
+		ErrorOr<iterator> insert(T&& value)
+		{
+			if (should_rehash_with_size(m_size + 1))
+				TRY(rehash(m_size * 2));
+			return insert_impl(BAN::move(value), HASH()(value));
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		void remove(const U& value)
+		{
+			if (auto it = find(value); it != end())
+				remove(it);
+		}
+
+		iterator remove(iterator it)
+		{
+			auto& bucket = *it.m_bucket;
+			bucket.element()->~T();
+			bucket.state = Bucket::REMOVED;
+			m_size--;
+			m_removed++;
+			return iterator(&bucket);
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		iterator find(const U& value)
+		{
+			return iterator(const_cast<Bucket*>(find_impl(value).m_bucket));
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		const_iterator find(const U& value) const
+		{
+			return find_impl(value);
+		}
+
+		void clear()
+		{
+			if (m_buckets == nullptr)
+				return;
+
+			for (size_type i = 0; i < m_capacity; i++)
+				if (m_buckets[i].state == Bucket::USED)
+					m_buckets[i].element()->~T();
+
+			BAN::deallocator(m_buckets);
+			m_buckets  = nullptr;
+			m_capacity = 0;
+			m_size     = 0;
+			m_removed  = 0;
+		}
+
+		ErrorOr<void> reserve(size_type size)
+		{
+			if (should_rehash_with_size(size))
+				TRY(rehash(size * 2));
+			return {};
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		bool contains(const U& value) const
+		{
+			return find(value) != end();
+		}
+
+		size_type capacity() const
+		{
+			return m_capacity;
+		}
+
+		size_type size() const
+		{
+			return m_size;
+		}
+
+		bool empty() const
+		{
+			return m_size == 0;
+		}
 
 	private:
-		ErrorOr<void> rebucket(size_type);
-		LinkedList<T>& get_bucket(const T&);
-		const LinkedList<T>& get_bucket(const T&) const;
+		ErrorOr<void> rehash(size_type new_capacity)
+		{
+			new_capacity = BAN::Math::max<size_t>(16, BAN::Math::max(new_capacity, m_size + 1));
+			new_capacity = BAN::Math::round_up_to_power_of_two(new_capacity);
+
+			void* new_buckets = BAN::allocator((new_capacity + 1) * sizeof(Bucket));
+			if (new_buckets == nullptr)
+				return BAN::Error::from_errno(ENOMEM);
+			memset(new_buckets, 0, (new_capacity + 1) * sizeof(Bucket));
+
+			Bucket* old_buckets = m_buckets;
+			const size_type old_capacity = m_capacity;
+
+			m_buckets  = static_cast<Bucket*>(new_buckets);
+			m_capacity = new_capacity;
+			m_size     = 0;
+			m_removed  = 0;
+
+			for (size_type i = 0; i < old_capacity; i++)
+			{
+				auto& old_bucket = old_buckets[i];
+				if (old_bucket.state != Bucket::USED)
+					continue;
+				insert_impl(BAN::move(*old_bucket.element()), old_bucket.hash);
+				old_bucket.element()->~T();
+			}
+
+			m_buckets[m_capacity].end = true;
+
+			BAN::deallocator(old_buckets);
+
+			return {};
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		const_iterator find_impl(const U& value) const
+		{
+			if (m_capacity == 0)
+				return end();
+
+			bool first = true;
+			const hash_t orig_hash = HASH()(value);
+			for (auto hash = orig_hash;; hash = get_next_hash_in_chain(hash, orig_hash), first = false)
+			{
+				auto& bucket = m_buckets[hash & (m_capacity - 1)];
+				if (bucket.state == Bucket::USED && bucket.hash == orig_hash && COMP()(*bucket.element(), value))
+					return const_iterator(&bucket);
+				if (bucket.state == Bucket::UNUSED)
+					return end();
+				if (!first && bucket.chain_start)
+					return end();
+			}
+		}
+
+		iterator insert_impl(T&& value, hash_t orig_hash)
+		{
+			ASSERT(!should_rehash_with_size(m_size + 1));
+
+			Bucket* target = nullptr;
+
+			bool first = true;
+			for (auto hash = orig_hash;; hash = get_next_hash_in_chain(hash, orig_hash), first = false)
+			{
+				auto& bucket = m_buckets[hash & (m_capacity - 1)];
+
+				if (!first)
+					bucket.chain_start = false;
+
+				if (bucket.state == Bucket::USED)
+				{
+					if (bucket.hash != orig_hash || !COMP()(*bucket.element(), value))
+						continue;
+					target = &bucket;
+					break;
+				}
+
+				if (target == nullptr)
+					target = &bucket;
+
+				if (bucket.state == Bucket::UNUSED)
+					break;
+			}
+
+			switch (target->state)
+			{
+				case Bucket::USED:
+					target->element()->~T();
+					break;
+				case Bucket::REMOVED:
+					m_removed--;
+					[[fallthrough]];
+				case Bucket::UNUSED:
+					m_size++;
+					break;
+			}
+
+			target->chain_start = first && target->state == Bucket::UNUSED;
+			target->hash = orig_hash;
+			target->state = Bucket::USED;
+
+			new (target->element()) T(BAN::move(value));
+
+			return iterator(target);
+		}
+
+		bool should_rehash_with_size(size_type size) const
+		{
+			if (m_capacity < 16)
+				return true;
+			if (size + m_removed > m_capacity / 4 * 3)
+				return true;
+			return false;
+		}
+
+		hash_t get_next_hash_in_chain(hash_t prev_hash, hash_t orig_hash) const
+		{
+			// TODO: does this even provide better performance than `return prev_hash + 1`
+			//       when using "good" hash functions
+			return prev_hash * 1103515245 + (orig_hash | 1);
+		}
 
 	private:
-		Vector<LinkedList<T>> m_buckets;
-		size_type m_size = 0;
+		Bucket* m_buckets { nullptr };
+		size_type m_capacity { 0 };
+		size_type m_size     { 0 };
+		size_type m_removed  { 0 };
 	};
 
-	template<typename T, typename HASH>
-	HashSet<T, HASH>::HashSet(const HashSet& other)
-		: m_buckets(other.m_buckets)
-		, m_size(other.m_size)
-	{
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>::HashSet(HashSet&& other)
-		: m_buckets(move(other.m_buckets))
-		, m_size(other.m_size)
-	{
-		other.clear();
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>& HashSet<T, HASH>::operator=(const HashSet& other)
-	{
-		clear();
-		m_buckets = other.m_buckets;
-		m_size = other.m_size;
-		return *this;
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>& HashSet<T, HASH>::operator=(HashSet&& other)
-	{
-		clear();
-		m_buckets = move(other.m_buckets);
-		m_size = other.m_size;
-		other.clear();
-		return *this;
-	}
-
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::insert(const T& key)
-	{
-		return insert(move(T(key)));
-	}
-
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::insert(T&& key)
-	{
-		if (!empty() && get_bucket(key).contains(key))
-			return {};
-
-		TRY(rebucket(m_size + 1));
-		TRY(get_bucket(key).push_back(move(key)));
-		m_size++;
-		return {};
-	}
-
-	template<typename T, typename HASH>
-	void HashSet<T, HASH>::remove(const T& key)
-	{
-		if (empty()) return;
-		auto& bucket = get_bucket(key);
-		for (auto it = bucket.begin(); it != bucket.end(); it++)
-		{
-			if (*it == key)
-			{
-				bucket.remove(it);
-				m_size--;
-				break;
-			}
-		}
-	}
-
-	template<typename T, typename HASH>
-	void HashSet<T, HASH>::clear()
-	{
-		m_buckets.clear();
-		m_size = 0;
-	}
-
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::reserve(size_type size)
-	{
-		TRY(rebucket(size));
-		return {};
-	}
-
-	template<typename T, typename HASH>
-	bool HashSet<T, HASH>::contains(const T& key) const
-	{
-		if (empty()) return false;
-		return get_bucket(key).contains(key);
-	}
-
-	template<typename T, typename HASH>
-	typename HashSet<T, HASH>::size_type HashSet<T, HASH>::size() const
-	{
-		return m_size;
-	}
-
-	template<typename T, typename HASH>
-	bool HashSet<T, HASH>::empty() const
-	{
-		return m_size == 0;
-	}
-
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::rebucket(size_type bucket_count)
-	{
-		if (m_buckets.size() >= bucket_count)
-			return {};
-
-		size_type new_bucket_count = Math::max<size_type>(bucket_count, m_buckets.size() * 2);
-		Vector<LinkedList<T>> new_buckets;
-		if (new_buckets.resize(new_bucket_count).is_error())
-			return Error::from_errno(ENOMEM);
-
-		for (auto& bucket : m_buckets)
-		{
-			for (auto it = bucket.begin(); it != bucket.end();)
-			{
-				size_type new_bucket_index = HASH()(*it) % new_buckets.size();
-				it = bucket.move_element_to_other_linked_list(new_buckets[new_bucket_index], new_buckets[new_bucket_index].end(), it);
-			}
-		}
-
-		m_buckets = move(new_buckets);
-		return {};
-	}
-
-	template<typename T, typename HASH>
-	LinkedList<T>& HashSet<T, HASH>::get_bucket(const T& key)
-	{
-		ASSERT(!m_buckets.empty());
-		size_type index = HASH()(key) % m_buckets.size();
-		return m_buckets[index];
-	}
-
-	template<typename T, typename HASH>
-	const LinkedList<T>& HashSet<T, HASH>::get_bucket(const T& key) const
-	{
-		ASSERT(!m_buckets.empty());
-		size_type index = HASH()(key) % m_buckets.size();
-		return m_buckets[index];
-	}
-
 }

BAN/include/BAN/Heap.h

@@ -0,0 +1,89 @@
+#pragma once
+
+#include <BAN/Iterators.h>
+#include <BAN/Swap.h>
+#include <BAN/Traits.h>
+
+namespace BAN
+{
+
+	namespace detail
+	{
+
+		template<typename It, typename Comp>
+		void heapify_up(It begin, size_t index, Comp comp)
+		{
+			size_t parent = (index - 1) / 2;
+			while (parent < index)
+			{
+				if (comp(*(begin + index), *(begin + parent)))
+					break;
+				swap(*(begin + parent), *(begin + index));
+				index = parent;
+				parent = (index - 1) / 2;
+			}
+		}
+
+		template<typename It, typename Comp>
+		void heapify_down(It begin, size_t index, size_t len, Comp comp)
+		{
+			for (;;)
+			{
+				const size_t lchild = 2 * index + 1;
+				const size_t rchild = 2 * index + 2;
+
+				size_t child = 0;
+				if (lchild < len && !comp(*(begin + lchild), *(begin + index)))
+				{
+					if (rchild < len && !comp(*(begin + rchild), *(begin + lchild)))
+						child = rchild;
+					else
+						child = lchild;
+				}
+				else if (rchild < len && !comp(*(begin + rchild), *(begin + index)))
+					child = rchild;
+				else
+					break;
+
+				swap(*(begin + child), *(begin + index));
+				index = child;
+			}
+		}
+
+	}
+
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
+	void make_heap(It begin, It end, Comp comp = {})
+	{
+		const size_t len = distance(begin, end);
+		if (len <= 1)
+			return;
+
+		size_t index = (len - 2) / 2;
+		while (index < len)
+			detail::heapify_down(begin, index--, len, comp);
+	}
+
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
+	void push_heap(It begin, It end, Comp comp = {})
+	{
+		const size_t len = distance(begin, end);
+		detail::heapify_up(begin, len - 1, comp);
+	}
+
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
+	void pop_heap(It begin, It end, Comp comp = {})
+	{
+		const size_t len = distance(begin, end);
+		swap(*begin, *(begin + len - 1));
+		detail::heapify_down(begin, 0, len - 1, comp);
+	}
+
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
+	void sort_heap(It begin, It end, Comp comp = {})
+	{
+		while (begin != end)
+			pop_heap(begin, end--, comp);
+	}
+
+}

BAN/include/BAN/IPv4.h

@@ -9,6 +9,10 @@ namespace BAN
 
 	struct IPv4Address
 	{
+		constexpr IPv4Address()
+			: IPv4Address(0)
+		{ }
+
 		constexpr IPv4Address(uint32_t u32_address)
 		{
 			raw = u32_address;

BAN/include/BAN/MacroUtils.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#define _ban_count_args_impl(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, ...) _9
+#define _ban_count_args(...) _ban_count_args_impl(__VA_ARGS__ __VA_OPT__(,) 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+
+#define _ban_concat_impl(a, b) a##b
+#define _ban_concat(a, b) _ban_concat_impl(a, b)
+
+#define _ban_stringify_impl(x) #x
+#define _ban_stringify(x) _ban_stringify_impl(x)
+
+#define _ban_fe_0(f)
+#define _ban_fe_1(f, _0)                                 f(0, _0)
+#define _ban_fe_2(f, _0, _1)                             f(0, _0) f(1, _1)
+#define _ban_fe_3(f, _0, _1, _2)                         f(0, _0) f(1, _1) f(2, _2)
+#define _ban_fe_4(f, _0, _1, _2, _3)                     f(0, _0) f(1, _1) f(2, _2) f(3, _3)
+#define _ban_fe_5(f, _0, _1, _2, _3, _4)                 f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4)
+#define _ban_fe_6(f, _0, _1, _2, _3, _4, _5)             f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5)
+#define _ban_fe_7(f, _0, _1, _2, _3, _4, _5, _6)         f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5) f(6, _6)
+#define _ban_fe_8(f, _0, _1, _2, _3, _4, _5, _6, _7)     f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5) f(6, _6) f(7, _7)
+#define _ban_fe_9(f, _0, _1, _2, _3, _4, _5, _6, _7, _8) f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5) f(6, _6) f(7, _7) f(8, _8)
+#define _ban_for_each(f, ...) _ban_concat(_ban_fe_, _ban_count_args(__VA_ARGS__))(f __VA_OPT__(,) __VA_ARGS__)
+
+#define _ban_fe_comma_0(f)
+#define _ban_fe_comma_1(f, _0)                                 f(0, _0)
+#define _ban_fe_comma_2(f, _0, _1)                             f(0, _0), f(1, _1)
+#define _ban_fe_comma_3(f, _0, _1, _2)                         f(0, _0), f(1, _1), f(2, _2)
+#define _ban_fe_comma_4(f, _0, _1, _2, _3)                     f(0, _0), f(1, _1), f(2, _2), f(3, _3)
+#define _ban_fe_comma_5(f, _0, _1, _2, _3, _4)                 f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4)
+#define _ban_fe_comma_6(f, _0, _1, _2, _3, _4, _5)             f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5)
+#define _ban_fe_comma_7(f, _0, _1, _2, _3, _4, _5, _6)         f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5), f(6, _6)
+#define _ban_fe_comma_8(f, _0, _1, _2, _3, _4, _5, _6, _7)     f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5), f(6, _6), f(7, _7)
+#define _ban_fe_comma_9(f, _0, _1, _2, _3, _4, _5, _6, _7, _8) f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5), f(6, _6), f(7, _7), f(8, _8)
+#define _ban_for_each_comma(f, ...) _ban_concat(_ban_fe_comma_, _ban_count_args(__VA_ARGS__))(f __VA_OPT__(,) __VA_ARGS__)
+
+#define _ban_get_0(a0, ...)                                     a0
+#define _ban_get_1(a0, a1, ...)                                 a1
+#define _ban_get_2(a0, a1, a2, ...)                             a2
+#define _ban_get_3(a0, a1, a2, a3, ...)                         a3
+#define _ban_get_4(a0, a1, a2, a3, a4, ...)                     a4
+#define _ban_get_5(a0, a1, a2, a3, a4, a5, ...)                 a5
+#define _ban_get_6(a0, a1, a2, a3, a4, a5, a6, ...)             a6
+#define _ban_get_7(a0, a1, a2, a3, a4, a5, a6, a7, ...)         a7
+#define _ban_get_8(a0, a1, a2, a3, a4, a5, a6, a7, a8, ...)     a8
+#define _ban_get_9(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, ...) a9
+#define _ban_get(n, ...) _ban_concat(_ban_get_, n)(__VA_ARGS__)

BAN/include/BAN/Math.h

@@ -36,12 +36,11 @@ namespace BAN::Math
 	template<integral T>
 	inline constexpr T gcd(T a, T b)
 	{
-		T t;
 		while (b)
 		{
-			t = b;
+			T temp = b;
 			b = a % b;
-			a = t;
+			a = temp;
 		}
 		return a;
 	}
@@ -66,25 +65,36 @@ namespace BAN::Math
 		return (x & (x - 1)) == 0;
 	}
 
-	template<BAN::integral T>
-	static constexpr bool will_multiplication_overflow(T a, T b)
+	template<integral T> requires(sizeof(T) <= 8)
+	inline constexpr T round_up_to_power_of_two(T x)
 	{
-		if (a == 0 || b == 0)
-			return false;
-		if ((a > 0) == (b > 0))
-			return a > BAN::numeric_limits<T>::max() / b;
-		else
-			return a < BAN::numeric_limits<T>::min() / b;
+		x--;
+		x |= x >> 1;
+		x |= x >> 2;
+		x |= x >> 4;
+		if constexpr(sizeof(T) >= 2)
+			x |= x >> 8;
+		if constexpr(sizeof(T) >= 4)
+			x |= x >> 16;
+		if constexpr(sizeof(T) >= 8)
+			x |= x >> 32;
+		return x + 1;
 	}
 
-	template<BAN::integral T>
-	static constexpr bool will_addition_overflow(T a, T b)
+	template<integral T>
+	__attribute__((always_inline))
+	inline constexpr bool will_multiplication_overflow(T a, T b)
 	{
-		if (a > 0 && b > 0)
-			return a > BAN::numeric_limits<T>::max() - b;
-		if (a < 0 && b < 0)
-			return a < BAN::numeric_limits<T>::min() - b;
-		return false;
+		T dummy;
+		return __builtin_mul_overflow(a, b, &dummy);
+	}
+
+	template<integral T>
+	__attribute__((always_inline))
+	inline constexpr bool will_addition_overflow(T a, T b)
+	{
+		T dummy;
+		return __builtin_add_overflow(a, b, &dummy);
 	}
 
 	template<typename T>
@@ -98,6 +108,19 @@ namespace BAN::Math
 		return sizeof(T) * 8 - __builtin_clzll(x) - 1;
 	}
 
+// This is ugly but my clangd does not like including
+// intrinsic headers at all
+#if !defined(__SSE__) || !defined(__SSE2__)
+#pragma GCC push_options
+#ifndef __SSE__
+#pragma GCC target("sse")
+#endif
+#ifndef __SSE2__
+#pragma GCC target("sse2")
+#endif
+#define BAN_MATH_POP_OPTIONS
+#endif
+
 	template<floating_point T>
 	inline constexpr T floor(T x)
 	{
@@ -159,7 +182,23 @@ namespace BAN::Math
 			"jne 1b;"
 			: "+t"(a)
 			: "u"(b)
-			: "ax"
+			: "ax", "cc"
+		);
+		return a;
+	}
+
+	template<floating_point T>
+	inline constexpr T remainder(T a, T b)
+	{
+		asm(
+			"1:"
+			"fprem1;"
+			"fnstsw %%ax;"
+			"testb $4, %%ah;"
+			"jne 1b;"
+			: "+t"(a)
+			: "u"(b)
+			: "ax", "cc"
 		);
 		return a;
 	}
@@ -167,7 +206,7 @@ namespace BAN::Math
 	template<floating_point T>
 	static T modf(T x, T* iptr)
 	{
-		const T frac = BAN::Math::fmod<T>(x, 1);
+		const T frac = BAN::Math::fmod<T>(x, (T)1.0);
 		*iptr = x - frac;
 		return frac;
 	}
@@ -175,15 +214,15 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T frexp(T num, int* exp)
 	{
-		if (num == 0.0)
+		if (num == (T)0.0)
 		{
 			*exp = 0;
-			return 0.0;
+			return (T)0.0;
 		}
 
-		T _exp;
-		asm("fxtract" : "+t"(num), "=u"(_exp));
-		*exp = (int)_exp + 1;
+		T e;
+		asm("fxtract" : "+t"(num), "=u"(e));
+		*exp = (int)e + 1;
 		return num / (T)2.0;
 	}
 
@@ -251,6 +290,7 @@ namespace BAN::Math
 			"fstp %%st(1);"
 			: "+t"(x)
 		);
+
 		return x;
 	}
 
@@ -263,18 +303,9 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T pow(T x, T y)
 	{
-		asm(
-			"fyl2x;"
-			"fld1;"
-			"fld %%st(1);"
-			"fprem;"
-			"f2xm1;"
-			"faddp;"
-			"fscale;"
-			: "+t"(x), "+u"(y)
-		);
-
-		return x;
+		if (x == (T)0.0)
+			return (T)0.0;
+		return exp2<T>(y * log2<T>(x));
 	}
 
 	template<floating_point T>
@@ -310,16 +341,27 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T sqrt(T x)
 	{
-		asm("fsqrt" : "+t"(x));
-		return x;
+		if constexpr(BAN::is_same_v<T, float>)
+		{
+			using v4sf = float __attribute__((vector_size(16)));
+			return __builtin_ia32_sqrtss((v4sf) { x, 0.0f, 0.0f, 0.0f })[0];
+		}
+		else if constexpr(BAN::is_same_v<T, double>)
+		{
+			using v2df = double __attribute__((vector_size(16)));
+			return __builtin_ia32_sqrtsd((v2df) { x, 0.0 })[0];
+		}
+		else if constexpr(BAN::is_same_v<T, long double>)
+		{
+			asm("fsqrt" : "+t"(x));
+			return x;
+		}
 	}
 
 	template<floating_point T>
 	inline constexpr T cbrt(T value)
 	{
-		if (value == 0.0)
-			return 0.0;
-		return pow<T>(value, 1.0 / 3.0);
+		return pow<T>(value, (T)1.0 / (T)3.0);
 	}
 
 	template<floating_point T>
@@ -346,30 +388,21 @@ namespace BAN::Math
 	inline constexpr T tan(T x)
 	{
 		T one, ret;
-		asm(
-			"fptan"
-			: "=t"(one), "=u"(ret)
-			: "0"(x)
-		);
+		asm("fptan" : "=t"(one), "=u"(ret) : "0"(x));
 		return ret;
 	}
 
 	template<floating_point T>
 	inline constexpr T atan2(T y, T x)
 	{
-		asm(
-			"fpatan"
-			: "+t"(x)
-			: "u"(y)
-			: "st(1)"
-		);
+		asm("fpatan" : "+t"(x) : "u"(y) : "st(1)");
 		return x;
 	}
 
 	template<floating_point T>
 	inline constexpr T atan(T x)
 	{
-		return atan2<T>(x, 1.0);
+		return atan2<T>(x, (T)1.0);
 	}
 
 	template<floating_point T>
@@ -378,10 +411,10 @@ namespace BAN::Math
 		if (x == (T)0.0)
 			return (T)0.0;
 		if (x == (T)1.0)
-			return  numbers::pi_v<T> / (T)2.0;
+			return +numbers::pi_v<T> / (T)2.0;
 		if (x == (T)-1.0)
 			return -numbers::pi_v<T> / (T)2.0;
-		return (T)2.0 * atan<T>(x / (T(1.0) + sqrt<T>((T)1.0 - x * x)));
+		return (T)2.0 * atan<T>(x / ((T)1.0 + sqrt<T>((T)1.0 - x * x)));
 	}
 
 	template<floating_point T>
@@ -411,7 +444,7 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T tanh(T x)
 	{
-		const T exp_px = exp<T>(x);
+		const T exp_px = exp<T>(+x);
 		const T exp_nx = exp<T>(-x);
 		return (exp_px - exp_nx) / (exp_px + exp_nx);
 	}
@@ -440,4 +473,9 @@ namespace BAN::Math
 		return sqrt<T>(x * x + y * y);
 	}
 
+#ifdef BAN_MATH_POP_OPTIONS
+#undef BAN_MATH_POP_OPTIONS
+#pragma GCC pop_options
+#endif
+
 }

BAN/include/BAN/New.h

@@ -9,10 +9,12 @@
 namespace BAN
 {
 	#if defined(__is_kernel)
-		static constexpr void*(&allocator)(size_t) = kmalloc;
-		static constexpr void(&deallocator)(void*) = kfree;
+		static constexpr void*(*allocator)(size_t) = kmalloc;
+		static constexpr void*(*reallocator)(void*, size_t) = nullptr;
+		static constexpr void(*deallocator)(void*) = kfree;
 	#else
-		static constexpr void*(&allocator)(size_t) = malloc;
-		static constexpr void(&deallocator)(void*) = free;
+		static constexpr void*(*allocator)(size_t) = malloc;
+		static constexpr void*(*reallocator)(void*, size_t) = realloc;
+		static constexpr void(*deallocator)(void*) = free;
 	#endif
 }

BAN/include/BAN/PriorityQueue.h

@@ -0,0 +1,64 @@
+#pragma once
+
+#include "BAN/Errors.h"
+#include <BAN/Vector.h>
+#include <BAN/Heap.h>
+
+namespace BAN
+{
+
+	template<typename T, typename Comp = less<T>>
+	class PriorityQueue
+	{
+	public:
+		PriorityQueue() = default;
+		PriorityQueue(Comp comp)
+			: m_comp(comp)
+		{ }
+
+		ErrorOr<void> push(const T& value)
+		{
+			TRY(m_data.push_back(value));
+			push_heap(m_data.begin(), m_data.end());
+			return {};
+		}
+
+		ErrorOr<void> push(T&& value)
+		{
+			TRY(m_data.push_back(move(value)));
+			push_heap(m_data.begin(), m_data.end());
+			return {};
+		}
+
+		template<typename... Args>
+		ErrorOr<void> emplace(Args&&... args) requires is_constructible_v<T, Args...>
+		{
+			TRY(m_data.emplace_back(forward<Args>(args)...));
+			push_heap(m_data.begin(), m_data.end());
+			return {};
+		}
+
+		void pop()
+		{
+			pop_heap(m_data.begin(), m_data.end());
+			m_data.pop_back();
+		}
+
+		BAN::ErrorOr<void> reserve(Vector<T>::size_type size)
+		{
+			return m_data.reserve(size);
+		}
+
+		T& top() { return m_data.front(); }
+		const T& top() const { return m_data.front(); }
+
+		bool empty() const { return m_data.empty(); }
+		Vector<T>::size_type size() const { return m_data.size(); }
+		Vector<T>::size_type capacity() const { return m_data.capacity(); }
+
+	private:
+		Comp m_comp;
+		Vector<T> m_data;
+	};
+
+}

BAN/include/BAN/RefPtr.h

@@ -2,9 +2,9 @@
 
 #include <BAN/Atomic.h>
 #include <BAN/Errors.h>
+#include <BAN/Hash.h>
 #include <BAN/Move.h>
 #include <BAN/NoCopyMove.h>
-#include <stdint.h>
 
 namespace BAN
 {
@@ -129,14 +129,9 @@ namespace BAN
 			return *this;
 		}
 
-		T* ptr() { ASSERT(!empty()); return m_pointer; }
-		const T* ptr() const { ASSERT(!empty()); return m_pointer; }
-
-		T& operator*() { return *ptr(); }
-		const T& operator*() const { return *ptr(); }
-
-		T* operator->() { return ptr(); }
-		const T* operator->() const { return ptr(); }
+		T* ptr() const { return m_pointer; }
+		T& operator*() const { ASSERT(!empty()); return *ptr(); }
+		T* operator->() const { ASSERT(!empty()); return ptr(); }
 
 		bool operator==(RefPtr other) const { return m_pointer == other.m_pointer; }
 		bool operator!=(RefPtr other) const { return m_pointer != other.m_pointer; }
@@ -158,4 +153,13 @@ namespace BAN
 		friend class RefPtr;
 	};
 
+	template<typename T>
+	struct hash<RefPtr<T>>
+	{
+		constexpr hash_t operator()(const RefPtr<T>& ptr) const
+		{
+			return hash<T*>()(ptr.ptr());
+		}
+	};
+
 }

BAN/include/BAN/Sort.h

@@ -1,5 +1,6 @@
 #pragma once
 
+#include <BAN/Heap.h>
 #include <BAN/Math.h>
 #include <BAN/Swap.h>
 #include <BAN/Traits.h>
@@ -8,7 +9,7 @@
 namespace BAN::sort
 {
 
-	template<typename It, typename Comp = less<typename It::value_type>>
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
 	void exchange_sort(It begin, It end, Comp comp = {})
 	{
 		for (It lhs = begin; lhs != end; ++lhs)
@@ -42,7 +43,7 @@ namespace BAN::sort
 
 	}
 
-	template<typename It, typename Comp = less<typename It::value_type>>
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
 	void quick_sort(It begin, It end, Comp comp = {})
 	{
 		if (distance(begin, end) <= 1)
@@ -52,14 +53,14 @@ namespace BAN::sort
 		quick_sort(++mid, end, comp);
 	}
 
-	template<typename It, typename Comp = less<typename It::value_type>>
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
 	void insertion_sort(It begin, It end, Comp comp = {})
 	{
 		if (distance(begin, end) <= 1)
 			return;
 		for (It it1 = next(begin, 1); it1 != end; ++it1)
 		{
-			typename It::value_type x = move(*it1);
+			auto x = move(*it1);
 			It it2 = it1;
 			for (; it2 != begin && comp(x, *prev(it2, 1)); --it2)
 				*it2 = move(*prev(it2, 1));
@@ -67,83 +68,7 @@ namespace BAN::sort
 		}
 	}
 
-	namespace detail
-	{
-
-		template<typename It, typename Comp>
-		void push_heap(It begin, size_t hole_index, size_t top_index, typename It::value_type value, Comp comp)
-		{
-			size_t parent = (hole_index - 1) / 2;
-			while (hole_index > top_index && comp(*next(begin, parent), value))
-			{
-				*next(begin, hole_index) = move(*next(begin, parent));
-				hole_index = parent;
-				parent = (hole_index - 1) / 2;
-			}
-			*next(begin, hole_index) = move(value);
-		}
-
-		template<typename It, typename Comp>
-		void adjust_heap(It begin, size_t hole_index, size_t len, typename It::value_type value, Comp comp)
-		{
-			const size_t top_index = hole_index;
-			size_t child = hole_index;
-			while (child < (len - 1) / 2)
-			{
-				child = 2 * (child + 1);
-				if (comp(*next(begin, child), *next(begin, child - 1)))
-					child--;
-				*next(begin, hole_index) = move(*next(begin, child));
-				hole_index = child;
-			}
-			if (len % 2 == 0 && child == (len - 2) / 2)
-			{
-				child = 2 * (child + 1);
-				*next(begin, hole_index) = move(*next(begin, child - 1));
-				hole_index = child - 1;
-			}
-			push_heap(begin, hole_index, top_index, move(value), comp);
-		}
-
-	}
-
-	template<typename It, typename Comp = less<typename It::value_type>>
-	void make_heap(It begin, It end, Comp comp = {})
-	{
-		const size_t len = distance(begin, end);
-		if (len <= 1)
-			return;
-
-		size_t parent = (len - 2) / 2;
-		while (true)
-		{
-			detail::adjust_heap(begin, parent, len, move(*next(begin, parent)), comp);
-
-			if (parent == 0)
-				break;
-
-			parent--;
-		}
-	}
-
-	template<typename It, typename Comp = less<typename It::value_type>>
-	void sort_heap(It begin, It end, Comp comp = {})
-	{
-		const size_t len = distance(begin, end);
-		if (len <= 1)
-			return;
-
-		size_t last = len;
-		while (last > 1)
-		{
-			last--;
-			typename It::value_type x = move(*next(begin, last));
-			*next(begin, last) = move(*begin);
-			detail::adjust_heap(begin, 0, last, move(x), comp);
-		}
-	}
-
-	template<typename It, typename Comp = less<typename It::value_type>>
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
 	void heap_sort(It begin, It end, Comp comp = {})
 	{
 		make_heap(begin, end, comp);
@@ -167,7 +92,7 @@ namespace BAN::sort
 
 	}
 
-	template<typename It, typename Comp = less<typename It::value_type>>
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
 	void intro_sort(It begin, It end, Comp comp = {})
 	{
 		const size_t len = distance(begin, end);
@@ -190,10 +115,10 @@ namespace BAN::sort
 	}
 
 	template<typename It, size_t radix = 256>
-	requires is_unsigned_v<typename It::value_type> && (radix > 0 && (radix & (radix - 1)) == 0)
+	requires is_unsigned_v<it_value_type_t<It>> && (radix > 0 && (radix & (radix - 1)) == 0)
 	BAN::ErrorOr<void> radix_sort(It begin, It end)
 	{
-		using value_type = typename It::value_type;
+		using value_type = it_value_type_t<It>;
 
 		const size_t len = distance(begin, end);
 		if (len <= 1)
@@ -231,7 +156,7 @@ namespace BAN::sort
 		return {};
 	}
 
-	template<typename It, typename Comp = less<typename It::value_type>>
+	template<typename It, typename Comp = less<it_value_type_t<It>>>
 	void sort(It begin, It end, Comp comp = {})
 	{
 		return intro_sort(begin, end, comp);

BAN/include/BAN/Span.h

@@ -69,7 +69,6 @@ namespace BAN
 
 		value_type* data() const
 		{
-			ASSERT(m_data);
 			return m_data;
 		}
 
@@ -84,7 +83,6 @@ namespace BAN
 
 		Span slice(size_type start, size_type length = ~size_type(0)) const
 		{
-			ASSERT(m_data);
 			ASSERT(start <= m_size);
 			if (length == ~size_type(0))
 				length = m_size - start;

BAN/include/BAN/String.h

@@ -14,6 +14,7 @@ namespace BAN
 	{
 	public:
 		using size_type = size_t;
+		using value_type = char;
 		using iterator = IteratorSimple<char, String>;
 		using const_iterator = ConstIteratorSimple<char, String>;
 		static constexpr size_type sso_capacity = 15;
@@ -352,10 +353,9 @@ namespace BAN::Formatter
 {
 
 	template<typename F>
-	void print_argument(F putc, const String& string, const ValueFormat&)
+	void print_argument(F putc, const String& string, const ValueFormat& format)
 	{
-		for (String::size_type i = 0; i < string.size(); i++)
-			putc(string[i]);
+		print_argument(putc, string.sv(), format);
 	}
 
 }

BAN/include/BAN/StringView.h

@@ -14,6 +14,7 @@ namespace BAN
 	{
 	public:
 		using size_type = size_t;
+		using value_type = char;
 		using const_iterator = ConstIteratorSimple<char, StringView>;
 
 	public:
@@ -21,7 +22,8 @@ namespace BAN
 		constexpr StringView(const char* string, size_type len = -1)
 		{
 			if (len == size_type(-1))
-				len = strlen(string);
+				for (len = 0; string[len];)
+					len++;
 			m_data = string;
 			m_size = len;
 		}
@@ -245,10 +247,12 @@ namespace BAN::Formatter
 {
 
 	template<typename F>
-	void print_argument(F putc, const StringView& sv, const ValueFormat&)
+	void print_argument(F putc, const StringView& sv, const ValueFormat& format)
 	{
 		for (StringView::size_type i = 0; i < sv.size(); i++)
 			putc(sv[i]);
+		for (int i = sv.size(); i < format.fill; i++)
+			putc(' ');
 	}
 
 }

BAN/include/BAN/Traits.h

@@ -61,6 +61,9 @@ namespace BAN
 	template<typename T> struct is_move_constructible { static constexpr bool value = is_constructible_v<T, T&&>; };
 	template<typename T> inline constexpr bool is_move_constructible_v = is_move_constructible<T>::value;
 
+	template<typename T> struct is_trivially_copyable { static constexpr bool value = __is_trivially_copyable(T); };
+	template<typename T> inline constexpr bool is_trivially_copyable_v = is_trivially_copyable<T>::value;
+
 	template<typename T> struct is_integral { static constexpr bool value = requires (T t, T* p, void (*f)(T)) { reinterpret_cast<T>(t); f(0); p + t; }; };
 	template<typename T> inline constexpr bool is_integral_v = is_integral<T>::value;
 	template<typename T> concept integral = is_integral_v<T>;
@@ -139,6 +142,10 @@ namespace BAN
 	template<typename T> using make_signed_t = typename make_signed<T>::type;
 #undef __BAN_TRAITS_MAKE_SIGNED_CV
 
+	template<typename T> struct it_value_type { using value_type = T::value_type; };
+	template<typename T> struct it_value_type<T*> { using value_type = T; };
+	template<typename T> using it_value_type_t = typename it_value_type<T>::value_type;
+
 	template<typename T> struct less	{ constexpr bool operator()(const T& lhs, const T& rhs) const { return lhs < rhs; } };
 	template<typename T> struct equal	{ constexpr bool operator()(const T& lhs, const T& rhs) const { return lhs == rhs; } };
 	template<typename T> struct greater	{ constexpr bool operator()(const T& lhs, const T& rhs) const { return lhs > rhs; } };

BAN/include/BAN/UTF8.h

@@ -18,7 +18,7 @@ namespace BAN::UTF8
 			return 3;
 		if ((first_byte & 0xF8) == 0xF0)
 			return 4;
-		return 0;
+		return UTF8::invalid;
 	}
 
 	template<typename T> requires (sizeof(T) == 1)
@@ -76,6 +76,8 @@ namespace BAN::UTF8
 			}
 		}
 
+		*ptr = '\0';
+
 		return true;
 	}
 

BAN/include/BAN/UniqPtr.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <BAN/Errors.h>
+#include <BAN/Hash.h>
 #include <BAN/NoCopyMove.h>
 
 namespace BAN
@@ -53,32 +54,12 @@ namespace BAN
 			return *this;
 		}
 
-		T& operator*()
-		{
-			ASSERT(m_pointer);
-			return *m_pointer;
-		}
+		T* ptr() const { return m_pointer; }
+		T& operator*() const { ASSERT(!empty()); return *ptr(); }
+		T* operator->() const { ASSERT(!empty()); return ptr(); }
 
-		const T& operator*() const
-		{
-			ASSERT(m_pointer);
-			return *m_pointer;
-		}
-
-		T* operator->()
-		{
-			ASSERT(m_pointer);
-			return m_pointer;
-		}
-
-		const T* operator->() const
-		{
-			ASSERT(m_pointer);
-			return m_pointer;
-		}
-
-		T* ptr() { return m_pointer; }
-		const T* ptr() const { return m_pointer; }
+		bool empty() const { return m_pointer == nullptr; }
+		explicit operator bool() const { return m_pointer; }
 
 		void clear()
 		{
@@ -87,8 +68,6 @@ namespace BAN
 			m_pointer = nullptr;
 		}
 
-		operator bool() const { return m_pointer != nullptr; }
-
 	private:
 		T* m_pointer = nullptr;
 
@@ -96,4 +75,13 @@ namespace BAN
 		friend class UniqPtr;
 	};
 
+	template<typename T>
+	struct hash<UniqPtr<T>>
+	{
+		constexpr hash_t operator()(const UniqPtr<T>& ptr) const
+		{
+			return hash<T*>()(ptr.ptr());
+		}
+	};
+
 }

BAN/include/BAN/Variant.h

@@ -126,14 +126,16 @@ namespace BAN
 		Variant(Variant&& other)
 			: m_index(other.m_index)
 		{
-			detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+			if (other.has_value())
+				detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 			other.clear();
 		}
 
 		Variant(const Variant& other)
 			: m_index(other.m_index)
 		{
-			detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+			if (other.has_value())
+				detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 		}
 
 		template<typename T>
@@ -157,12 +159,13 @@ namespace BAN
 
 		Variant& operator=(Variant&& other)
 		{
-			if (m_index == other.m_index)
+			if (m_index == other.m_index && m_index != invalid_index())
 				detail::move_assign<Ts...>(m_index, other.m_storage, m_storage);
 			else
 			{
 				clear();
-				detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+				if (other.has_value())
+					detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 				m_index = other.m_index;
 			}
 			other.clear();
@@ -171,12 +174,13 @@ namespace BAN
 
 		Variant& operator=(const Variant& other)
 		{
-			if (m_index == other.m_index)
+			if (m_index == other.m_index && m_index != invalid_index())
 				detail::copy_assign<Ts...>(m_index, other.m_storage, m_storage);
 			else
 			{
 				clear();
-				detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+				if (other.has_value())
+					detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 				m_index = other.m_index;
 			}
 			return *this;

BAN/include/BAN/Vector.h

@@ -56,7 +56,7 @@ namespace BAN
 		bool contains(const T&) const;
 
 		Span<T> span() { return Span(m_data, m_size); }
-		const Span<T> span() const { return Span(m_data, m_size); }
+		Span<const T> span() const { return Span(m_data, m_size); }
 
 		const T& operator[](size_type) const;
 		T& operator[](size_type);
@@ -381,19 +381,46 @@ namespace BAN
 	template<typename T>
 	ErrorOr<void> Vector<T>::ensure_capacity(size_type size)
 	{
+		static_assert(alignof(T) <= alignof(max_align_t), "over aligned types not supported");
+
 		if (m_capacity >= size)
 			return {};
-		size_type new_cap = BAN::Math::max<size_type>(size, m_capacity * 2);
-		T* new_data = (T*)BAN::allocator(new_cap * sizeof(T));
-		if (new_data == nullptr)
-			return Error::from_errno(ENOMEM);
-		for (size_type i = 0; i < m_size; i++)
+
+		const size_type new_cap = BAN::Math::max<size_type>(size, m_capacity * 2);
+
+		if constexpr (BAN::is_trivially_copyable_v<T>)
 		{
-			new (new_data + i) T(move(m_data[i]));
-			m_data[i].~T();
+			if constexpr (BAN::reallocator)
+			{
+				auto* new_data = static_cast<T*>(BAN::reallocator(m_data, new_cap * sizeof(T)));
+				if (new_data == nullptr)
+					return Error::from_errno(ENOMEM);
+				m_data = new_data;
+			}
+			else
+			{
+				auto* new_data = static_cast<T*>(BAN::allocator(new_cap * sizeof(T)));
+				if (new_data == nullptr)
+					return Error::from_errno(ENOMEM);
+				memcpy(new_data, m_data, m_size * sizeof(T));
+				BAN::deallocator(m_data);
+				m_data = new_data;
+			}
 		}
-		BAN::deallocator(m_data);
-		m_data = new_data;
+		else
+		{
+			auto* new_data = static_cast<T*>(BAN::allocator(new_cap * sizeof(T)));
+			if (new_data == nullptr)
+				return Error::from_errno(ENOMEM);
+			for (size_type i = 0; i < m_size; i++)
+			{
+				new (new_data + i) T(move(m_data[i]));
+				m_data[i].~T();
+			}
+			BAN::deallocator(m_data);
+			m_data = new_data;
+		}
+
 		m_capacity = new_cap;
 		return {};
 	}

BAN/include/BAN/WeakPtr.h

@@ -20,7 +20,7 @@ namespace BAN
 	class WeakLink : public RefCounted<WeakLink<T>>
 	{
 	public:
-		RefPtr<T> try_lock()
+		RefPtr<T> try_lock() const
 		{
 #if __is_kernel
 			Kernel::SpinLockGuard _(m_weak_lock);
@@ -44,7 +44,7 @@ namespace BAN
 	private:
 		T* m_ptr;
 #if __is_kernel
-		Kernel::SpinLock m_weak_lock;
+		mutable Kernel::SpinLock m_weak_lock;
 #endif
 		friend class RefPtr<WeakLink<T>>;
 	};
@@ -99,7 +99,7 @@ namespace BAN
 			return *this;
 		}
 
-		RefPtr<T> lock()
+		RefPtr<T> lock() const
 		{
 			if (m_link)
 				return m_link->try_lock();

CMakeLists.txt

@@ -27,17 +27,14 @@ set(BUILD_SHARED_LIBS True)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}")
-
 # include headers of ${library} to ${target}
 function(banan_include_headers target library)
-	target_include_directories(${target} PRIVATE $<TARGET_PROPERTY:${library},SOURCE_DIR>/include)
+	target_include_directories(${target} PUBLIC $<TARGET_PROPERTY:${library},SOURCE_DIR>/include)
 endfunction()
 
 # include headers and link ${library} to ${target}
 function(banan_link_library target library)
-	target_link_libraries(${target} PRIVATE ${library})
+	target_link_libraries(${target} PUBLIC ${library})
 	banan_include_headers(${target} ${library})
 endfunction()
 

README.md

@@ -23,7 +23,7 @@ If you want to try out DOOM, you should first enter the GUI environment using th
 - [x] Basic graphical environment
   - [x] Terminal emulator
   - [x] Status bar
-  - [ ] Program launcher
+  - [x] Program launcher
   - [ ] Some nice apps
 - [x] ELF dynamic linking
 - [x] copy-on-write memory
@@ -38,10 +38,14 @@ If you want to try out DOOM, you should first enter the GUI environment using th
 - [x] PS2 keyboard (all scancode sets)
 - [x] PS2 mouse
 - [x] USB
+  - [x] xHCI
+  - [ ] EHCI
+  - [ ] OHCI
+  - [ ] UHCI
   - [x] Keyboard
   - [x] Mouse
   - [x] Mass storage
-  - [ ] Hubs
+  - [x] Hubs
   - [ ] ...
 - [ ] virtio devices (network, storage)
 
@@ -114,6 +118,8 @@ To change the bootloader you can set environment variable BANAN\_BOOTLOADER; sup
 
 To run with UEFI set environment variable BANAN\_UEFI\_BOOT=1. You will also have to set OVMF\_PATH to the correct OVMF (default */usr/share/ovmf/x64/OVMF.fd*).
 
+To build an image with no physical root filesystem, but an initrd set environment variable BANAN\_INITRD=1. This can be used when testing on hardware with unsupported USB controller.
+
 If you have corrupted your disk image or want to create new one, you can either manually delete *build/banan-os.img* and build system will automatically create you a new one or you can run the following command.
 ```sh
 ./bos image-full

bootloader/bios/utils.S

@@ -306,20 +306,8 @@ memset32:
 	movw $GDT_DATA32, %dx
 	movw %dx, %es
 
-	movl %ecx, %edx
-
-	andl $3, %ecx
 	rep stosb %es:(%edi)
 
-	movl %edx, %ecx
-	shrl $2, %ecx
-
-	movb %al, %ah
-	movw %ax, %dx
-	shll $16, %eax
-	movw %dx, %ax
-	rep stosl %es:(%edi)
-
 	ljmpl $GDT_CODE16, $.memset32_pmode16
 
  .code16
@@ -370,14 +358,8 @@ memcpy32:
 	movw %dx, %ds
 	movw %dx, %es
 
-	movl %ecx, %edx
-	andl $3, %ecx
 	rep movsb %ds:(%esi), %es:(%edi)
 
-	movl %edx, %ecx
-	shrl $2, %ecx
-	rep movsl %ds:(%esi), %es:(%edi)
-
 	ljmpl $GDT_CODE16, $.memcpy32_pmode16
 
  .code16

kernel/CMakeLists.txt

@@ -1,11 +1,16 @@
 set(KERNEL_SOURCES
 	font/prefs.psf.o
 	kernel/ACPI/ACPI.cpp
-	kernel/ACPI/BatterySystem.cpp
 	kernel/ACPI/AML/Namespace.cpp
 	kernel/ACPI/AML/Node.cpp
 	kernel/ACPI/AML/OpRegion.cpp
+	kernel/ACPI/BatterySystem.cpp
+	kernel/ACPI/EmbeddedController.cpp
 	kernel/APIC.cpp
+	kernel/Audio/AC97/Controller.cpp
+	kernel/Audio/Controller.cpp
+	kernel/Audio/HDAudio/AudioFunctionGroup.cpp
+	kernel/Audio/HDAudio/Controller.cpp
 	kernel/BootInfo.cpp
 	kernel/CPUID.cpp
 	kernel/Credentials.cpp
@@ -17,8 +22,10 @@ set(KERNEL_SOURCES
 	kernel/Device/RandomDevice.cpp
 	kernel/Device/ZeroDevice.cpp
 	kernel/ELF.cpp
+	kernel/Epoll.cpp
 	kernel/Errors.cpp
 	kernel/FS/DevFS/FileSystem.cpp
+	kernel/FS/EventFD.cpp
 	kernel/FS/Ext2/FileSystem.cpp
 	kernel/FS/Ext2/Inode.cpp
 	kernel/FS/FAT/FileSystem.cpp
@@ -30,6 +37,7 @@ set(KERNEL_SOURCES
 	kernel/FS/ProcFS/Inode.cpp
 	kernel/FS/TmpFS/FileSystem.cpp
 	kernel/FS/TmpFS/Inode.cpp
+	kernel/FS/USTARModule.cpp
 	kernel/FS/VirtualFileSystem.cpp
 	kernel/GDT.cpp
 	kernel/IDT.cpp
@@ -42,6 +50,8 @@ set(KERNEL_SOURCES
 	kernel/Interruptable.cpp
 	kernel/InterruptController.cpp
 	kernel/kernel.cpp
+	kernel/Lock/SpinLock.cpp
+	kernel/Memory/ByteRingBuffer.cpp
 	kernel/Memory/DMARegion.cpp
 	kernel/Memory/FileBackedRegion.cpp
 	kernel/Memory/Heap.cpp
@@ -55,6 +65,7 @@ set(KERNEL_SOURCES
 	kernel/Networking/E1000/E1000.cpp
 	kernel/Networking/E1000/E1000E.cpp
 	kernel/Networking/IPv4Layer.cpp
+	kernel/Networking/Loopback.cpp
 	kernel/Networking/NetworkInterface.cpp
 	kernel/Networking/NetworkLayer.cpp
 	kernel/Networking/NetworkManager.cpp
@@ -71,7 +82,6 @@ set(KERNEL_SOURCES
 	kernel/Processor.cpp
 	kernel/Random.cpp
 	kernel/Scheduler.cpp
-	kernel/ThreadBlocker.cpp
 	kernel/SSP.cpp
 	kernel/Storage/ATA/AHCI/Controller.cpp
 	kernel/Storage/ATA/AHCI/Device.cpp
@@ -89,17 +99,23 @@ set(KERNEL_SOURCES
 	kernel/Terminal/FramebufferTerminal.cpp
 	kernel/Terminal/PseudoTerminal.cpp
 	kernel/Terminal/Serial.cpp
+	kernel/Terminal/TerminalDriver.cpp
+	kernel/Terminal/TextModeTerminal.cpp
 	kernel/Terminal/TTY.cpp
 	kernel/Terminal/VirtualTTY.cpp
 	kernel/Thread.cpp
+	kernel/ThreadBlocker.cpp
 	kernel/Timer/HPET.cpp
 	kernel/Timer/PIT.cpp
 	kernel/Timer/RTC.cpp
 	kernel/Timer/Timer.cpp
+	kernel/USB/Controller.cpp
 	kernel/USB/Device.cpp
 	kernel/USB/HID/HIDDriver.cpp
+	kernel/USB/HID/Joystick.cpp
 	kernel/USB/HID/Keyboard.cpp
 	kernel/USB/HID/Mouse.cpp
+	kernel/USB/Hub/HubDriver.cpp
 	kernel/USB/MassStorage/MassStorageDriver.cpp
 	kernel/USB/MassStorage/SCSIDevice.cpp
 	kernel/USB/USBManager.cpp
@@ -123,6 +139,8 @@ if("${BANAN_ARCH}" STREQUAL "x86_64")
 		arch/x86_64/Signal.S
 		arch/x86_64/Syscall.S
 		arch/x86_64/Thread.S
+		arch/x86_64/User.S
+		arch/x86_64/Yield.S
 	)
 elseif("${BANAN_ARCH}" STREQUAL "i686")
 	set(KERNEL_SOURCES
@@ -133,6 +151,8 @@ elseif("${BANAN_ARCH}" STREQUAL "i686")
 		arch/i686/Signal.S
 		arch/i686/Syscall.S
 		arch/i686/Thread.S
+		arch/i686/User.S
+		arch/i686/Yield.S
 	)
 else()
 	message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}")
@@ -148,6 +168,13 @@ set(BAN_SOURCES
 set(KLIBC_SOURCES
 	klibc/ctype.cpp
 	klibc/string.cpp
+	klibc/arch/${BANAN_ARCH}/string.S
+)
+
+set(LIBDEFLATE_SOURCE
+	../userspace/libraries/LibDEFLATE/Compressor.cpp
+	../userspace/libraries/LibDEFLATE/Decompressor.cpp
+	../userspace/libraries/LibDEFLATE/HuffmanTree.cpp
 )
 
 set(LIBFONT_SOURCES
@@ -160,18 +187,25 @@ set(LIBINPUT_SOURCE
 	../userspace/libraries/LibInput/KeyEvent.cpp
 )
 
+set(LIBQR_SOURCE
+	../userspace/libraries/LibQR/QRCode.cpp
+)
+
 set(KERNEL_SOURCES
 	${KERNEL_SOURCES}
 	${BAN_SOURCES}
 	${KLIBC_SOURCES}
+	${LIBDEFLATE_SOURCE}
 	${LIBFONT_SOURCES}
 	${LIBINPUT_SOURCE}
+	${LIBQR_SOURCE}
 )
 
 add_executable(kernel ${KERNEL_SOURCES})
 
 target_compile_definitions(kernel PRIVATE __is_kernel)
 target_compile_definitions(kernel PRIVATE __arch=${BANAN_ARCH})
+target_compile_definitions(kernel PRIVATE LIBDEFLATE_AVOID_STACK=1)
 
 target_compile_options(kernel PRIVATE
 	-O2 -g
@@ -225,9 +259,11 @@ add_custom_command(
 
 banan_include_headers(kernel ban)
 banan_include_headers(kernel libc)
-banan_include_headers(kernel libfont)
+banan_include_headers(kernel libdeflate)
 banan_include_headers(kernel libelf)
+banan_include_headers(kernel libfont)
 banan_include_headers(kernel libinput)
+banan_include_headers(kernel libqr)
 
 banan_install_headers(kernel)
 set_target_properties(kernel PROPERTIES OUTPUT_NAME banan-os.kernel)
@@ -241,7 +277,7 @@ endif()
 
 add_custom_command(
 	OUTPUT font/prefs.psf.o
-	COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && objcopy -O ${ELF_FORMAT} -B i386 -I binary font/prefs.psf ${CMAKE_CURRENT_BINARY_DIR}/font/prefs.psf.o
+	COMMAND cd ${CMAKE_CURRENT_SOURCE_DIR} && mkdir -p ${CMAKE_CURRENT_BINARY_DIR}/font && objcopy -O ${ELF_FORMAT} -B i386 -I binary font/prefs.psf ${CMAKE_CURRENT_BINARY_DIR}/font/prefs.psf.o
 )
 
 set(CMAKE_CXX_LINK_EXECUTABLE "${CMAKE_CXX_COMPILER} <CMAKE_CXX_LINK_FLAGS> <FLAGS> <LINK_FLAGS> -o <TARGET> ${CMAKE_CURRENT_BINARY_DIR}/crti.o ${CMAKE_CURRENT_BINARY_DIR}/crtbegin.o <OBJECTS> ${CMAKE_CURRENT_BINARY_DIR}/crtend.o ${CMAKE_CURRENT_BINARY_DIR}/crtn.o -lgcc ")

kernel/arch/i686/PageTable.cpp

@@ -16,11 +16,18 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
 	SpinLock PageTable::s_fast_page_lock;
 
+	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
+	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
+
+	static bool s_is_initialized = false;
+
 	static PageTable* s_kernel = nullptr;
 	static bool s_has_nxe = false;
 	static bool s_has_pge = false;
@@ -28,6 +35,28 @@ namespace Kernel
 
 	static paddr_t s_global_pdpte = 0;
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
+	static uint64_t* allocate_zeroed_page_aligned_page()
+	{
+		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
+		ASSERT(page);
+		memset(page, 0, PAGE_SIZE);
+		return (uint64_t*)page;
+	}
+
+	template<typename T>
+	static paddr_t V2P(const T vaddr)
+	{
+		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+	}
+
+	template<typename T>
+	static uint64_t* P2V(const T paddr)
+	{
+		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
+	}
+
 	static inline PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
@@ -46,31 +75,22 @@ namespace Kernel
 		return result;
 	}
 
-	void PageTable::initialize_pre_heap()
+	void PageTable::initialize_fast_page()
+	{
+		s_fast_page_pt = g_boot_fast_page_pt;
+	}
+
+	static void detect_cpu_features()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_pat())
 			s_has_pat = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->initialize_kernel();
-		s_kernel->initial_load();
 	}
 
-	void PageTable::initialize_post_heap()
-	{
-		// NOTE: this is no-op as our 32 bit target does not use hhdm
-	}
-
-	void PageTable::initial_load()
+	void PageTable::enable_cpu_features()
 	{
 		if (s_has_nxe)
 		{
@@ -111,8 +131,56 @@ namespace Kernel
 			"movl %%eax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel);
+
+		auto* pdpt = allocate_zeroed_page_aligned_page();
+		ASSERT(pdpt);
+
+		s_kernel->m_highest_paging_struct = V2P(pdpt);
+		s_kernel->map_kernel_memory();
+
+		PageTable::with_fast_page(s_kernel->m_highest_paging_struct, [] {
+			s_global_pdpte = PageTable::fast_page_as_sized<paddr_t>(3);
+		});
+
+		// update fast page pt
+		{
+			constexpr vaddr_t vaddr = fast_page();
+			constexpr uint16_t pdpte = (vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						auto* vaddr = allocate_zeroed_page_aligned_page();
+						ASSERT(vaddr);
+						table[entry] = V2P(vaddr);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pdpt = s_kernel->m_highest_paging_struct;
+			const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -126,40 +194,12 @@ namespace Kernel
 		return true;
 	}
 
-	static uint64_t* allocate_zeroed_page_aligned_page()
+	void PageTable::map_kernel_memory()
 	{
-		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-		ASSERT(page);
-		memset(page, 0, PAGE_SIZE);
-		return (uint64_t*)page;
-	}
-
-	template<typename T>
-	static paddr_t V2P(const T vaddr)
-	{
-		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-	}
-
-	template<typename T>
-	static vaddr_t P2V(const T paddr)
-	{
-		return (paddr_t)paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET;
-	}
-
-	void PageTable::initialize_kernel()
-	{
-		ASSERT(s_global_pdpte == 0);
-		s_global_pdpte = V2P(allocate_zeroed_page_aligned_page());
-
-		map_kernel_memory();
-
-		prepare_fast_page();
-
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
-		ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0);
 		map_range_at(
 			V2P(g_kernel_start),
-			(vaddr_t)g_kernel_start,
+			reinterpret_cast<vaddr_t>(g_kernel_start),
 			g_kernel_end - g_kernel_start,
 			Flags::Present
 		);
@@ -167,7 +207,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		map_range_at(
 			V2P(g_kernel_execute_start),
-			(vaddr_t)g_kernel_execute_start,
+			reinterpret_cast<vaddr_t>(g_kernel_execute_start),
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
 		);
@@ -175,7 +215,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		map_range_at(
 			V2P(g_kernel_writable_start),
-			(vaddr_t)g_kernel_writable_start,
+			reinterpret_cast<vaddr_t>(g_kernel_writable_start),
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
 		);
@@ -183,64 +223,34 @@ namespace Kernel
 		// Map userspace memory
 		map_range_at(
 			V2P(g_userspace_start),
-			(vaddr_t)g_userspace_start,
+			reinterpret_cast<vaddr_t>(g_userspace_start),
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
-		ASSERT(pdpt[pdpte] & Flags::Present);
-
-		uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte]) & PAGE_ADDR_MASK);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
-		ASSERT(paddr);
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
 
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
+		ASSERT(!(*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(s_kernel->m_highest_paging_struct));
-		uint64_t* pd   = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
-		uint64_t* pt   = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
-
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
+		ASSERT((*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = 0;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(s_kernel->m_highest_paging_struct));
-		uint64_t* pd   = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
-		uint64_t* pt   = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
-
-		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = 0;
-
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -249,25 +259,23 @@ namespace Kernel
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
-		return page_table;
-	}
 
-	void PageTable::map_kernel_memory()
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pdpte);
+		uint64_t* pdpt = allocate_zeroed_page_aligned_page();
+		if (pdpt == nullptr)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
+		}
 
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = V2P(kmalloc(32, 32, true));
-		ASSERT(m_highest_paging_struct);
+		page_table->m_highest_paging_struct = V2P(pdpt);
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
 		pdpt[0] = 0;
 		pdpt[1] = 0;
 		pdpt[2] = 0;
 		pdpt[3] = s_global_pdpte | Flags::Present;
 		static_assert(KERNEL_OFFSET == 0xC0000000);
+
+		return page_table;
 	}
 
 	PageTable::~PageTable()
@@ -275,18 +283,17 @@ namespace Kernel
 		if (m_highest_paging_struct == 0)
 			return;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
-
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
 		for (uint32_t pdpte = 0; pdpte < 3; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+			uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 			for (uint32_t pde = 0; pde < 512; pde++)
 			{
 				if (!(pd[pde] & Flags::Present))
 					continue;
-				kfree(reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK)));
+				kfree(P2V(pd[pde] & s_page_addr_mask));
 			}
 			kfree(pd);
 		}
@@ -297,15 +304,43 @@ namespace Kernel
 	{
 		SpinLockGuard _(m_lock);
 		ASSERT(m_highest_paging_struct < 0x100000000);
-		const uint32_t pdpt_lo = m_highest_paging_struct;
-		asm volatile("movl %0, %%cr3" :: "r"(pdpt_lo));
+		asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
 		Processor::set_current_page_table(this);
 	}
 
-	void PageTable::invalidate(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::invalidate_range(vaddr_t vaddr, size_t pages, bool send_smp_message)
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
-		asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory");
+
+		const bool is_userspace = (vaddr < KERNEL_OFFSET);
+		if (is_userspace && this != &PageTable::current())
+			;
+		else if (pages <= 32 || !s_is_initialized)
+		{
+			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
+				asm volatile("invlpg (%0)" :: "r"(vaddr));
+		}
+		else if (is_userspace || !s_has_pge)
+		{
+			asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
+		}
+		else
+		{
+			asm volatile(
+				"movl %%cr4, %%eax;"
+
+				"andl $~0x80, %%eax;"
+				"movl %%eax, %%cr4;"
+
+				"movl %0, %%cr3;"
+
+				"orl $0x80, %%eax;"
+				"movl %%eax, %%cr4;"
+				:
+				: "r"(static_cast<uint32_t>(m_highest_paging_struct))
+				: "eax"
+			);
+		}
 
 		if (send_smp_message)
 		{
@@ -313,13 +348,14 @@ namespace Kernel
 				.type = Processor::SMPMessage::Type::FlushTLB,
 				.flush_tlb = {
 					.vaddr      = vaddr,
-					.page_count = 1
+					.page_count = pages,
+					.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
 				}
 			});
 		}
 	}
 
-	void PageTable::unmap_page(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr % PAGE_SIZE == 0);
@@ -338,12 +374,16 @@ namespace Kernel
 		if (is_page_free(vaddr))
 			Kernel::panic("trying to unmap unmapped page 0x{H}", vaddr);
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
-		uint64_t* pd   = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
-		uint64_t* pt   = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
+		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
+		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
+
+		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
 
 		pt[pte] = 0;
-		invalidate(vaddr, send_smp_message);
+
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::unmap_range(vaddr_t vaddr, size_t size)
@@ -355,17 +395,10 @@ namespace Kernel
 		SpinLockGuard _(m_lock);
 		for (vaddr_t page = 0; page < page_count; page++)
 			unmap_page(vaddr + page * PAGE_SIZE, false);
-
-		Processor::broadcast_smp_message({
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = page_count
-			}
-		});
+		invalidate_range(vaddr, page_count, true);
 	}
 
-	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool send_smp_message)
+	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -400,11 +433,11 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
 		if (!(pdpt[pdpte] & Flags::Present))
 			pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present;
 
-		uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 		if ((pd[pde] & uwr_flags) != uwr_flags)
 		{
 			if (!(pd[pde] & Flags::Present))
@@ -415,10 +448,14 @@ namespace Kernel
 		if (!(flags & Flags::Present))
 			uwr_flags &= ~Flags::Present;
 
-		uint64_t* pt = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
+		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+
+		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
+
 		pt[pte] = paddr | uwr_flags | extra_flags;
 
-		invalidate(vaddr, send_smp_message);
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
@@ -432,14 +469,49 @@ namespace Kernel
 		SpinLockGuard _(m_lock);
 		for (size_t page = 0; page < page_count; page++)
 			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false);
+		invalidate_range(vaddr, page_count, true);
+	}
 
-		Processor::broadcast_smp_message({
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = page_count
+	void PageTable::remove_writable_from_range(vaddr_t vaddr, size_t size)
+	{
+		ASSERT(vaddr);
+		ASSERT(vaddr % PAGE_SIZE == 0);
+
+		uint32_t pdpte = (vaddr >> 30) & 0x1FF;
+		uint32_t pde   = (vaddr >> 21) & 0x1FF;
+		uint32_t pte   = (vaddr >> 12) & 0x1FF;
+
+		const uint32_t e_pdpte = ((vaddr + size - 1) >> 30) & 0x1FF;
+		const uint32_t e_pde   = ((vaddr + size - 1) >> 21) & 0x1FF;
+		const uint32_t e_pte   = ((vaddr + size - 1) >> 12) & 0x1FF;
+
+		SpinLockGuard _(m_lock);
+
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		for (; pdpte <= e_pdpte; pdpte++)
+		{
+			if (!(pdpt[pdpte] & Flags::Present))
+				continue;
+			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			for (; pde < 512; pde++)
+			{
+				if (pdpte == e_pdpte && pde > e_pde)
+					break;
+				if (!(pd[pde] & Flags::ReadWrite))
+					continue;
+				uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				for (; pte < 512; pte++)
+				{
+					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
+						break;
+					pt[pte] &= ~static_cast<uint64_t>(Flags::ReadWrite);
+				}
+				pte = 0;
 			}
-		});
+			pde = 0;
+		}
+
+		invalidate_range(vaddr, size / PAGE_SIZE, true);
 	}
 
 	uint64_t PageTable::get_page_data(vaddr_t vaddr) const
@@ -452,15 +524,15 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		uint64_t* pdpt = (uint64_t*)P2V(m_highest_paging_struct);
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
 		if (!(pdpt[pdpte] & Flags::Present))
 			return 0;
 
-		uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK);
+		const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 		if (!(pd[pde] & Flags::Present))
 			return 0;
 
-		uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK);
+		const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
 		if (!(pt[pte] & Flags::Used))
 			return 0;
 
@@ -474,8 +546,7 @@ namespace Kernel
 
 	paddr_t PageTable::physical_address_of(vaddr_t vaddr) const
 	{
-		uint64_t page_data = get_page_data(vaddr);
-		return (page_data & PAGE_ADDR_MASK) & ~(1ull << 63);
+		return get_page_data(vaddr) & s_page_addr_mask;
 	}
 
 	bool PageTable::is_page_free(vaddr_t vaddr) const
@@ -496,13 +567,13 @@ namespace Kernel
 		return true;
 	}
 
-	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free)
+	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool send_smp_message)
 	{
 		SpinLockGuard _(m_lock);
 		ASSERT(vaddr % PAGE_SIZE == 0);
 		if (only_free && !is_page_free(vaddr))
 			return false;
-		map_page_at(0, vaddr, Flags::Reserved);
+		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, send_smp_message);
 		return true;
 	}
 
@@ -516,7 +587,9 @@ namespace Kernel
 		if (only_free && !is_range_free(vaddr, bytes))
 			return false;
 		for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
-			reserve_page(vaddr + offset);
+			reserve_page(vaddr + offset, true, false);
+		invalidate_range(vaddr, bytes / PAGE_SIZE, true);
+
 		return true;
 	}
 
@@ -529,48 +602,47 @@ namespace Kernel
 		if (size_t rem = last_address % PAGE_SIZE)
 			last_address -= rem;
 
-		const uint32_t s_pdpte = (first_address >> 30) & 0x1FF;
-		const uint32_t s_pde   = (first_address >> 21) & 0x1FF;
-		const uint32_t s_pte   = (first_address >> 12) & 0x1FF;
+		uint32_t pdpte = (first_address >> 30) & 0x1FF;
+		uint32_t pde   = (first_address >> 21) & 0x1FF;
+		uint32_t pte   = (first_address >> 12) & 0x1FF;
 
-		const uint32_t e_pdpte = (last_address >> 30) & 0x1FF;
-		const uint32_t e_pde   = (last_address >> 21) & 0x1FF;
-		const uint32_t e_pte   = (last_address >> 12) & 0x1FF;
+		const uint32_t e_pdpte = ((last_address - 1) >> 30) & 0x1FF;
+		const uint32_t e_pde   = ((last_address - 1) >> 21) & 0x1FF;
+		const uint32_t e_pte   = ((last_address - 1) >> 12) & 0x1FF;
 
 		SpinLockGuard _(m_lock);
 
 		// Try to find free page that can be mapped without
 		// allocations (page table with unused entries)
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
-		for (uint32_t pdpte = s_pdpte; pdpte < 4; pdpte++)
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		for (; pdpte <= e_pdpte; pdpte++)
 		{
-			if (pdpte > e_pdpte)
-				break;
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
-			for (uint32_t pde = s_pde; pde < 512; pde++)
+			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			for (; pde < 512; pde++)
 			{
 				if (pdpte == e_pdpte && pde > e_pde)
 					break;
 				if (!(pd[pde] & Flags::Present))
 					continue;
-				uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK);
-				for (uint32_t pte = s_pte; pte < 512; pte++)
+				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				for (; pte < 512; pte++)
 				{
-					if (pdpte == e_pdpte && pde == e_pde && pte >= e_pte)
+					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 						break;
-					if (!(pt[pte] & Flags::Used))
-					{
-						vaddr_t vaddr = 0;
-						vaddr |= (vaddr_t)pdpte << 30;
-						vaddr |= (vaddr_t)pde   << 21;
-						vaddr |= (vaddr_t)pte   << 12;
-						ASSERT(reserve_page(vaddr));
-						return vaddr;
-					}
+					if (pt[pte] & Flags::Used)
+						continue;
+					vaddr_t vaddr = 0;
+					vaddr |= (vaddr_t)pdpte << 30;
+					vaddr |= (vaddr_t)pde   << 21;
+					vaddr |= (vaddr_t)pte   << 12;
+					ASSERT(reserve_page(vaddr));
+					return vaddr;
 				}
+				pte = 0;
 			}
+			pde = 0;
 		}
 
 		// Find any free page
@@ -583,7 +655,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	vaddr_t PageTable::reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address)
@@ -616,7 +688,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	static void dump_range(vaddr_t start, vaddr_t end, PageTable::flags_t flags)
@@ -639,7 +711,7 @@ namespace Kernel
 		flags_t flags = 0;
 		vaddr_t start = 0;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
 		for (uint32_t pdpte = 0; pdpte < 4; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
@@ -648,7 +720,7 @@ namespace Kernel
 				start = 0;
 				continue;
 			}
-			uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK);
+			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 			for (uint64_t pde = 0; pde < 512; pde++)
 			{
 				if (!(pd[pde] & Flags::Present))
@@ -657,7 +729,7 @@ namespace Kernel
 					start = 0;
 					continue;
 				}
-				uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK);
+				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
 				for (uint64_t pte = 0; pte < 512; pte++)
 				{
 					if (parse_flags(pt[pte]) != flags)

kernel/arch/i686/Signal.S

@@ -1,35 +1,86 @@
 .section .userspace, "ax"
 
 // stack contains
-//    return address
-//    signal number
-//    signal handler
+//    (4 bytes)  return address (on return stack)
+//    (4 bytes)  return stack
+//    (4 bytes)  return rflags
+//    (8 bytes)  restore sigmask
+//    (36 bytes) siginfo_t
+//    (4 bytes)  signal number
+//    (4 bytes)  signal handler
 
 .global signal_trampoline
 signal_trampoline:
+	pushl %esi // gregs
+	pushl %edi
+	pushl %edx
+	pushl %ecx
+	pushl %ebx
+	pushl %eax
 	pushl %ebp
-	movl %esp, %ebp
 
-	pusha
+	movl 84(%esp), %eax
+	pushl %eax; addl $4, (%esp)
+	pushl (%eax)
 
-	movl 40(%esp), %edi
-	movl 36(%esp), %eax
+	// FIXME: populate these
+	xorl %eax, %eax
+	pushl %eax // stack
+	pushl %eax
+	pushl %eax
+	pushl %eax // sigset
+	pushl %eax
+	pushl %eax // link
+
+	movl    %esp,  %edx // ucontext
+	leal 68(%esp), %esi // siginfo
+	movl 64(%esp), %edi // signal number
+	movl 60(%esp), %eax // handlers
 
 	// align stack to 16 bytes
-	movl %esp, %ebx
-	andl $0x0F, %ebx
-	subl %ebx, %esp
+	movl %esp, %ebp
+	andl $-16, %esp
 
-	subl $12, %esp
+	subl $512, %esp
+	fxsave (%esp)
+
+	subl $4, %esp
+	pushl %edx
+	pushl %esi
 	pushl %edi
 	call *%eax
 	addl $16, %esp
 
-	// restore stack
-	addl %ebx, %esp
-	popa
+	fxrstor (%esp)
+	addl $512, %esp
 
-	leave
+	// restore stack
+	movl %ebp, %esp
+	addl $24, %esp
+
+	// restore sigmask
+	movl $83, %eax      // SYS_SIGPROCMASK
+	movl $3,  %ebx      // SIG_SETMASK
+	leal 72(%esp), %ecx // set
+	xorl %edx, %edx     // oset
+	int $0xF0
+
+	// restore registers
 	addl $8, %esp
+	popl %ebp
+	popl %eax
+	popl %ebx
+	popl %ecx
+	popl %edx
+	popl %edi
+	popl %esi
+
+	// skip handler, number, siginfo_t, sigmask
+	addl $52, %esp
+
+	// restore flags
+	popf
+
+	movl (%esp), %esp
 
 	ret

kernel/arch/i686/Syscall.S

@@ -1,12 +1,6 @@
 // arguments in EAX, EBX, ECX, EDX, ESI, EDI
 .global asm_syscall_handler
 asm_syscall_handler:
-	# save segment registers
-	pushw %ds
-	pushw %es
-	pushw %fs
-	pushw %gs
-
 	# save general purpose registers
 	pushl %ebx
 	pushl %ecx
@@ -14,16 +8,14 @@ asm_syscall_handler:
 	pushl %edi
 	pushl %esi
 	pushl %ebp
+	cld
 
 	# align stack
 	movl %esp, %ebp
-	subl $15, %esp
-	andl $0xFFFFFFF0, %esp
+	andl $-16, %esp
 
 	# push arguments
-	subl $4, %esp
-	pushl %ebp
-	addl $32, (%esp)
+	subl $8, %esp
 	pushl %edi
 	pushl %esi
 	pushl %edx
@@ -43,6 +35,15 @@ asm_syscall_handler:
 
 	movl %ebp, %esp
 
+	# restore userspace segments
+	movw $(0x20 | 3), %bx
+	movw %bx, %ds
+	movw %bx, %es
+	movw $(0x30 | 3), %bx
+	movw %bx, %fs
+	movw $(0x38 | 3), %bx
+	movw %bx, %gs
+
 	# restore general purpose registers
 	popl %ebp
 	popl %esi
@@ -51,12 +52,6 @@ asm_syscall_handler:
 	popl %ecx
 	popl %ebx
 
-	# restore segment registers
-	popw %gs
-	popw %fs
-	popw %es
-	popw %ds
-
 	iret
 
 .global sys_fork_trampoline
@@ -68,7 +63,7 @@ sys_fork_trampoline:
 
 	call read_ip
 	testl %eax, %eax
-	jz .reload_stack
+	jz .done
 
 	movl %esp, %ebx
 
@@ -84,9 +79,3 @@ sys_fork_trampoline:
 	popl %ebx
 	popl %ebp
 	ret
-
- .reload_stack:
-	call get_thread_start_sp
-	movl %eax, %esp
-	xorl %eax, %eax
-	jmp .done

kernel/arch/i686/Thread.S

@@ -7,9 +7,6 @@ read_ip:
 # void start_kernel_thread()
 .global start_kernel_thread
 start_kernel_thread:
-	call get_thread_start_sp
-	movl %eax, %esp
-
 	# STACK LAYOUT
 	#   on_exit arg
 	#   on_exit func
@@ -31,39 +28,15 @@ start_kernel_thread:
 	subl $12, %esp
 	pushl %edi
 	call *%esi
-	addl $16, %esp
-
 
 .global start_userspace_thread
 start_userspace_thread:
-	call get_thread_start_sp
-	movl %eax, %esp
-
-	# STACK LAYOUT
-	#   entry
-	#   argc
-	#   argv
-	#   envp
-	#   userspace stack
-
-	call get_userspace_thread_stack_top
-
 	movw $(0x20 | 3), %bx
 	movw %bx, %ds
 	movw %bx, %es
+	movw $(0x30 | 3), %bx
 	movw %bx, %fs
+	movw $(0x38 | 3), %bx
 	movw %bx, %gs
-	xorw %bx, %bx
 
-	popl %edi
-	popl %esi
-	popl %edx
-	popl %ecx
-	popl %ebx
-
-	pushl $(0x20 | 3)
-	pushl %eax
-	pushl $0x202
-	pushl $(0x18 | 3)
-	pushl %ebx
 	iret

kernel/arch/i686/User.S

@@ -0,0 +1,54 @@
+# bool safe_user_memcpy(void*, const void*, size_t)
+.global safe_user_memcpy
+.global safe_user_memcpy_end
+.global safe_user_memcpy_fault
+safe_user_memcpy:
+	xorl %eax, %eax
+	xchgl 4(%esp), %edi
+	xchgl 8(%esp), %esi
+	movl 12(%esp), %ecx
+	movl %edi, %edx
+	rep movsb
+	movl 4(%esp), %edi
+	movl 8(%esp), %esi
+	incl %eax
+safe_user_memcpy_fault:
+	ret
+safe_user_memcpy_end:
+
+# bool safe_user_strncpy(void*, const void*, size_t)
+.global safe_user_strncpy
+.global safe_user_strncpy_end
+.global safe_user_strncpy_fault
+safe_user_strncpy:
+	xchgl 4(%esp), %edi
+	xchgl 8(%esp), %esi
+	movl 12(%esp), %ecx
+
+	testl %ecx, %ecx
+	jz safe_user_strncpy_fault
+
+ .safe_user_strncpy_loop:
+	movb (%esi), %al
+	movb %al, (%edi)
+	testb %al, %al
+	jz .safe_user_strncpy_done
+
+	incl %edi
+	incl %esi
+	decl %ecx
+	jnz .safe_user_strncpy_loop
+
+ safe_user_strncpy_fault:
+	xorl %eax, %eax
+	jmp .safe_user_strncpy_return
+
+ .safe_user_strncpy_done:
+	movl $1, %eax
+
+ .safe_user_strncpy_return:
+	movl 4(%esp), %edi
+	movl 8(%esp), %esi
+	ret
+
+safe_user_strncpy_end:

kernel/arch/i686/Yield.S

@@ -0,0 +1,25 @@
+.global asm_yield_trampoline
+asm_yield_trampoline:
+	leal 4(%esp), %ecx
+	movl 4(%esp), %esp
+
+	pushl -4(%ecx)
+	pushl %ecx
+	pushl %eax
+	pushl %ebx
+	pushl %esi
+	pushl %edi
+	pushl %ebp
+
+	pushl %esp
+	call scheduler_on_yield
+	addl $4, %esp
+
+	popl %ebp
+	popl %edi
+	popl %esi
+	popl %ebx
+	popl %eax
+	movl 4(%esp), %ecx
+	movl 0(%esp), %esp
+	jmp *%ecx

kernel/arch/i686/boot.S

@@ -11,9 +11,28 @@
 
 .code32
 
-# multiboot2 header
+// video mode info, page align modules
+.set multiboot_flags, (1 << 2) | (1 << 0)
+
 .section .multiboot, "aw"
-	.align 8
+multiboot_start:
+	.long 0x1BADB002
+	.long multiboot_flags
+	.long -(0x1BADB002 + multiboot_flags)
+
+	.long 0
+	.long 0
+	.long 0
+	.long 0
+	.long 0
+
+	.long 0
+	.long FB_WIDTH
+	.long FB_HEIGHT
+	.long FB_BPP
+multiboot_end:
+
+.section .multiboot2, "aw"
 multiboot2_start:
 	.long 0xE85250D6
 	.long 0
@@ -36,6 +55,12 @@ multiboot2_start:
 	.long 12
 	.long V2P(_start)
 
+	# page align modules
+	.align 8
+	.short 6
+	.short 0
+	.long 8
+
 	.align 8
 	.short 0
 	.short 0
@@ -43,7 +68,6 @@ multiboot2_start:
 multiboot2_end:
 
 .section .bananboot, "aw"
-	.align 8
 bananboot_start:
 	.long 0xBABAB007
 	.long -(0xBABAB007 + FB_WIDTH + FB_HEIGHT + FB_BPP)
@@ -53,10 +77,10 @@ bananboot_start:
 bananboot_end:
 
 .section .bss, "aw", @nobits
-	.align 4096
-	boot_stack_bottom:
+    .global g_boot_stack_top
+	g_boot_stack_bottom:
 		.skip 4096 * 4
-	boot_stack_top:
+	g_boot_stack_top:
 
 	.global g_kernel_cmdline
 	g_kernel_cmdline:
@@ -74,8 +98,7 @@ bananboot_end:
 boot_pdpt:
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
-	.quad 0
-	.quad 0
+	.skip 2 * 8
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
 .align 4096
@@ -88,13 +111,16 @@ boot_pd:
 	.endr
 boot_pts:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.rept 512
 			.long i + (PG_READ_WRITE | PG_PRESENT)
 			.long 0
 			.set i, i + 0x1000
 		.endr
 	.endr
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -161,6 +187,13 @@ enable_sse:
 	movl %eax, %cr4
 	ret
 
+enable_tsc:
+	# allow userspace to use RDTSC
+	movl %cr4, %ecx
+	andl $0xFFFFFFFB, %ecx
+	movl %ecx, %cr4
+	ret
+
 initialize_paging:
 	# enable PAE
 	movl %cr4, %ecx
@@ -188,7 +221,7 @@ _start:
 	movl %ebx, V2P(bootloader_info)
 
 	# load boot stack
-	movl $V2P(boot_stack_top), %esp
+	movl $V2P(g_boot_stack_top), %esp
 
 	# load boot GDT
 	lgdt V2P(boot_gdtr)
@@ -203,10 +236,11 @@ gdt_flush:
 	# do processor initialization
 	call check_requirements
 	call enable_sse
+	call enable_tsc
 	call initialize_paging
 
 	# load higher half stack pointer
-	movl $boot_stack_top, %esp
+	movl $g_boot_stack_top, %esp
 
 	# jump to higher half
 	leal higher_half, %ecx
@@ -242,7 +276,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -252,21 +286,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
+ap_stack_paddr:
+	.skip 4
+ap_stack_vaddr:
+	.skip 4
+ap_prepare_paging:
+	.skip 4
+ap_page_table:
+	.skip 4
+ap_ready:
 	.skip 4
-ap_stack_loaded:
-	.skip 1
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -275,32 +315,36 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
+	movl AP_REL(ap_stack_paddr), %esp
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
+	leal V2P(enable_tsc),        %ecx; call *%ecx
 	leal V2P(initialize_paging), %ecx; call *%ecx
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_flush_gdt)
+	ljmpl $0x08, $AP_REL(ap_flush_gdt)
 
 ap_flush_gdt:
-	# move stack pointer to higher half
-	movl %esp, %esp
-	addl $KERNEL_OFFSET, %esp
-
-	# jump to higher half
-	leal ap_higher_half, %ecx
+	movl $ap_higher_half, %ecx
 	jmp *%ecx
 
 ap_higher_half:
+	movl AP_REL(ap_prepare_paging), %eax
+	call *%eax
+
+	# load AP's initial values
+	movl AP_REL(ap_stack_vaddr), %esp
+	movl AP_REL(ap_page_table), %eax
+	movl $1, AP_REL(ap_ready)
+	movl %eax, %cr3
+
 	# clear rbp for stacktrace
 	xorl %ebp, %ebp
 
 1:	pause
 	cmpb $0, g_ap_startup_done
-	jz 1b
+	je 1b
 
 	lock incb g_ap_running_count
 

kernel/arch/i686/interrupts.S

@@ -1,133 +1,107 @@
-.macro push_userspace
-	pushw %gs
-	pushw %fs
-	pushw %es
-	pushw %ds
+.macro intr_header, n
 	pushal
-.endm
-
-.macro load_kernel_segments
+	testb $3, \n+8*4(%esp)
+	jz 1f
 	movw $0x10, %ax
 	movw %ax, %ds
 	movw %ax, %es
 	movw %ax, %fs
-
 	movw $0x28, %ax
 	movw %ax, %gs
+1:	cld
 .endm
 
-.macro pop_userspace
-	popal
-	popw %ds
-	popw %es
-	popw %fs
-	popw %gs
+.macro intr_footer, n
+	testb $3, \n+8*4(%esp)
+	jz 1f
+	call cpp_check_signal
+	movw $(0x20 | 3), %bx
+	movw %bx, %ds
+	movw %bx, %es
+	movw $(0x30 | 3), %bx
+	movw %bx, %fs
+	movw $(0x38 | 3), %bx
+	movw %bx, %gs
+1:	popal
 .endm
 
 isr_stub:
-	push_userspace
-	load_kernel_segments
-
+	intr_header 12
 	movl %cr0, %eax; pushl %eax
 	movl %cr2, %eax; pushl %eax
 	movl %cr3, %eax; pushl %eax
 	movl %cr4, %eax; pushl %eax
 
-	movl %esp,     %eax // register ptr
-	leal 64(%esp), %ebx // interrupt stack ptr
-	movl 60(%esp), %ecx // error code
-	movl 56(%esp), %edx // isr number
+	movl 48(%esp), %edi // isr number
+	movl 52(%esp), %esi // error code
+	leal 56(%esp), %edx // interrupt stack ptr
+	movl %esp,     %ecx // register ptr
+
+	# stack frame for stack trace
+	leal 56(%esp), %eax
+	pushl (%eax)
+	pushl %ebp
 
 	movl %esp, %ebp
-	subl $15, %esp
-	andl $0xFFFFFFF0, %esp
+	andl $-16, %esp
 
-	pushl %eax
-	pushl %ebx
 	pushl %ecx
 	pushl %edx
+	pushl %esi
+	pushl %edi
 	call cpp_isr_handler
 
 	movl %ebp, %esp
-	addl $16, %esp
+	addl $24, %esp
 
-	pop_userspace
+	intr_footer 12
 	addl $8, %esp
 	iret
 
 irq_stub:
-	push_userspace
-	load_kernel_segments
+	intr_header 12
 
-	movl 40(%esp), %eax # interrupt number
+	movl 32(%esp), %edi # interrupt number
 
 	movl %esp, %ebp
-	subl $15, %esp
-	andl $0xFFFFFFF0, %esp
+	andl $-16, %esp
 
 	subl $12, %esp
-	pushl %eax
+	pushl %edi
 	call cpp_irq_handler
 
 	movl %ebp, %esp
 
-	pop_userspace
+	intr_footer 12
 	addl $8, %esp
 	iret
 
-.global asm_yield_handler
-asm_yield_handler:
-	# This can only be called from kernel, so no segment saving is needed
-	pushal
-
-	movl %esp,     %eax # interrupt registers ptr
-	leal 32(%esp), %ebx # interrupt stack ptr
-
-	movl %esp, %ebp
-	subl $15, %esp
-	andl $0xFFFFFFF0, %esp
-
-	subl $8, %esp
-	pushl %eax
-	pushl %ebx
-	call cpp_yield_handler
-
-	movl %ebp, %esp
-
-	popal
-	iret
-
 .global asm_ipi_handler
 asm_ipi_handler:
-	push_userspace
-	load_kernel_segments
+	intr_header 4
 
 	movl %esp, %ebp
-	subl $15, %esp
-	andl $0xFFFFFFF0, %esp
+	andl $-16, %esp
 
 	call cpp_ipi_handler
 
 	movl %ebp, %esp
 
-	pop_userspace
+	intr_footer 4
 	iret
 
-
 .global asm_timer_handler
 asm_timer_handler:
-	push_userspace
-	load_kernel_segments
+	intr_header 4
 
 	movl %esp, %ebp
-	subl $15, %esp
-	andl $0xFFFFFFF0, %esp
+	andl $-16, %esp
 
 	call cpp_timer_handler
 
 	movl %ebp, %esp
 
-	pop_userspace
+	intr_footer 4
 	iret
 
 .macro isr n
@@ -186,211 +160,26 @@ isr 29
 isr 30
 isr 31
 
-irq 0
-irq 1
-irq 2
-irq 3
-irq 4
-irq 5
-irq 6
-irq 7
-irq 8
-irq 9
-irq 10
-irq 11
-irq 12
-irq 13
-irq 14
-irq 15
-irq 16
-irq 17
-irq 18
-irq 19
-irq 20
-irq 21
-irq 22
-irq 23
-irq 24
-irq 25
-irq 26
-irq 27
-irq 28
-irq 29
-irq 30
-irq 31
-irq 32
-irq 33
-irq 34
-irq 35
-irq 36
-irq 37
-irq 38
-irq 39
-irq 40
-irq 41
-irq 42
-irq 43
-irq 44
-irq 45
-irq 46
-irq 47
-irq 48
-irq 49
-irq 50
-irq 51
-irq 52
-irq 53
-irq 54
-irq 55
-irq 56
-irq 57
-irq 58
-irq 59
-irq 60
-irq 61
-irq 62
-irq 63
-irq 64
-irq 65
-irq 66
-irq 67
-irq 68
-irq 69
-irq 70
-irq 71
-irq 72
-irq 73
-irq 74
-irq 75
-irq 76
-irq 77
-irq 78
-irq 79
-irq 80
-irq 81
-irq 82
-irq 83
-irq 84
-irq 85
-irq 86
-irq 87
-irq 88
-irq 89
-irq 90
-irq 91
-irq 92
-irq 93
-irq 94
-irq 95
-irq 96
-irq 97
-irq 98
-irq 99
-irq 100
-irq 101
-irq 102
-irq 103
-irq 104
-irq 105
-irq 106
-irq 107
-irq 108
-irq 109
-irq 110
-irq 111
-irq 112
-irq 113
-irq 114
-irq 115
-irq 116
-irq 117
-irq 118
-irq 119
-irq 120
-irq 121
-irq 122
-irq 123
-irq 124
-irq 125
-irq 126
-irq 127
-irq 128
-irq 129
-irq 130
-irq 131
-irq 132
-irq 133
-irq 134
-irq 135
-irq 136
-irq 137
-irq 138
-irq 139
-irq 140
-irq 141
-irq 142
-irq 143
-irq 144
-irq 145
-irq 146
-irq 147
-irq 148
-irq 149
-irq 150
-irq 151
-irq 152
-irq 153
-irq 154
-irq 155
-irq 156
-irq 157
-irq 158
-irq 159
-irq 160
-irq 161
-irq 162
-irq 163
-irq 164
-irq 165
-irq 166
-irq 167
-irq 168
-irq 169
-irq 170
-irq 171
-irq 172
-irq 173
-irq 174
-irq 175
-irq 176
-irq 177
-irq 178
-irq 179
-irq 180
-irq 181
-irq 182
-irq 183
-irq 184
-irq 185
-irq 186
-irq 187
-irq 188
-irq 189
-irq 190
-irq 191
-irq 192
-irq 193
-irq 194
-irq 195
-irq 196
-irq 197
-irq 198
-irq 199
-irq 200
-irq 201
-irq 202
-irq 203
-irq 204
-irq 205
-irq 206
-irq 207
+.irp i,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, \
+        10, 11, 12, 13, 14, 15, 16, 17, 18, 19, \
+        20, 21, 22, 23, 24, 25, 26, 27, 28, 29, \
+        30, 31, 32, 33, 34, 35, 36, 37, 38, 39, \
+        40, 41, 42, 43, 44, 45, 46, 47, 48, 49, \
+        50, 51, 52, 53, 54, 55, 56, 57, 58, 59, \
+        60, 61, 62, 63, 64, 65, 66, 67, 68, 69, \
+        70, 71, 72, 73, 74, 75, 76, 77, 78, 79, \
+        80, 81, 82, 83, 84, 85, 86, 87, 88, 89, \
+        90, 91, 92, 93, 94, 95, 96, 97, 98, 99, \
+       100,101,102,103,104,105,106,107,108,109, \
+       110,111,112,113,114,115,116,117,118,119, \
+       120,121,122,123,124,125,126,127,128,129, \
+       130,131,132,133,134,135,136,137,138,139, \
+       140,141,142,143,144,145,146,147,148,149, \
+       150,151,152,153,154,155,156,157,158,159, \
+       160,161,162,163,164,165,166,167,168,169, \
+       170,171,172,173,174,175,176,177,178,179, \
+       180,181,182,183,184,185,186,187,188,189, \
+       190,191,192,193,194,195,196,197,198,199, \
+       200,201,202,203,204,205,206,207
+	irq \i
+.endr

kernel/arch/i686/linker.ld

@@ -11,20 +11,21 @@ SECTIONS
 	{
 		g_kernel_execute_start = .;
 		*(.multiboot)
+		*(.multiboot2)
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{

kernel/arch/x86_64/PageTable.cpp

@@ -2,7 +2,6 @@
 #include <kernel/CPUID.h>
 #include <kernel/Lock/SpinLock.h>
 #include <kernel/Memory/Heap.h>
-#include <kernel/Memory/kmalloc.h>
 #include <kernel/Memory/PageTable.h>
 
 extern uint8_t g_kernel_start[];
@@ -17,13 +16,15 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
 	SpinLock PageTable::s_fast_page_lock;
 
 	static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000;
-	static bool s_is_hddm_initialized = false;
+	static bool s_is_initialized = false;
 
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
@@ -35,6 +36,8 @@ namespace Kernel
 
 	static paddr_t s_global_pml4_entries[512] { 0 };
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
 	static constexpr inline bool is_canonical(uintptr_t addr)
 	{
 		constexpr uintptr_t mask = 0xFFFF800000000000;
@@ -54,68 +57,27 @@ namespace Kernel
 		return addr;
 	}
 
-	struct FuncsKmalloc
+	static paddr_t allocate_zeroed_page_aligned_page()
 	{
-		static paddr_t allocate_zeroed_page_aligned_page()
-		{
-			void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-			ASSERT(page);
-			memset(page, 0, PAGE_SIZE);
-			return kmalloc_paddr_of(reinterpret_cast<vaddr_t>(page)).value();
-		}
+		const paddr_t paddr = Heap::get().take_free_page();
+		ASSERT(paddr);
+		memset(reinterpret_cast<void*>(paddr + s_hhdm_offset), 0, PAGE_SIZE);
+		return paddr;
+	}
 
-		static void unallocate_page(paddr_t paddr)
-		{
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-		}
-
-		static uint64_t* P2V(paddr_t paddr)
-		{
-			return reinterpret_cast<uint64_t*>(paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-		}
-	};
-
-	struct FuncsHHDM
+	static void unallocate_page(paddr_t paddr)
 	{
-		static paddr_t allocate_zeroed_page_aligned_page()
-		{
-			const paddr_t paddr = Heap::get().take_free_page();
-			ASSERT(paddr);
-			memset(reinterpret_cast<void*>(paddr + s_hhdm_offset), 0, PAGE_SIZE);
-			return paddr;
-		}
+		Heap::get().release_page(paddr);
+	}
 
-		static void unallocate_page(paddr_t paddr)
-		{
-			Heap::get().release_page(paddr);
-		}
+	static uint64_t* P2V(paddr_t paddr)
+	{
+		ASSERT(paddr != 0);
+		ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset));
+		return reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
+	}
 
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			ASSERT(vaddr >= s_hhdm_offset);
-			ASSERT(vaddr < KERNEL_OFFSET);
-			return vaddr - s_hhdm_offset;
-		}
-
-		static uint64_t* P2V(paddr_t paddr)
-		{
-			ASSERT(paddr != 0);
-			ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset));
-			return reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
-		}
-	};
-
-	static paddr_t   (*allocate_zeroed_page_aligned_page)() = &FuncsKmalloc::allocate_zeroed_page_aligned_page;
-	static void      (*unallocate_page)(paddr_t)            = &FuncsKmalloc::unallocate_page;
-	static paddr_t   (*V2P)(vaddr_t)                        = &FuncsKmalloc::V2P;
-	static uint64_t* (*P2V)(paddr_t)                        = &FuncsKmalloc::P2V;
-
-	static inline PageTable::flags_t parse_flags(uint64_t entry)
+	static PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
 
@@ -137,7 +99,7 @@ namespace Kernel
 	//   0: 4 KiB
 	//   1: 2 MiB
 	//   2: 1 GiB
-	static void init_map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
+	static void map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
 	{
 		ASSERT(0 <= page_size && page_size <= 2);
 
@@ -184,7 +146,7 @@ namespace Kernel
 		const uint64_t noexec_flag = s_has_nxe ? (static_cast<uint64_t>(1) << 63) : 0;
 
 		const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, noexec_flag);
-		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags;
+		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags | noexec_flag;
 
 		paddr_t lowest_paddr = pdpt;
 		uint16_t lowest_entry = pdpte;
@@ -207,23 +169,11 @@ namespace Kernel
 		});
 	}
 
-	static void init_map_hhdm(paddr_t pml4)
+	static void initialize_hhdm(paddr_t pml4)
 	{
 		for (const auto& entry : g_boot_info.memory_map_entries)
 		{
-			bool should_map = false;
-			switch (entry.type)
-			{
-				case MemoryMapEntry::Type::Available:
-					should_map = true;
-					break;
-				case MemoryMapEntry::Type::ACPIReclaim:
-				case MemoryMapEntry::Type::ACPINVS:
-				case MemoryMapEntry::Type::Reserved:
-					should_map = false;
-					break;
-			}
-			if (!should_map)
+			if (entry.type != MemoryMapEntry::Type::Available)
 				continue;
 
 			constexpr size_t one_gib = 1024 * 1024 * 1024;
@@ -235,156 +185,39 @@ namespace Kernel
 			{
 				if (s_has_gib && paddr % one_gib == 0 && paddr + one_gib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 2);
+					map_hhdm_page(pml4, paddr, 2);
 					paddr += one_gib;
 				}
 				else if (paddr % two_mib == 0 && paddr + two_mib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 1);
+					map_hhdm_page(pml4, paddr, 1);
 					paddr += two_mib;
 				}
 				else
 				{
-					init_map_hhdm_page(pml4, paddr, 0);
+					map_hhdm_page(pml4, paddr, 0);
 					paddr += PAGE_SIZE;
 				}
 			}
 		}
 	}
 
-	static paddr_t copy_page_from_kmalloc_to_heap(paddr_t kmalloc_paddr)
+	void PageTable::initialize_fast_page()
 	{
-		const paddr_t heap_paddr = Heap::get().take_free_page();
-		ASSERT(heap_paddr);
-
-		const vaddr_t kmalloc_vaddr = kmalloc_vaddr_of(kmalloc_paddr).value();
-
-		PageTable::with_fast_page(heap_paddr, [kmalloc_vaddr] {
-			memcpy(PageTable::fast_page_as_ptr(), reinterpret_cast<void*>(kmalloc_vaddr), PAGE_SIZE);
-		});
-
-		return heap_paddr;
+		s_fast_page_pt = g_boot_fast_page_pt;
 	}
 
-	static void copy_paging_structure_to_heap(uint64_t* old_table, uint64_t* new_table, int depth)
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t old_entry = old_table[index];
-			if (old_entry == 0)
-			{
-				new_table[index] = 0;
-				continue;
-			}
-
-			const paddr_t old_paddr = old_entry & page_addr_mask;
-			const paddr_t new_paddr = copy_page_from_kmalloc_to_heap(old_paddr);
-			new_table[index] = new_paddr | (old_entry & page_flag_mask);
-
-			uint64_t* next_old_table = reinterpret_cast<uint64_t*>(old_paddr + s_hhdm_offset);
-			uint64_t* next_new_table = reinterpret_cast<uint64_t*>(new_paddr + s_hhdm_offset);
-			copy_paging_structure_to_heap(next_old_table, next_new_table, depth - 1);
-		}
-	}
-
-	static void free_kmalloc_paging_structure(uint64_t* table, int depth)
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t entry = table[index];
-			if (entry == 0)
-				continue;
-
-			const paddr_t paddr = entry & page_addr_mask;
-
-			uint64_t* next_table = reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
-			free_kmalloc_paging_structure(next_table, depth - 1);
-
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-	}
-
-	void PageTable::initialize_pre_heap()
+	static void detect_cpu_features()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_1gib_pages())
 			s_has_gib = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-		s_kernel->prepare_fast_page();
-		s_kernel->initialize_kernel();
-
-		for (auto pml4e : s_global_pml4_entries)
-			ASSERT(pml4e == 0);
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		s_global_pml4_entries[511] = pml4[511];
 	}
 
-	void PageTable::initialize_post_heap()
-	{
-		ASSERT(s_kernel);
-
-		init_map_hhdm(s_kernel->m_highest_paging_struct);
-
-		const paddr_t old_pml4_paddr = s_kernel->m_highest_paging_struct;
-		const paddr_t new_pml4_paddr = copy_page_from_kmalloc_to_heap(old_pml4_paddr);
-
-		uint64_t* old_pml4 = reinterpret_cast<uint64_t*>(kmalloc_vaddr_of(old_pml4_paddr).value());
-		uint64_t* new_pml4 = reinterpret_cast<uint64_t*>(new_pml4_paddr + s_hhdm_offset);
-
-		const paddr_t old_pdpt_paddr = old_pml4[511] & s_page_addr_mask;
-		const paddr_t new_pdpt_paddr = Heap::get().take_free_page();
-		ASSERT(new_pdpt_paddr);
-
-		uint64_t* old_pdpt = reinterpret_cast<uint64_t*>(old_pdpt_paddr + s_hhdm_offset);
-		uint64_t* new_pdpt = reinterpret_cast<uint64_t*>(new_pdpt_paddr + s_hhdm_offset);
-		copy_paging_structure_to_heap(old_pdpt, new_pdpt, 2);
-
-		new_pml4[511] = new_pdpt_paddr | (old_pml4[511] & s_page_flag_mask);
-		s_global_pml4_entries[511] = new_pml4[511];
-
-		s_kernel->m_highest_paging_struct = new_pml4_paddr;
-		s_kernel->load();
-
-		free_kmalloc_paging_structure(old_pdpt, 2);
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pdpt_paddr).value()));
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pml4_paddr).value()));
-
-		allocate_zeroed_page_aligned_page = &FuncsHHDM::allocate_zeroed_page_aligned_page;
-		unallocate_page                   = &FuncsHHDM::unallocate_page;
-		V2P                               = &FuncsHHDM::V2P;
-		P2V                               = &FuncsHHDM::P2V;
-
-		s_is_hddm_initialized = true;
-
-		// This is a hack to unmap fast page. fast page pt is copied
-		// while it is mapped, so we need to manually unmap it
-		SpinLockGuard _(s_fast_page_lock);
-		unmap_fast_page();
-	}
-
-	void PageTable::initial_load()
+	void PageTable::enable_cpu_features()
 	{
 		if (s_has_nxe)
 		{
@@ -423,8 +256,63 @@ namespace Kernel
 			"movq %%rax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		const paddr_t boot_pml4_paddr = ({
+			paddr_t paddr;
+			asm volatile("movq %%cr3, %0" : "=r"(paddr));
+			paddr;
+		});
+
+		initialize_hhdm(boot_pml4_paddr);
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel != nullptr);
+
+		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		ASSERT(s_kernel->m_highest_paging_struct);
+
+		uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+		s_kernel->map_kernel_memory();
+		s_global_pml4_entries[511] = pml4[511];
+
+		// update fast page pt
+		{
+			constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+			constexpr uint16_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+			constexpr uint16_t pdpte = (uc_vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (uc_vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						table[entry] = allocate_zeroed_page_aligned_page();
+						ASSERT(table[entry]);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pml4 = s_kernel->m_highest_paging_struct;
+			const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, Flags::ReadWrite | Flags::Present);
+			const paddr_t pd   = get_or_allocate_entry(pdpt, pdpte, Flags::ReadWrite | Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd,   pde,   Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -440,12 +328,12 @@ namespace Kernel
 		return true;
 	}
 
-	void PageTable::initialize_kernel()
+	void PageTable::map_kernel_memory()
 	{
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
 		const vaddr_t kernel_start = reinterpret_cast<vaddr_t>(g_kernel_start);
 		map_range_at(
-			V2P(kernel_start),
+			kernel_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_start,
 			g_kernel_end - g_kernel_start,
 			Flags::Present
@@ -454,7 +342,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		const vaddr_t kernel_execute_start = reinterpret_cast<vaddr_t>(g_kernel_execute_start);
 		map_range_at(
-			V2P(kernel_execute_start),
+			kernel_execute_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_execute_start,
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
@@ -463,7 +351,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		const vaddr_t kernel_writable_start = reinterpret_cast<vaddr_t>(g_kernel_writable_start);
 		map_range_at(
-			V2P(kernel_writable_start),
+			kernel_writable_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_writable_start,
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
@@ -472,108 +360,58 @@ namespace Kernel
 		// Map userspace memory
 		const vaddr_t userspace_start = reinterpret_cast<vaddr_t>(g_userspace_start);
 		map_range_at(
-			V2P(userspace_start),
+			userspace_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			userspace_start,
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-
-		uint64_t* pml4 = P2V(m_highest_paging_struct);
-		ASSERT(!(pml4[pml4e] & Flags::Present));
-		pml4[pml4e] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		ASSERT(!(pdpt[pdpte] & Flags::Present));
-		pdpt[pdpte] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
-		ASSERT(paddr);
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
 
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
+		ASSERT(!(*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
-
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
+		ASSERT((*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = 0;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = 0;
-
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
 	{
 		SpinLockGuard _(s_kernel->m_lock);
+
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
+
+		page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		if (page_table->m_highest_paging_struct == 0)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
+		}
+
+		uint64_t* pml4 = P2V(page_table->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+
 		return page_table;
 	}
 
-	void PageTable::map_kernel_memory()
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pml4_entries[511]);
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-
-		PageTable::with_fast_page(m_highest_paging_struct, [] {
-			for (size_t i = 0; i < 512; i++)
-			{
-				if (s_global_pml4_entries[i] == 0)
-					continue;
-				ASSERT(i >= 256);
-				PageTable::fast_page_as_sized<uint64_t>(i) = s_global_pml4_entries[i];
-			}
-		});
-	}
-
 	PageTable::~PageTable()
 	{
 		if (m_highest_paging_struct == 0)
@@ -611,10 +449,39 @@ namespace Kernel
 		Processor::set_current_page_table(this);
 	}
 
-	void PageTable::invalidate(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::invalidate_range(vaddr_t vaddr, size_t pages, bool send_smp_message)
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
-		asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory");
+
+		const bool is_userspace = (vaddr < KERNEL_OFFSET);
+		if (is_userspace && this != &PageTable::current())
+			;
+		else if (pages <= 32 || !s_is_initialized)
+		{
+			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
+				asm volatile("invlpg (%0)" :: "r"(vaddr));
+		}
+		else if (is_userspace || !s_has_pge)
+		{
+			asm volatile("movq %0, %%cr3" :: "r"(m_highest_paging_struct));
+		}
+		else
+		{
+			asm volatile(
+				"movq %%cr4, %%rax;"
+
+				"andq $~0x80, %%rax;"
+				"movq %%rax, %%cr4;"
+
+				"movq %0, %%cr3;"
+
+				"orq $0x80, %%rax;"
+				"movq %%rax, %%cr4;"
+				:
+				: "r"(m_highest_paging_struct)
+				: "rax"
+			);
+		}
 
 		if (send_smp_message)
 		{
@@ -622,13 +489,14 @@ namespace Kernel
 				.type = Processor::SMPMessage::Type::FlushTLB,
 				.flush_tlb = {
 					.vaddr      = vaddr,
-					.page_count = 1
+					.page_count = pages,
+					.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
 				}
 			});
 		}
 	}
 
-	void PageTable::unmap_page(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -657,30 +525,27 @@ namespace Kernel
 		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
 		uint64_t* pt   = P2V(pd[pde]     & s_page_addr_mask);
 
+		const paddr_t old_paddr = pt[pte] & PAGE_ADDR_MASK;
+
 		pt[pte] = 0;
-		invalidate(vaddr, send_smp_message);
+
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::unmap_range(vaddr_t vaddr, size_t size)
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
 
-		size_t page_count = range_page_count(vaddr, size);
+		const size_t page_count = range_page_count(vaddr, size);
 
 		SpinLockGuard _(m_lock);
 		for (vaddr_t page = 0; page < page_count; page++)
 			unmap_page(vaddr + page * PAGE_SIZE, false);
-
-		Processor::broadcast_smp_message({
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = page_count
-			}
-		});
+		invalidate_range(vaddr, page_count, true);
 	}
 
-	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool send_smp_message)
+	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -741,9 +606,12 @@ namespace Kernel
 		if (!(flags & Flags::Present))
 			uwr_flags &= ~Flags::Present;
 
+		const paddr_t old_paddr = pt[pte] & PAGE_ADDR_MASK;
+
 		pt[pte] = paddr | uwr_flags | extra_flags;
 
-		invalidate(vaddr, send_smp_message);
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
@@ -759,14 +627,66 @@ namespace Kernel
 		SpinLockGuard _(m_lock);
 		for (size_t page = 0; page < page_count; page++)
 			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false);
+		invalidate_range(vaddr, page_count, true);
+	}
 
-		Processor::broadcast_smp_message({
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = page_count
+	void PageTable::remove_writable_from_range(vaddr_t vaddr, size_t size)
+	{
+		ASSERT(vaddr);
+		ASSERT(vaddr % PAGE_SIZE == 0);
+
+		ASSERT(is_canonical(vaddr));
+		ASSERT(is_canonical(vaddr + size - 1));
+
+		const vaddr_t uc_vaddr_start = uncanonicalize(vaddr);
+		const vaddr_t uc_vaddr_end   = uncanonicalize(vaddr + size - 1);
+
+		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
+		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
+		uint16_t pde   = (uc_vaddr_start >> 21) & 0x1FF;
+		uint16_t pte   = (uc_vaddr_start >> 12) & 0x1FF;
+
+		const uint16_t e_pml4e = (uc_vaddr_end >> 39) & 0x1FF;
+		const uint16_t e_pdpte = (uc_vaddr_end >> 30) & 0x1FF;
+		const uint16_t e_pde   = (uc_vaddr_end >> 21) & 0x1FF;
+		const uint16_t e_pte   = (uc_vaddr_end >> 12) & 0x1FF;
+
+		SpinLockGuard _(m_lock);
+
+		const uint64_t* pml4 = P2V(m_highest_paging_struct);
+		for (; pml4e <= e_pml4e; pml4e++)
+		{
+			if (!(pml4[pml4e] & Flags::ReadWrite))
+				continue;
+			const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
+			for (; pdpte < 512; pdpte++)
+			{
+				if (pml4e == e_pml4e && pdpte > e_pdpte)
+					break;
+				if (!(pdpt[pdpte] & Flags::ReadWrite))
+					continue;
+				const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+				for (; pde < 512; pde++)
+				{
+					if (pml4e == e_pml4e && pdpte == e_pdpte && pde > e_pde)
+						break;
+					if (!(pd[pde] & Flags::ReadWrite))
+						continue;
+					uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+					for (; pte < 512; pte++)
+					{
+						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
+							break;
+						pt[pte] &= ~static_cast<uint64_t>(Flags::ReadWrite);
+					}
+					pte = 0;
+				}
+				pde = 0;
 			}
-		});
+			pdpte = 0;
+		}
+
+		invalidate_range(vaddr, size / PAGE_SIZE, true);
 	}
 
 	uint64_t PageTable::get_page_data(vaddr_t vaddr) const
@@ -813,13 +733,13 @@ namespace Kernel
 		return page_data & s_page_addr_mask;
 	}
 
-	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free)
+	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool invalidate)
 	{
 		SpinLockGuard _(m_lock);
 		ASSERT(vaddr % PAGE_SIZE == 0);
 		if (only_free && !is_page_free(vaddr))
 			return false;
-		map_page_at(0, vaddr, Flags::Reserved);
+		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, invalidate);
 		return true;
 	}
 
@@ -833,7 +753,8 @@ namespace Kernel
 		if (only_free && !is_range_free(vaddr, bytes))
 			return false;
 		for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
-			reserve_page(vaddr + offset);
+			reserve_page(vaddr + offset, true, false);
+		invalidate_range(vaddr, bytes / PAGE_SIZE, true);
 		return true;
 	}
 
@@ -847,9 +768,9 @@ namespace Kernel
 			last_address -= rem;
 
 		ASSERT(is_canonical(first_address));
-		ASSERT(is_canonical(last_address));
+		ASSERT(is_canonical(last_address - 1));
 		const vaddr_t uc_vaddr_start = uncanonicalize(first_address);
-		const vaddr_t uc_vaddr_end   = uncanonicalize(last_address);
+		const vaddr_t uc_vaddr_end   = uncanonicalize(last_address - 1);
 
 		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
 		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
@@ -866,10 +787,8 @@ namespace Kernel
 		// Try to find free page that can be mapped without
 		// allocations (page table with unused entries)
 		const uint64_t* pml4 = P2V(m_highest_paging_struct);
-		for (; pml4e < 512; pml4e++)
+		for (; pml4e <= e_pml4e; pml4e++)
 		{
-			if (pml4e > e_pml4e)
-				break;
 			if (!(pml4[pml4e] & Flags::Present))
 				continue;
 			const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
@@ -889,22 +808,24 @@ namespace Kernel
 					const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
 					for (; pte < 512; pte++)
 					{
-						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte >= e_pte)
+						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 							break;
-						if (!(pt[pte] & Flags::Used))
-						{
-							vaddr_t vaddr = 0;
-							vaddr |= static_cast<uint64_t>(pml4e) << 39;
-							vaddr |= static_cast<uint64_t>(pdpte) << 30;
-							vaddr |= static_cast<uint64_t>(pde)   << 21;
-							vaddr |= static_cast<uint64_t>(pte)   << 12;
-							vaddr = canonicalize(vaddr);
-							ASSERT(reserve_page(vaddr));
-							return vaddr;
-						}
+						if (pt[pte] & Flags::Used)
+							continue;
+						vaddr_t vaddr = 0;
+						vaddr |= static_cast<uint64_t>(pml4e) << 39;
+						vaddr |= static_cast<uint64_t>(pdpte) << 30;
+						vaddr |= static_cast<uint64_t>(pde)   << 21;
+						vaddr |= static_cast<uint64_t>(pte)   << 12;
+						vaddr = canonicalize(vaddr);
+						ASSERT(reserve_page(vaddr));
+						return vaddr;
 					}
+					pte = 0;
 				}
+				pde = 0;
 			}
+			pdpte = 0;
 		}
 
 		for (vaddr_t uc_vaddr = uc_vaddr_start; uc_vaddr < uc_vaddr_end; uc_vaddr += PAGE_SIZE)
@@ -916,7 +837,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	vaddr_t PageTable::reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address)
@@ -929,7 +850,7 @@ namespace Kernel
 			last_address -= rem;
 
 		ASSERT(is_canonical(first_address));
-		ASSERT(is_canonical(last_address));
+		ASSERT(is_canonical(last_address - 1));
 
 		SpinLockGuard _(m_lock);
 
@@ -958,7 +879,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	bool PageTable::is_page_free(vaddr_t page) const

kernel/arch/x86_64/Signal.S

@@ -1,57 +1,97 @@
 .section .userspace, "ax"
 
 // stack contains
-//    return address
-//    signal number
-//    signal handler
+//    (8 bytes)  return address (on return stack)
+//    (8 bytes)  return stack
+//    (8 bytes)  return rflags
+//    (8 bytes)  restore sigmask
+//    (56 bytes) siginfo_t
+//    (8 bytes)  signal number
+//    (8 bytes)  signal handler
 
 .global signal_trampoline
 signal_trampoline:
-	pushq %rax
-	pushq %rbx
-	pushq %rcx
-	pushq %rdx
-	pushq %rbp
-	pushq %rdi
-	pushq %rsi
-	pushq %r8
-	pushq %r9
-	pushq %r10
-	pushq %r11
-	pushq %r12
-	pushq %r13
+	pushq %r15 // gregs
 	pushq %r14
-	pushq %r15
+	pushq %r13
+	pushq %r12
+	pushq %r11
+	pushq %r10
+	pushq %r9
+	pushq %r8
+	pushq %rsi
+	pushq %rdi
+	pushq %rdx
+	pushq %rcx
+	pushq %rbx
+	pushq %rax
+	pushq %rbp
 
-	movq 128(%rsp), %rdi
-	movq 120(%rsp), %rax
+	movq 208(%rsp), %rax
+	pushq %rax; addq $(128 + 8), (%rsp)
+	pushq (%rax)
+
+	// FIXME: populate these
+	xorq %rax, %rax
+	pushq %rax // stack
+	pushq %rax
+	pushq %rax
+	pushq %rax // sigset
+	pushq %rax // link
+
+	movq  %rsp,     %rdx // ucontext
+	leaq 192(%rsp), %rsi // siginfo
+	movq 184(%rsp), %rdi // signal number
+	movq 176(%rsp), %rax // handler
 
 	// align stack to 16 bytes
-	movq %rsp, %rbx
-	andq $0x0F, %rbx
-	subq %rbx, %rsp
+	movq %rsp, %rbp
+	andq $-16, %rsp
+
+	subq $512, %rsp
+	fxsave64 (%rsp)
 
 	call *%rax
 
-	// restore stack
-	addq %rbx, %rsp
-	popq %r15
-	popq %r14
-	popq %r13
-	popq %r12
-	popq %r11
-	popq %r10
-	popq %r9
-	popq %r8
-	popq %rsi
-	popq %rdi
-	popq %rbp
-	popq %rdx
-	popq %rcx
-	popq %rbx
-	popq %rax
+	fxrstor64 (%rsp)
+	addq $512, %rsp
 
+	// restore stack
+	movq %rbp, %rsp
+	addq $40, %rsp
+
+	// restore sigmask
+	movq $83, %rdi       // SYS_SIGPROCMASK
+	movq $3,  %rsi       // SIG_SETMASK
+	leaq 192(%rsp), %rdx // set
+	xorq %r10, %r10      // oset
+	syscall
+
+	// restore registers
 	addq $16, %rsp
+	popq %rbp
+	popq %rax
+	popq %rbx
+	popq %rcx
+	popq %rdx
+	popq %rdi
+	popq %rsi
+	popq %r8
+	popq %r9
+	popq %r10
+	popq %r11
+	popq %r12
+	popq %r13
+	popq %r14
+	popq %r15
+
+	// skip handler, number, siginfo_t, sigmask
+	addq $80, %rsp
+
+	// restore flags
+	popfq
+
+	movq (%rsp), %rsp
 
 	// return over red-zone
 	ret $128

kernel/arch/x86_64/Syscall.S

@@ -1,48 +1,26 @@
-// arguments in RAX, RBX, RCX, RDX, RSI, RDI
-// System V ABI: RDI, RSI, RDX, RCX, R8, R9
 .global asm_syscall_handler
 asm_syscall_handler:
-	pushq %rbx
-	pushq %rcx
-	pushq %rdx
-	pushq %rdi
-	pushq %rsi
-	pushq %rbp
-	pushq %r8
-	pushq %r9
-	pushq %r10
+	swapgs
+
+	movq %rsp, %rax
+	movq %gs:8, %rsp
+
+	pushq $(0x20 | 3)
+	pushq %rax
 	pushq %r11
-	pushq %r12
-	pushq %r13
-	pushq %r14
-	pushq %r15
+	pushq $(0x28 | 3)
+	pushq %rcx
+	subq $8, %rsp
 
-	movq %rsi, %r8
-	movq %rdi, %r9
-	movq %rax, %rdi
-	movq %rbx, %rsi
-	xchgq %rcx, %rdx
-	leaq 112(%rsp), %rbx
-	pushq %rbx
+	movq %r10, %rcx
 	call cpp_syscall_handler
-	addq $8, %rsp
 
-	popq %r15
-	popq %r14
-	popq %r13
-	popq %r12
-	popq %r11
-	popq %r10
-	popq %r9
-	popq %r8
-	popq %rbp
-	popq %rsi
-	popq %rdi
-	popq %rdx
-	popq %rcx
-	popq %rbx
-	iretq
+	movq  8(%rsp), %rcx
+	movq 24(%rsp), %r11
+	movq 32(%rsp), %rsp
 
+	swapgs
+	sysretq
 
 .global sys_fork_trampoline
 sys_fork_trampoline:
@@ -55,7 +33,7 @@ sys_fork_trampoline:
 
 	call read_ip
 	testq %rax, %rax
-	je .reload_stack
+	jz .done
 
 	movq %rax, %rsi
 	movq %rsp, %rdi
@@ -69,9 +47,3 @@ sys_fork_trampoline:
 	popq %rbp
 	popq %rbx
 	ret
-
- .reload_stack:
-	call get_thread_start_sp
-	movq %rax, %rsp
-	xorq %rax, %rax
-	jmp .done

kernel/arch/x86_64/Thread.S

@@ -7,9 +7,6 @@ read_ip:
 # void start_kernel_thread()
 .global start_kernel_thread
 start_kernel_thread:
-	call get_thread_start_sp
-	movq %rax, %rsp
-
 	# STACK LAYOUT
 	#   on_exit arg
 	#   on_exit func
@@ -27,27 +24,5 @@ start_kernel_thread:
 
 .global start_userspace_thread
 start_userspace_thread:
-	call get_thread_start_sp
-	movq %rax, %rsp
-
-	# STACK LAYOUT
-	#   entry
-	#   argc
-	#   argv
-	#   envp
-	#   userspace stack
-
-	call get_userspace_thread_stack_top
-
-	popq %rdi
-	popq %rsi
-	popq %rdx
-	popq %rcx
-	popq %rbx
-
-	pushq $(0x20 | 3)
-	pushq %rax
-	pushq $0x202
-	pushq $(0x18 | 3)
-	pushq %rbx
+	swapgs
 	iretq

kernel/arch/x86_64/User.S

@@ -0,0 +1,87 @@
+# bool safe_user_memcpy(void*, const void*, size_t)
+.global safe_user_memcpy
+.global safe_user_memcpy_end
+.global safe_user_memcpy_fault
+safe_user_memcpy:
+	xorq %rax, %rax
+	movq %rdx, %rcx
+	rep movsb
+	incq %rax
+safe_user_memcpy_fault:
+	ret
+safe_user_memcpy_end:
+
+# bool safe_user_strncpy(void*, const void*, size_t)
+.global safe_user_strncpy
+.global safe_user_strncpy_end
+.global safe_user_strncpy_fault
+safe_user_strncpy:
+	movq %rdx, %rcx
+	testq %rcx, %rcx
+	jz safe_user_strncpy_fault
+
+ .safe_user_strncpy_align_loop:
+	testb $0x7, %sil
+	jz .safe_user_strncpy_align_done
+
+	movb (%rsi), %al
+	movb %al, (%rdi)
+	testb %al, %al
+	jz .safe_user_strncpy_done
+
+	incq %rdi
+	incq %rsi
+	decq %rcx
+	jnz .safe_user_strncpy_align_loop
+	jmp safe_user_strncpy_fault
+
+ .safe_user_strncpy_align_done:
+	movq $0x0101010101010101, %r8
+	movq $0x8080808080808080, %r9
+
+ .safe_user_strncpy_qword_loop:
+	cmpq $8, %rcx
+	jb .safe_user_strncpy_qword_done
+
+	movq (%rsi), %rax
+	movq %rax, %r10
+	movq %rax, %r11
+
+	# https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord
+	subq %r8, %r10
+	notq %r11
+	andq %r11, %r10
+	andq %r9, %r10
+	jnz .safe_user_strncpy_byte_loop
+
+	movq %rax, (%rdi)
+
+	addq $8, %rdi
+	addq $8, %rsi
+	subq $8, %rcx
+	jnz .safe_user_strncpy_qword_loop
+	jmp safe_user_strncpy_fault
+
+ .safe_user_strncpy_qword_done:
+	testq %rcx, %rcx
+	jz safe_user_strncpy_fault
+
+ .safe_user_strncpy_byte_loop:
+	movb (%rsi), %al
+	movb %al, (%rdi)
+	testb %al, %al
+	jz .safe_user_strncpy_done
+
+	incq %rdi
+	incq %rsi
+	decq %rcx
+	jnz .safe_user_strncpy_byte_loop
+
+ safe_user_strncpy_fault:
+	xorq %rax, %rax
+	ret
+
+ .safe_user_strncpy_done:
+	movb $1, %al
+	ret
+safe_user_strncpy_end:

kernel/arch/x86_64/Yield.S

@@ -0,0 +1,29 @@
+.global asm_yield_trampoline
+asm_yield_trampoline:
+	leaq 8(%rsp), %rcx
+	movq %rdi, %rsp
+
+	subq $8, %rsp
+	pushq -8(%rcx)
+	pushq %rcx
+	pushq %rax
+	pushq %rbx
+	pushq %rbp
+	pushq %r12
+	pushq %r13
+	pushq %r14
+	pushq %r15
+
+	movq %rsp, %rdi
+	call scheduler_on_yield
+
+	popq %r15
+	popq %r14
+	popq %r13
+	popq %r12
+	popq %rbp
+	popq %rbx
+	popq %rax
+	movq 8(%rsp), %rcx
+	movq 0(%rsp), %rsp
+	jmp *%rcx

kernel/arch/x86_64/boot.S

@@ -11,9 +11,28 @@
 
 .code32
 
-# multiboot2 header
+// custom addresses, video mode info, page align modules
+.set multiboot_flags, (1 << 16) | (1 << 2) | (1 << 0)
+
 .section .multiboot, "aw"
-	.align 8
+multiboot_start:
+	.long 0x1BADB002
+	.long multiboot_flags
+	.long -(0x1BADB002 + multiboot_flags)
+
+	.long V2P(multiboot_start)
+	.long V2P(g_kernel_start)
+	.long V2P(g_kernel_bss_start)
+	.long V2P(g_kernel_end)
+	.long V2P(_start)
+
+	.long 0
+	.long FB_WIDTH
+	.long FB_HEIGHT
+	.long FB_BPP
+multiboot_end:
+
+.section .multiboot2, "aw"
 multiboot2_start:
 	.long 0xE85250D6
 	.long 0
@@ -36,6 +55,12 @@ multiboot2_start:
 	.long 12
 	.long V2P(_start)
 
+	# page align modules
+	.align 8
+	.short 6
+	.short 0
+	.long 8
+
 	.align 8
 	.short 0
 	.short 0
@@ -43,7 +68,6 @@ multiboot2_start:
 multiboot2_end:
 
 .section .bananboot, "aw"
-	.align 8
 bananboot_start:
 	.long 0xBABAB007
 	.long -(0xBABAB007 + FB_WIDTH + FB_HEIGHT + FB_BPP)
@@ -53,9 +77,10 @@ bananboot_start:
 bananboot_end:
 
 .section .bss, "aw", @nobits
-	boot_stack_bottom:
-		.skip 4096 * 64
-	boot_stack_top:
+    .global g_boot_stack_top
+	g_boot_stack_bottom:
+		.skip 4096 * 4
+	g_boot_stack_top:
 
 	.global g_kernel_cmdline
 	g_kernel_cmdline:
@@ -72,27 +97,25 @@ bananboot_end:
 .align 4096
 boot_pml4:
 	.quad V2P(boot_pdpt_lo) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 510
-		.quad 0
-	.endr
+	.skip 510 * 8
 	.quad V2P(boot_pdpt_hi) + (PG_READ_WRITE | PG_PRESENT)
 boot_pdpt_lo:
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 511
-		.quad 0
-	.endr
+	.skip 511 * 8
 boot_pdpt_hi:
-	.rept 510
-		.quad 0
-	.endr
+	.skip 510 * 8
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.quad 0
+	.skip 8
 boot_pd:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.quad i + (PG_PAGE_SIZE | PG_READ_WRITE | PG_PRESENT)
 		.set i, i + 0x200000
 	.endr
+	.quad V2P(g_boot_fast_page_pt) + (PG_READ_WRITE | PG_PRESENT)
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -155,6 +178,13 @@ enable_sse:
 	movl %eax, %cr4
 	ret
 
+enable_tsc:
+	# allow userspace to use RDTSC
+	movl %cr4, %ecx
+	andl $0xFFFFFFFB, %ecx
+	movl %ecx, %cr4
+	ret
+
 initialize_paging:
 	# enable PAE
 	movl %cr4, %ecx
@@ -187,10 +217,11 @@ _start:
 	movl %eax, V2P(bootloader_magic)
 	movl %ebx, V2P(bootloader_info)
 
-	movl $V2P(boot_stack_top), %esp
+	movl $V2P(g_boot_stack_top), %esp
 
 	call check_requirements
 	call enable_sse
+	call enable_tsc
 	call initialize_paging
 
 	# flush gdt and jump to 64 bit
@@ -240,7 +271,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -250,21 +281,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
-	.skip 4
-ap_stack_loaded:
-	.skip 1
+ap_stack_paddr:
+	.skip 8
+ap_stack_vaddr:
+	.skip 8
+ap_prepare_paging:
+	.skip 8
+ap_page_table:
+	.skip 8
+ap_ready:
+	.skip 8
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -273,36 +310,42 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
+	movl AP_REL(ap_stack_paddr), %esp
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
+	leal V2P(enable_tsc),        %ecx; call *%ecx
 	leal V2P(initialize_paging), %ecx; call *%ecx
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_long_mode)
+	ljmpl $0x08, $AP_REL(ap_long_mode)
 
 .code64
 ap_long_mode:
-	# move stack pointer to higher half
-	movl %esp, %esp
-	addq $KERNEL_OFFSET, %rsp
+	movq $ap_higher_half, %rax
+	jmp *%rax
+
+ap_higher_half:
+	movq AP_REL(ap_prepare_paging), %rax
+	call *%rax
+
+	# load AP's initial values
+	movq AP_REL(ap_stack_vaddr), %rsp
+	movq AP_REL(ap_page_table), %rax
+	movq $1, AP_REL(ap_ready)
+	movq %rax, %cr3
 
 	# clear rbp for stacktrace
 	xorq %rbp, %rbp
 
-	xorb %al, %al
 1:	pause
-	cmpb %al, g_ap_startup_done
-	jz 1b
+	cmpb $0, g_ap_startup_done
+	je 1b
 
 	lock incb g_ap_running_count
 
-	# jump to ap_main in higher half
-	movabsq $ap_main, %rcx
-	call *%rcx
-	jmp V2P(system_halt)
+	call ap_main
+	jmp system_halt
 
 ap_gdt:
 	.quad 0x0000000000000000 # null descriptor

kernel/arch/x86_64/interrupts.S

@@ -1,4 +1,4 @@
-.macro pushaq
+.macro intr_header, n
 	pushq %rax
 	pushq %rcx
 	pushq %rdx
@@ -14,10 +14,18 @@
 	pushq %r13
 	pushq %r14
 	pushq %r15
+	testb $3, \n+15*8(%rsp)
+	jz 1f
+	swapgs
+1:	cld
 .endm
 
-.macro popaq
-	popq %r15
+.macro intr_footer, n
+	testb $3, \n+15*8(%rsp)
+	jz 1f
+	call cpp_check_signal
+	swapgs
+1:	popq %r15
 	popq %r14
 	popq %r13
 	popq %r12
@@ -35,8 +43,7 @@
 .endm
 
 isr_stub:
-	pushaq
-
+	intr_header 24
 	movq %cr0, %rax; pushq %rax
 	movq %cr2, %rax; pushq %rax
 	movq %cr3, %rax; pushq %rax
@@ -49,39 +56,33 @@ isr_stub:
 	call cpp_isr_handler
 	addq $32, %rsp
 
-	popaq
+	intr_footer 24
 	addq $16, %rsp
 	iretq
 
 irq_stub:
-	pushaq
+	intr_header 24
+	xorq %rbp, %rbp
 	movq 120(%rsp), %rdi	# irq number
 	call cpp_irq_handler
-	popaq
+	intr_footer 24
 	addq $16, %rsp
 	iretq
 
-.global asm_yield_handler
-asm_yield_handler:
-	pushaq
-	leaq 120(%rsp), %rdi # interrupt stack ptr
-	movq     %rsp,  %rsi # interrupt register ptr
-	call cpp_yield_handler
-	popaq
-	iretq
-
 .global asm_ipi_handler
 asm_ipi_handler:
-	pushaq
+	intr_header 8
+	xorq %rbp, %rbp
 	call cpp_ipi_handler
-	popaq
+	intr_footer 8
 	iretq
 
 .global asm_timer_handler
 asm_timer_handler:
-	pushaq
+	intr_header 8
+	xorq %rbp, %rbp
 	call cpp_timer_handler
-	popaq
+	intr_footer 8
 	iretq
 
 .macro isr n
@@ -140,211 +141,26 @@ isr 29
 isr 30
 isr 31
 
-irq 0
-irq 1
-irq 2
-irq 3
-irq 4
-irq 5
-irq 6
-irq 7
-irq 8
-irq 9
-irq 10
-irq 11
-irq 12
-irq 13
-irq 14
-irq 15
-irq 16
-irq 17
-irq 18
-irq 19
-irq 20
-irq 21
-irq 22
-irq 23
-irq 24
-irq 25
-irq 26
-irq 27
-irq 28
-irq 29
-irq 30
-irq 31
-irq 32
-irq 33
-irq 34
-irq 35
-irq 36
-irq 37
-irq 38
-irq 39
-irq 40
-irq 41
-irq 42
-irq 43
-irq 44
-irq 45
-irq 46
-irq 47
-irq 48
-irq 49
-irq 50
-irq 51
-irq 52
-irq 53
-irq 54
-irq 55
-irq 56
-irq 57
-irq 58
-irq 59
-irq 60
-irq 61
-irq 62
-irq 63
-irq 64
-irq 65
-irq 66
-irq 67
-irq 68
-irq 69
-irq 70
-irq 71
-irq 72
-irq 73
-irq 74
-irq 75
-irq 76
-irq 77
-irq 78
-irq 79
-irq 80
-irq 81
-irq 82
-irq 83
-irq 84
-irq 85
-irq 86
-irq 87
-irq 88
-irq 89
-irq 90
-irq 91
-irq 92
-irq 93
-irq 94
-irq 95
-irq 96
-irq 97
-irq 98
-irq 99
-irq 100
-irq 101
-irq 102
-irq 103
-irq 104
-irq 105
-irq 106
-irq 107
-irq 108
-irq 109
-irq 110
-irq 111
-irq 112
-irq 113
-irq 114
-irq 115
-irq 116
-irq 117
-irq 118
-irq 119
-irq 120
-irq 121
-irq 122
-irq 123
-irq 124
-irq 125
-irq 126
-irq 127
-irq 128
-irq 129
-irq 130
-irq 131
-irq 132
-irq 133
-irq 134
-irq 135
-irq 136
-irq 137
-irq 138
-irq 139
-irq 140
-irq 141
-irq 142
-irq 143
-irq 144
-irq 145
-irq 146
-irq 147
-irq 148
-irq 149
-irq 150
-irq 151
-irq 152
-irq 153
-irq 154
-irq 155
-irq 156
-irq 157
-irq 158
-irq 159
-irq 160
-irq 161
-irq 162
-irq 163
-irq 164
-irq 165
-irq 166
-irq 167
-irq 168
-irq 169
-irq 170
-irq 171
-irq 172
-irq 173
-irq 174
-irq 175
-irq 176
-irq 177
-irq 178
-irq 179
-irq 180
-irq 181
-irq 182
-irq 183
-irq 184
-irq 185
-irq 186
-irq 187
-irq 188
-irq 189
-irq 190
-irq 191
-irq 192
-irq 193
-irq 194
-irq 195
-irq 196
-irq 197
-irq 198
-irq 199
-irq 200
-irq 201
-irq 202
-irq 203
-irq 204
-irq 205
-irq 206
-irq 207
+.irp i,  0,  1,  2,  3,  4,  5,  6,  7,  8,  9, \
+        10, 11, 12, 13, 14, 15, 16, 17, 18, 19, \
+        20, 21, 22, 23, 24, 25, 26, 27, 28, 29, \
+        30, 31, 32, 33, 34, 35, 36, 37, 38, 39, \
+        40, 41, 42, 43, 44, 45, 46, 47, 48, 49, \
+        50, 51, 52, 53, 54, 55, 56, 57, 58, 59, \
+        60, 61, 62, 63, 64, 65, 66, 67, 68, 69, \
+        70, 71, 72, 73, 74, 75, 76, 77, 78, 79, \
+        80, 81, 82, 83, 84, 85, 86, 87, 88, 89, \
+        90, 91, 92, 93, 94, 95, 96, 97, 98, 99, \
+       100,101,102,103,104,105,106,107,108,109, \
+       110,111,112,113,114,115,116,117,118,119, \
+       120,121,122,123,124,125,126,127,128,129, \
+       130,131,132,133,134,135,136,137,138,139, \
+       140,141,142,143,144,145,146,147,148,149, \
+       150,151,152,153,154,155,156,157,158,159, \
+       160,161,162,163,164,165,166,167,168,169, \
+       170,171,172,173,174,175,176,177,178,179, \
+       180,181,182,183,184,185,186,187,188,189, \
+       190,191,192,193,194,195,196,197,198,199, \
+       200,201,202,203,204,205,206,207
+	irq \i
+.endr

kernel/arch/x86_64/linker.ld

@@ -11,20 +11,21 @@ SECTIONS
 	{
 		g_kernel_execute_start = .;
 		*(.multiboot)
+		*(.multiboot2)
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{
@@ -43,6 +44,7 @@ SECTIONS
 	}
 	.bss ALIGN(4K) : AT(ADDR(.bss) - KERNEL_OFFSET)
 	{
+		g_kernel_bss_start = .;
 		*(COMMON)
 		*(.bss)
 		g_kernel_writable_end = .;

kernel/include/kernel/ACPI/ACPI.h

@@ -2,6 +2,7 @@
 
 #include <BAN/Vector.h>
 #include <kernel/ACPI/AML/Namespace.h>
+#include <kernel/ACPI/EmbeddedController.h>
 #include <kernel/ACPI/Headers.h>
 #include <kernel/Memory/Types.h>
 #include <kernel/ThreadBlocker.h>
@@ -30,11 +31,17 @@ namespace Kernel::ACPI
 
 		BAN::ErrorOr<void> initialize_acpi_devices();
 
+		AML::Namespace* acpi_namespace() { return m_namespace; }
+
 		BAN::ErrorOr<void> poweroff();
 		BAN::ErrorOr<void> reset();
 
+		BAN::ErrorOr<void> register_gpe_handler(uint8_t gpe, void (*callback)(void*), void* argument);
+
 		void handle_irq() override;
 
+		BAN::Span<BAN::UniqPtr<EmbeddedController>> embedded_controllers() { return m_embedded_controllers.span(); }
+
 	private:
 		ACPI() = default;
 		BAN::ErrorOr<void> initialize_impl();
@@ -46,6 +53,14 @@ namespace Kernel::ACPI
 
 		BAN::ErrorOr<void> load_aml_tables(BAN::StringView name, bool all);
 
+		BAN::ErrorOr<void> route_interrupt_link_device(const AML::Scope& device, uint64_t& routed_irq_mask);
+
+		BAN::ErrorOr<void> initialize_embedded_controller(const AML::Scope& embedded_controller);
+		BAN::ErrorOr<void> initialize_embedded_controllers();
+
+		BAN::Optional<GAS> find_gpe_block(size_t index);
+		bool enable_gpe(uint8_t gpe);
+
 	private:
 		paddr_t m_header_table_paddr = 0;
 		vaddr_t m_header_table_vaddr = 0;
@@ -64,8 +79,23 @@ namespace Kernel::ACPI
 
 		ThreadBlocker m_event_thread_blocker;
 
+		BAN::Vector<BAN::UniqPtr<EmbeddedController>> m_embedded_controllers;
+
+		struct GPEHandler
+		{
+			bool has_callback { false };
+			union {
+				AML::Reference* method;
+				struct
+				{
+					void (*callback)(void*);
+					void* argument;
+				};
+			};
+		};
+		bool m_has_any_gpes { false };
 		AML::Scope m_gpe_scope;
-		BAN::Array<AML::Reference*, 0xFF> m_gpe_methods { nullptr };
+		BAN::Array<GPEHandler, 0xFF> m_gpe_methods;
 
 		bool m_hardware_reduced { false };
 		AML::Namespace* m_namespace { nullptr };

kernel/include/kernel/ACPI/AML/Namespace.h

@@ -27,11 +27,12 @@ namespace Kernel::ACPI::AML
 
 		BAN::ErrorOr<void> parse(BAN::ConstByteSpan);
 
-		BAN::ErrorOr<Node> evaluate(BAN::StringView);
+		BAN::ErrorOr<Node> evaluate(BAN::StringView path) { return evaluate(Scope {}, path); }
+		BAN::ErrorOr<Node> evaluate(const Scope& scope, BAN::StringView);
 
 		// returns empty scope if object already exited
-		BAN::ErrorOr<Scope> add_named_object(const Scope& scope, const NameString& name_string, Node&& node);
-		BAN::ErrorOr<Scope> add_alias(const Scope& scope, const NameString& name_string, Reference* reference);
+		BAN::ErrorOr<Scope> add_named_object(ParseContext& context, const NameString& name_string, Node&& node);
+		BAN::ErrorOr<Scope> add_alias(ParseContext& scope, const NameString& name_string, Reference* reference);
 
 		BAN::ErrorOr<void> remove_named_object(const Scope& absolute_path);
 
@@ -43,10 +44,13 @@ namespace Kernel::ACPI::AML
 		};
 		BAN::ErrorOr<FindResult> find_named_object(const Scope& scope, const NameString& name_string, bool force_absolute = false);
 
+		BAN::ErrorOr<Scope> find_reference_scope(const Reference* reference);
+
 		BAN::ErrorOr<void> for_each_child(const Scope&, const BAN::Function<BAN::Iteration(BAN::StringView, Reference*)>&);
 		BAN::ErrorOr<void> for_each_child(const Scope&, const BAN::Function<BAN::Iteration(const Scope&, Reference*)>&);
 
 		BAN::ErrorOr<BAN::Vector<Scope>> find_device_with_eisa_id(BAN::StringView eisa_id);
+		BAN::ErrorOr<BAN::Vector<Scope>> find_device_with_eisa_id(BAN::Span<BAN::StringView> eisa_ids);
 
 	private:
 		BAN::ErrorOr<Scope> resolve_path(const Scope& scope, const NameString& name_string);

kernel/include/kernel/ACPI/AML/Node.h

@@ -89,6 +89,9 @@ namespace Kernel::ACPI::AML
 		GAS::AddressSpaceID address_space;
 		uint64_t offset;
 		uint64_t length;
+		alignas(Scope) uint8_t scope_storage[sizeof(Scope)];
+		Scope& scope() { return *reinterpret_cast<Scope*>(scope_storage); }
+		const Scope& scope() const { return *reinterpret_cast<const Scope*>(scope_storage); }
 	};
 
 	struct FieldUnit
@@ -150,6 +153,27 @@ namespace Kernel::ACPI::AML
 		Node(Node&& other) { *this = BAN::move(other); }
 		Node& operator=(Node&&);
 
+		static BAN::ErrorOr<Node> create_string(BAN::StringView string)
+		{
+			const auto* u8_data = reinterpret_cast<const uint8_t*>(string.data());
+			auto result = TRY(create_buffer({ u8_data, string.size() }));
+			result.type = Node::Type::String;
+			return result;
+		}
+
+		static BAN::ErrorOr<Node> create_buffer(BAN::ConstByteSpan buffer)
+		{
+			Node node;
+			node.type = Node::Type::Buffer;
+			node.as.str_buf = static_cast<Buffer*>(kmalloc(sizeof(Buffer) + buffer.size()));
+			if (node.as.str_buf == nullptr)
+				return BAN::Error::from_errno(ENOMEM);
+			node.as.str_buf->ref_count = 1;
+			node.as.str_buf->size = buffer.size();
+			memcpy(node.as.str_buf->bytes, buffer.data(), buffer.size());
+			return node;
+		}
+
 		enum class Type
 		{
 			Uninitialized,
@@ -278,13 +302,17 @@ namespace Kernel::ACPI::AML
 	BAN::ErrorOr<NameString> parse_name_string(BAN::ConstByteSpan& aml_data);
 	BAN::ErrorOr<BAN::ConstByteSpan> parse_pkg(BAN::ConstByteSpan& aml_data);
 
-	BAN::ErrorOr<Node> convert_node(Node&& source, uint8_t conversion, size_t max_length);
+	BAN::ErrorOr<Node> convert_node(Node&& source, uint8_t conversion, uint64_t max_length);
 	BAN::ErrorOr<Node> convert_node(Node&& source, const Node& target);
 
 	BAN::ErrorOr<Node> evaluate_node(const Scope& node_path, const Node& node);
 
 	// If method has no return, it will return <integer 0>
 	BAN::ErrorOr<Node> method_call(const Scope& scope, const Node& method, BAN::Array<Reference*, 7>&& args, uint32_t call_depth = 0);
+	BAN::ErrorOr<Node> method_call(const Scope& scope, const Node& method,
+		Node&& arg0 = {}, Node&& arg1 = {}, Node&& arg2 = {}, Node&& arg3 = {}, Node&& arg4 = {}, Node&& arg5 = {}, Node&& arg6 = {});
+
+	BAN::ErrorOr<void> resolve_package_element(Package::Element& element, bool error_if_not_exists);
 
 }
 

kernel/include/kernel/ACPI/AML/OpRegion.h

@@ -10,7 +10,7 @@ namespace Kernel::ACPI::AML
 	BAN::ErrorOr<void> parse_index_field_op(ParseContext& context);
 	BAN::ErrorOr<void> parse_bank_field_op(ParseContext& context);
 
-	BAN::ErrorOr<Node> convert_from_field_unit(const Node& node, uint8_t conversion, size_t max_length);
+	BAN::ErrorOr<Node> convert_from_field_unit(const Node& node, uint8_t conversion, uint64_t max_length);
 	BAN::ErrorOr<void> store_to_field_unit(const Node& source, const Node& target);
 
 }

kernel/include/kernel/ACPI/EmbeddedController.h

@@ -0,0 +1,69 @@
+#pragma once
+
+#include <BAN/Atomic.h>
+#include <BAN/UniqPtr.h>
+
+#include <kernel/ACPI/AML/Scope.h>
+#include <kernel/Lock/Mutex.h>
+#include <kernel/Thread.h>
+
+namespace Kernel::ACPI
+{
+
+	class EmbeddedController
+	{
+	public:
+		static BAN::ErrorOr<BAN::UniqPtr<EmbeddedController>> create(AML::Scope&& scope, uint16_t command_port, uint16_t data_port, BAN::Optional<uint8_t> gpe);
+		~EmbeddedController();
+
+		BAN::ErrorOr<uint8_t> read_byte(uint8_t offset);
+		BAN::ErrorOr<void> write_byte(uint8_t offset, uint8_t value);
+
+		const AML::Scope& scope() const { return m_scope; }
+
+	private:
+		EmbeddedController(AML::Scope&& scope, uint16_t command_port, uint16_t data_port, bool has_gpe)
+			: m_scope(BAN::move(scope))
+			, m_command_port(command_port)
+			, m_data_port(data_port)
+			, m_has_gpe(has_gpe)
+		{ }
+
+	private:
+		void wait_status_bit(uint8_t bit, uint8_t value);
+
+		uint8_t read_one(uint16_t port);
+		void write_one(uint16_t port, uint8_t value);
+
+		static void handle_gpe_wrapper(void*);
+		void handle_gpe();
+
+		BAN::ErrorOr<void> call_query_method(uint8_t notification);
+
+		void thread_task();
+
+		struct Command
+		{
+			uint8_t command;
+			BAN::Optional<uint8_t> data1;
+			BAN::Optional<uint8_t> data2;
+			uint8_t* response;
+			BAN::Atomic<bool> done;
+		};
+		BAN::ErrorOr<void> send_command(Command& command);
+
+	private:
+		const AML::Scope m_scope;
+		const uint16_t m_command_port;
+		const uint16_t m_data_port;
+		const bool m_has_gpe;
+
+		Mutex m_mutex;
+		ThreadBlocker m_thread_blocker;
+
+		BAN::Optional<Command*> m_queued_command;
+
+		Thread* m_thread { nullptr };
+	};
+
+}

kernel/include/kernel/ACPI/Resource.h

@@ -0,0 +1,179 @@
+#pragma once
+
+#include <BAN/ByteSpan.h>
+#include <BAN/Debug.h>
+#include <BAN/Optional.h>
+
+namespace Kernel::ACPI
+{
+
+	struct ResourceData
+	{
+		enum class Type
+		{
+			IRQ,
+			DMA,
+			IOPort,
+			FixedIOPort,
+			FixedDMA,
+
+			// TODO: large stuff the stuff :)
+		};
+
+		union {
+			struct {
+				uint16_t irq_mask;
+				union {
+					struct {
+						uint8_t edge_triggered : 1;
+						uint8_t                : 2;
+						uint8_t active_low     : 1;
+						uint8_t shared         : 1;
+						uint8_t wake_capable   : 1;
+						uint8_t                : 2;
+					};
+					uint8_t raw;
+				};
+			} irq;
+			struct {
+				uint8_t channel_mask;
+				union {
+					struct {
+						uint8_t type          : 2; // 0: 8 bit, 1: 8 and 16 bit, 2: 16 bit only
+						uint8_t bus_master    : 1;
+						uint8_t               : 2;
+						uint8_t channel_speed : 2; // 0: compatibility, 1: type A, 2: type B, 3: type F
+						uint8_t               : 1;
+					};
+					uint8_t raw;
+				};
+			} dma;
+			struct {
+				uint16_t range_min_base;
+				uint16_t range_max_base;
+				uint8_t base_alignment;
+				uint8_t range_length;
+			} io_port;
+			struct {
+				uint16_t range_base;
+				uint8_t range_length;
+			} fixed_io_port;
+			struct {
+				uint16_t request_line;
+				uint16_t channel;
+				uint8_t transfer_width; // 0: 8 bit, 1: 16 bit, 2: 32 bit, 3: 64 bit, 4: 128 bit
+			} fixed_dma;
+		} as;
+		Type type;
+	};
+
+	class ResourceParser
+	{
+	public:
+		ResourceParser(BAN::ConstByteSpan buffer)
+			: m_buffer(buffer)
+		{}
+
+		BAN::Optional<ResourceData> get_next()
+		{
+			for (;;)
+			{
+				if (m_buffer.empty())
+					return {};
+
+				if (m_buffer[0] & 0x80)
+				{
+					dprintln("Skipping large resource 0x{2H}", m_buffer[0] & 0x7F);
+					const uint16_t length = (m_buffer[2] << 8) | m_buffer[1];
+					if (m_buffer.size() < static_cast<size_t>(3 + length))
+						return {};
+					m_buffer = m_buffer.slice(3 + length);
+					continue;
+				}
+
+				const uint8_t length = m_buffer[0] & 0x07;
+				if (m_buffer.size() < static_cast<size_t>(1 + length))
+					return {};
+
+				BAN::Optional<ResourceData> result;
+				switch ((m_buffer[0] >> 3) & 0x0F)
+				{
+					case 0x04:
+						if (length < 2)
+							break;
+						result = ResourceData {
+							.as = { .irq = {
+								.irq_mask = static_cast<uint16_t>((m_buffer[2] << 8) | m_buffer[1]),
+								.raw = (length >= 3) ? m_buffer[3] : static_cast<uint8_t>(1),
+							}},
+							.type = ResourceData::Type::IRQ,
+						};
+						break;
+					case 0x05:
+						if (length < 2)
+							break;
+						result = ResourceData {
+							.as = { .dma = {
+								.channel_mask = m_buffer[1],
+								.raw = m_buffer[2],
+							}},
+							.type = ResourceData::Type::DMA,
+						};
+						break;
+					case 0x08:
+						if (length < 7)
+							break;
+						result = ResourceData {
+							.as = { .io_port = {
+								.range_min_base = static_cast<uint16_t>(((m_buffer[3] << 8) | m_buffer[2]) & ((m_buffer[1] & 1) ? 0xFFFF : 0x03FF)),
+								.range_max_base = static_cast<uint16_t>(((m_buffer[5] << 8) | m_buffer[4]) & ((m_buffer[1] & 1) ? 0xFFFF : 0x03FF)),
+								.base_alignment = m_buffer[6],
+								.range_length = m_buffer[7],
+							}},
+							.type = ResourceData::Type::IOPort,
+						};
+						break;
+					case 0x09:
+						if (length < 3)
+							break;
+						result = ResourceData {
+							.as = { .fixed_io_port = {
+								.range_base = static_cast<uint16_t>(((m_buffer[2] << 8) | m_buffer[1]) & 0x03FF),
+								.range_length = m_buffer[3],
+							}},
+							.type = ResourceData::Type::FixedIOPort,
+						};
+						break;
+					case 0x0A:
+						if (length < 5)
+							break;
+						result = ResourceData {
+							.as = { .fixed_dma = {
+								.request_line = static_cast<uint16_t>((m_buffer[2] << 8) | m_buffer[1]),
+								.channel      = static_cast<uint16_t>((m_buffer[4] << 8) | m_buffer[3]),
+								.transfer_width = m_buffer[5],
+							}},
+							.type = ResourceData::Type::FixedDMA,
+						};
+						break;
+					case 0x0F:
+						// End tag
+						return {};
+					case 0x06:
+					case 0x07:
+					case 0x0E:
+						dprintln("Skipping short resource 0x{2H}", (m_buffer[0] >> 3) & 0x0F);
+						break;
+				}
+
+				m_buffer = m_buffer.slice(1 + length);
+				if (result.has_value())
+					return result.release_value();
+			}
+		}
+
+	private:
+		BAN::ConstByteSpan m_buffer;
+	};
+
+}

kernel/include/kernel/API/SharedPage.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace Kernel::API
+{
+
+	enum SharedPageFeature : uint32_t
+	{
+		SPF_GETTIME = 1 << 0,
+	};
+
+	struct SharedPage
+	{
+		uint16_t gdt_cpu_offset;
+
+		uint32_t features;
+
+		struct
+		{
+			uint8_t shift;
+			uint64_t mult;
+			uint64_t realtime_seconds;
+		} gettime_shared;
+
+		struct
+		{
+			struct
+			{
+				uint32_t seq;
+				uint64_t last_ns;
+				uint64_t last_tsc;
+			} gettime_local;
+		} cpus[];
+	};
+
+}

kernel/include/kernel/API/Syscall.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <BAN/MacroUtils.h>
+
+#if defined(__x86_64__)
+	#define _kas_instruction "syscall"
+	#define _kas_result rax
+	#define _kas_arguments rdi, rsi, rdx, r10, r8, r9
+	#define _kas_globbers rcx, rdx, rdi, rsi, r8, r9, r10, r11
+#elif defined(__i686__)
+	#define _kas_instruction "int $0xF0"
+	#define _kas_result eax
+	#define _kas_arguments eax, ebx, ecx, edx, esi, edi
+	#define _kas_globbers
+#endif
+
+#define _kas_argument_var(index, value) register long _kas_a##index asm(_ban_stringify(_ban_get(index, _kas_arguments))) = (long)(value);
+#define _kas_dummy_var(index, value) register long _kas_d##index asm(#value);
+#define _kas_input(index, _) "r"(_kas_a##index)
+#define _kas_output(index, _) , "=r"(_kas_d##index)
+#define _kas_globber(_, value) #value
+
+#define _kas_syscall(...) ({ \
+		register long _kas_ret asm(_ban_stringify(_kas_result)); \
+		_ban_for_each(_kas_argument_var, __VA_ARGS__) \
+		_ban_for_each(_kas_dummy_var, _kas_globbers) \
+		asm volatile( \
+			_kas_instruction \
+			: "=r"(_kas_ret) _ban_for_each(_kas_output, _kas_globbers) \
+			: _ban_for_each_comma(_kas_input, __VA_ARGS__) \
+			: "cc", "memory"); \
+		(void)_kas_a0; /* require 1 argument */ \
+		_kas_ret; \
+	})

kernel/include/kernel/APIC.h

@@ -23,6 +23,8 @@ namespace Kernel
 		virtual void broadcast_ipi() override;
 		virtual void enable() override;
 
+		BAN::ErrorOr<uint8_t> reserve_gsi(uint32_t gsi);
+
 		void initialize_timer();
 
 	private:

kernel/include/kernel/Audio/AC97/Controller.h

@@ -0,0 +1,59 @@
+#pragma once
+
+#include <kernel/Audio/Controller.h>
+#include <kernel/Memory/DMARegion.h>
+
+namespace Kernel
+{
+
+	class AC97AudioController final : public AudioController, public Interruptable
+	{
+	public:
+		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
+
+		void handle_irq() override;
+
+	protected:
+		void handle_new_data() override;
+
+		uint32_t get_channels() const override { return 2; }
+		uint32_t get_sample_rate() const override { return 48000; }
+
+		uint32_t get_total_pins() const override { return 1; }
+		uint32_t get_current_pin() const override { return 0; }
+		BAN::ErrorOr<void> set_current_pin(uint32_t pin) override { if (pin != 0) return BAN::Error::from_errno(EINVAL); return {}; }
+
+		BAN::ErrorOr<void> set_volume_mdB(int32_t) override;
+
+	private:
+		AC97AudioController(PCI::Device& pci_device)
+			: m_pci_device(pci_device)
+		{ }
+
+		uint32_t get_volume_data() const;
+
+		BAN::ErrorOr<void> initialize();
+		BAN::ErrorOr<void> initialize_bld();
+		BAN::ErrorOr<void> initialize_interrupts();
+
+		bool queue_samples_to_bld();
+
+	private:
+		static constexpr size_t m_bdl_entries = 32;
+		static constexpr size_t m_samples_per_entry = 0x1000;
+
+		// We only store samples in 2 BDL entries at a time to reduce the amount of samples queued.
+		// This is to reduce latency as you cannot remove data already passed to the BDLs
+		static constexpr size_t m_used_bdl_entries = 2;
+
+		PCI::Device& m_pci_device;
+		BAN::UniqPtr<PCI::BarRegion> m_mixer;
+		BAN::UniqPtr<PCI::BarRegion> m_bus_master;
+
+		BAN::UniqPtr<DMARegion> m_bdl_region;
+
+		uint32_t m_bdl_tail { 0 };
+		uint32_t m_bdl_head { 0 };
+	};
+
+}

kernel/include/kernel/Audio/AC97/Definitions.h

@@ -0,0 +1,15 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace Kernel::AC97
+{
+
+	struct BufferDescriptorListEntry
+	{
+		uint32_t address;
+		uint16_t samples;
+		uint16_t flags; // bit 14: last entry, bit 15: IOC
+	};
+
+}

kernel/include/kernel/Audio/Controller.h

@@ -0,0 +1,58 @@
+#pragma once
+
+#include <kernel/Device/Device.h>
+#include <kernel/Memory/ByteRingBuffer.h>
+#include <kernel/PCI.h>
+
+#include <sys/ioctl.h>
+
+namespace Kernel
+{
+
+	class AudioController : public CharacterDevice
+	{
+	public:
+		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
+
+		dev_t rdev() const override { return m_rdev; }
+		BAN::StringView name() const override { return m_name; }
+
+	protected:
+		AudioController();
+		BAN::ErrorOr<void> initialize();
+
+		virtual void handle_new_data() = 0;
+
+		virtual uint32_t get_channels() const = 0;
+		virtual uint32_t get_sample_rate() const = 0;
+
+		virtual uint32_t get_total_pins() const = 0;
+		virtual uint32_t get_current_pin() const = 0;
+		virtual BAN::ErrorOr<void> set_current_pin(uint32_t) = 0;
+
+		virtual BAN::ErrorOr<void> set_volume_mdB(int32_t) = 0;
+
+		bool can_read_impl() const override { return false; }
+		bool can_write_impl() const override { SpinLockGuard _(m_spinlock); return !m_sample_data->full(); }
+		bool has_error_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
+
+		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
+
+		BAN::ErrorOr<long> ioctl_impl(int cmd, void* arg) override;
+
+	protected:
+		ThreadBlocker m_sample_data_blocker;
+		mutable SpinLock m_spinlock;
+
+		static constexpr size_t m_sample_data_capacity = 1 << 20;
+		BAN::UniqPtr<ByteRingBuffer> m_sample_data;
+
+		snd_volume_info m_volume_info {};
+
+	private:
+		const dev_t m_rdev;
+		char m_name[10] {};
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/AudioFunctionGroup.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include <kernel/Audio/Controller.h>
+#include <kernel/Audio/HDAudio/Controller.h>
+
+namespace Kernel
+{
+
+	class HDAudioController;
+
+	class HDAudioFunctionGroup final : public AudioController
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<HDAudioFunctionGroup>> create(BAN::RefPtr<HDAudioController>, uint8_t cid, HDAudio::AFGNode&&);
+
+		void on_stream_interrupt(uint8_t stream_index);
+
+	protected:
+		// FIXME: allow setting these :D
+		uint32_t get_channels() const override { return 2; }
+		uint32_t get_sample_rate() const override { return 48000; }
+
+		uint32_t get_total_pins() const override;
+		uint32_t get_current_pin() const override;
+		BAN::ErrorOr<void> set_current_pin(uint32_t) override;
+
+		BAN::ErrorOr<void> set_volume_mdB(int32_t) override;
+
+		void handle_new_data() override;
+
+	private:
+		HDAudioFunctionGroup(BAN::RefPtr<HDAudioController> controller, uint8_t cid, HDAudio::AFGNode&& afg_node)
+			: m_controller(controller)
+			, m_afg_node(BAN::move(afg_node))
+			, m_cid(cid)
+		{ }
+		~HDAudioFunctionGroup();
+
+		BAN::ErrorOr<void> initialize();
+		BAN::ErrorOr<void> initialize_stream();
+		BAN::ErrorOr<void> initialize_output();
+
+		BAN::ErrorOr<void> enable_output_path(uint8_t index);
+		BAN::ErrorOr<void> disable_output_path(uint8_t index);
+
+		void reset_stream();
+
+		BAN::ErrorOr<void> recurse_output_paths(const HDAudio::AFGWidget& widget, BAN::Vector<const HDAudio::AFGWidget*>& path);
+
+		uint16_t get_format_data() const;
+
+		size_t bdl_offset() const;
+
+		void queue_bdl_data();
+
+	private:
+		// use 6 512 sample BDL entries
+		// each entry is ~10.7 ms at 48 kHz
+		// -> total buffered audio is 64 ms
+		static constexpr size_t m_bdl_entry_sample_frames = 512;
+		static constexpr size_t m_bdl_entry_count         = 6;
+
+		BAN::RefPtr<HDAudioController> m_controller;
+		const HDAudio::AFGNode m_afg_node;
+		const uint8_t m_cid;
+
+		BAN::Vector<BAN::Vector<const HDAudio::AFGWidget*>> m_output_paths;
+		BAN::Vector<const HDAudio::AFGWidget*> m_output_pins;
+		size_t m_output_path_index { SIZE_MAX };
+
+		uint8_t m_stream_id    { 0xFF };
+		uint8_t m_stream_index { 0xFF };
+		BAN::UniqPtr<DMARegion> m_bdl_region;
+
+		size_t m_bdl_head { 0 };
+		size_t m_bdl_tail { 0 };
+		bool m_stream_running { false };
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/Controller.h

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <kernel/Audio/Controller.h>
+#include <kernel/Audio/HDAudio/Definitions.h>
+#include <kernel/Memory/DMARegion.h>
+
+namespace Kernel
+{
+
+	class HDAudioController : public Interruptable, public BAN::RefCounted<HDAudioController>
+	{
+	public:
+		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
+
+		BAN::ErrorOr<uint32_t> send_command(HDAudio::CORBEntry);
+
+		uint8_t get_stream_index(HDAudio::StreamType type, uint8_t index) const;
+
+		BAN::ErrorOr<uint8_t> allocate_stream_id();
+		void deallocate_stream_id(uint8_t id);
+
+		BAN::ErrorOr<uint8_t> allocate_stream(HDAudio::StreamType type, void* afg);
+		void deallocate_stream(uint8_t index);
+
+		PCI::BarRegion& bar0() { return *m_bar0; }
+
+		bool is_64bit() const { return m_is64bit; }
+
+		void handle_irq() override;
+
+	private:
+		HDAudioController(PCI::Device& pci_device)
+			: m_pci_device(pci_device)
+		{ }
+
+		BAN::ErrorOr<void> initialize();
+		BAN::ErrorOr<void> initialize_ring_buffers();
+
+		BAN::ErrorOr<void> reset_controller();
+
+		BAN::ErrorOr<HDAudio::Codec> initialize_codec(uint8_t codec);
+		BAN::ErrorOr<HDAudio::AFGNode> initialize_node(uint8_t codec, uint8_t node);
+		BAN::ErrorOr<HDAudio::AFGWidget> initialize_widget(uint8_t codec, uint8_t node);
+
+	private:
+		struct RingBuffer
+		{
+			vaddr_t vaddr;
+			uint32_t index;
+			uint32_t size;
+		};
+
+	private:
+		PCI::Device& m_pci_device;
+		BAN::UniqPtr<PCI::BarRegion> m_bar0;
+		bool m_is64bit { false };
+
+		bool m_use_immediate_command { false };
+
+		uint8_t m_output_streams { 0 };
+		uint8_t m_input_streams { 0 };
+		uint8_t m_bidir_streams { 0 };
+		void* m_allocated_streams[30] {};
+
+		// NOTE: stream ids are from 1 to 15
+		uint16_t m_allocated_stream_ids { 0 };
+
+		Mutex m_command_mutex;
+		SpinLock m_rb_lock;
+		ThreadBlocker m_rb_blocker;
+
+		RingBuffer m_corb;
+		RingBuffer m_rirb;
+		BAN::UniqPtr<DMARegion> m_ring_buffer_region;
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/Definitions.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include <BAN/Vector.h>
+
+namespace Kernel::HDAudio
+{
+
+	struct CORBEntry
+	{
+		union {
+			struct {
+				uint32_t data          : 8;
+				uint32_t command       : 12;
+				uint32_t node_index    : 8;
+				uint32_t codec_address : 4;
+			};
+			uint32_t raw;
+		};
+	};
+	static_assert(sizeof(CORBEntry) == sizeof(uint32_t));
+
+	struct BDLEntry
+	{
+		paddr_t address;
+		uint32_t length;
+		uint32_t ioc;
+	};
+	static_assert(sizeof(BDLEntry) == 16);
+
+	struct AFGWidget
+	{
+		enum class Type
+		{
+			OutputConverter,
+			InputConverter,
+			Mixer,
+			Selector,
+			PinComplex,
+			Power,
+			VolumeKnob,
+			BeepGenerator,
+		};
+
+		Type type;
+		uint8_t id;
+
+		union
+		{
+			struct
+			{
+				bool input;
+				bool output;
+				bool display; // HDMI or DP
+				uint32_t config;
+			} pin_complex;
+		};
+
+		struct Amplifier
+		{
+			uint8_t offset;
+			uint8_t num_steps;
+			uint8_t step_size;
+			bool mute;
+		};
+
+		BAN::Optional<Amplifier> output_amplifier;
+
+		BAN::Vector<uint16_t> connections;
+	};
+
+	struct AFGNode
+	{
+		uint8_t id;
+		BAN::Vector<AFGWidget> widgets;
+	};
+
+	struct Codec
+	{
+		uint8_t id;
+		BAN::Vector<AFGNode> nodes;
+	};
+
+	enum class StreamType
+	{
+		Input,
+		Output,
+		Bidirectional,
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/Registers.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace Kernel::HDAudio
+{
+
+	enum Regs : uint8_t
+	{
+		GCAP = 0x00,
+		VMIN = 0x02,
+		VMAJ = 0x03,
+		GCTL = 0x08,
+		STATESTS = 0x0E,
+
+		INTCTL = 0x20,
+		INTSTS = 0x24,
+
+		CORBLBASE = 0x40,
+		CORBUBASE = 0x44,
+		CORBWP    = 0x48,
+		CORBRP    = 0x4A,
+		CORBCTL   = 0x4C,
+		CORBSTS   = 0x4D,
+		CORBSIZE  = 0x4E,
+
+		RIRBLBASE = 0x50,
+		RIRBUBASE = 0x54,
+		RIRBWP    = 0x58,
+		RINTCNT   = 0x5A,
+		RIRBCTL   = 0x5C,
+		RIRBSTS   = 0x5D,
+		RIRBSIZE  = 0x5E,
+
+		ICOI = 0x60,
+		ICII = 0x64,
+		ICIS = 0x68,
+
+		SDCTL   = 0x00,
+		SDSTS   = 0x03,
+		SDLPIB  = 0x04,
+		SDCBL   = 0x08,
+		SDLVI   = 0x0C,
+		SDFIFOD = 0x10,
+		SDFMT   = 0x12,
+		SDBDPL  = 0x18,
+		SDBDPU  = 0x1C,
+	};
+
+}

kernel/include/kernel/BananBootloader.h

@@ -3,7 +3,8 @@
 #include <stdint.h>
 
 #define BANAN_BOOTLOADER_MAGIC	0xD3C60CFF
-#define BANAN_BOOTLOADER_FB_RGB	1
+#define BANAN_BOOTLOADER_FB_RGB		1
+#define BANAN_BOOTLOADER_FB_TEXT	2
 
 struct BananBootFramebufferInfo
 {

kernel/include/kernel/BootInfo.h

@@ -15,6 +15,7 @@ namespace Kernel
 			None,
 			Unknown,
 			RGB,
+			Text,
 		};
 
 		paddr_t  address;
@@ -40,6 +41,12 @@ namespace Kernel
 		Type     type;
 	};
 
+	struct BootModule
+	{
+		paddr_t start;
+		uint64_t size;
+	};
+
 	struct BootInfo
 	{
 		BAN::String     command_line;
@@ -47,6 +54,7 @@ namespace Kernel
 		RSDP            rsdp         {};
 		paddr_t         kernel_paddr {};
 
+		BAN::Vector<BootModule> modules;
 		BAN::Vector<MemoryMapEntry>	memory_map_entries;
 	};
 

kernel/include/kernel/CPUID.h

@@ -81,5 +81,6 @@ namespace CPUID
 	bool has_pge();
 	bool has_pat();
 	bool has_1gib_pages();
+	bool has_invariant_tsc();
 
 }

kernel/include/kernel/Credentials.h

@@ -35,10 +35,8 @@ namespace Kernel
 
 		bool has_egid(gid_t) const;
 
-		BAN::ErrorOr<void> initialize_supplementary_groups();
-
-	private:
-		BAN::ErrorOr<BAN::String> find_username() const;
+		BAN::Span<const gid_t> groups() const { return m_supplementary.span(); }
+		BAN::ErrorOr<void> set_groups(BAN::Span<const gid_t> groups);
 
 	private:
 		uid_t m_ruid, m_euid, m_suid;

kernel/include/kernel/Debug.h

@@ -65,14 +65,20 @@
 #define DEBUG_XHCI 0
 #define DEBUG_USB 0
 #define DEBUG_USB_HID 0
+#define DEBUG_USB_HUB 0
 #define DEBUG_USB_KEYBOARD 0
 #define DEBUG_USB_MOUSE 0
 #define DEBUG_USB_MASS_STORAGE 0
 
+#define DEBUG_HDAUDIO 0
+
 
 namespace Debug
 {
 	void dump_stack_trace();
+	void dump_stack_trace(uintptr_t ip, uintptr_t bp);
+	void dump_qr_code();
+
 	void putchar(char);
 	void print_prefix(const char*, int);
 

kernel/include/kernel/Device/DebugDevice.h

@@ -3,7 +3,7 @@
 namespace Kernel
 {
 
-	class DebugDevice : public CharacterDevice
+	class DebugDevice final : public CharacterDevice
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<DebugDevice>> create(mode_t, uid_t, gid_t);
@@ -24,6 +24,7 @@ namespace Kernel
 		virtual bool can_read_impl() const override { return false; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		const dev_t m_rdev;

kernel/include/kernel/Device/Device.h

@@ -16,7 +16,11 @@ namespace Kernel
 		virtual bool is_partition() const { return false; }
 		virtual bool is_storage_device() const { return false; }
 
-		virtual BAN::ErrorOr<BAN::UniqPtr<MemoryRegion>> mmap_region(PageTable&, off_t offset, size_t len, AddressRange, MemoryRegion::Type, PageTable::flags_t) { (void)offset; (void)len; return BAN::Error::from_errno(EINVAL); }
+		virtual BAN::ErrorOr<BAN::UniqPtr<MemoryRegion>> mmap_region(PageTable&, off_t offset, size_t len, AddressRange, MemoryRegion::Type, PageTable::flags_t, int status_flags)
+		{
+			(void)offset; (void)len; (void)status_flags;
+			return BAN::Error::from_errno(ENOTSUP);
+		}
 
 		virtual dev_t rdev() const override = 0;
 
@@ -28,7 +32,7 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> fsync_impl() final override { return BAN::Error::from_errno(EINVAL); }
 	};
 
-	class BlockDevice : public Device
+	class BlockDevice : public Device, public BAN::Weakable<BlockDevice>
 	{
 	public:
 		virtual BAN::ErrorOr<void> read_blocks(uint64_t first_block, size_t block_count, BAN::ByteSpan) = 0;

kernel/include/kernel/Device/DeviceNumbers.h

@@ -16,11 +16,14 @@ namespace Kernel
 		Debug,
 		Keyboard,
 		Mouse,
+		Joystick,
 		SCSI,
 		NVMeController,
 		NVMeNamespace,
 		Ethernet,
+		Loopback,
 		TmpFS,
+		AudioController,
 	};
 
 }

kernel/include/kernel/Device/FramebufferDevice.h

@@ -10,12 +10,15 @@ namespace Kernel
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<FramebufferDevice>> create_from_boot_framebuffer();
+		static BAN::RefPtr<FramebufferDevice> boot_framebuffer();
 		~FramebufferDevice();
 
 		uint32_t width() const { return m_width; }
 		uint32_t height() const { return m_height; }
 
+		uint32_t get_pixel(uint32_t x, uint32_t y) const;
 		void set_pixel(uint32_t x, uint32_t y, uint32_t rgb);
+		void fill(uint32_t rgb);
 
 		// positive rows -> empty pixels on bottom
 		// negative rows -> empty pixels on top
@@ -25,7 +28,7 @@ namespace Kernel
 		void sync_pixels_linear(uint32_t first_pixel, uint32_t pixel_count);
 		void sync_pixels_rectangle(uint32_t top_right_x, uint32_t top_right_y, uint32_t width, uint32_t height);
 
-		virtual BAN::ErrorOr<BAN::UniqPtr<MemoryRegion>> mmap_region(PageTable&, off_t offset, size_t len, AddressRange, MemoryRegion::Type, PageTable::flags_t) override;
+		virtual BAN::ErrorOr<BAN::UniqPtr<MemoryRegion>> mmap_region(PageTable&, off_t offset, size_t len, AddressRange, MemoryRegion::Type, PageTable::flags_t, int status_flags) override;
 
 		virtual dev_t rdev() const override { return m_rdev; }
 		virtual BAN::StringView name() const override { return m_name.sv(); }
@@ -34,9 +37,12 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 
+		BAN::ErrorOr<long> ioctl_impl(int cmd, void* arg) override;
+
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		FramebufferDevice(mode_t mode, uid_t uid, gid_t gid, dev_t rdev, paddr_t paddr, uint32_t width, uint32_t height, uint32_t pitch, uint8_t bpp);

kernel/include/kernel/Device/NullDevice.h

@@ -5,7 +5,7 @@
 namespace Kernel
 {
 
-	class NullDevice : public CharacterDevice
+	class NullDevice final : public CharacterDevice
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<NullDevice>> create(mode_t, uid_t, gid_t);
@@ -26,6 +26,7 @@ namespace Kernel
 		virtual bool can_read_impl() const override { return false; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		const dev_t m_rdev;

kernel/include/kernel/Device/RandomDevice.h

@@ -3,7 +3,7 @@
 namespace Kernel
 {
 
-	class RandomDevice : public CharacterDevice
+	class RandomDevice final : public CharacterDevice
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<RandomDevice>> create(mode_t, uid_t, gid_t);
@@ -24,6 +24,7 @@ namespace Kernel
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		const dev_t m_rdev;

kernel/include/kernel/Device/ZeroDevice.h

@@ -3,7 +3,7 @@
 namespace Kernel
 {
 
-	class ZeroDevice : public CharacterDevice
+	class ZeroDevice final : public CharacterDevice
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ZeroDevice>> create(mode_t, uid_t, gid_t);
@@ -24,6 +24,7 @@ namespace Kernel
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		const dev_t m_rdev;

kernel/include/kernel/ELF.h

@@ -8,11 +8,18 @@ namespace Kernel::ELF
 
 	struct LoadResult
 	{
-		bool has_interpreter;
+		struct TLS
+		{
+			vaddr_t addr;
+			size_t size;
+		};
+
+		bool open_execfd;
 		vaddr_t entry_point;
+		BAN::Optional<TLS> master_tls;
 		BAN::Vector<BAN::UniqPtr<MemoryRegion>> regions;
 	};
 
-	BAN::ErrorOr<LoadResult> load_from_inode(BAN::RefPtr<Inode>, const Credentials&, PageTable&);
+	BAN::ErrorOr<LoadResult> load_from_inode(BAN::RefPtr<Inode> root, BAN::RefPtr<Inode> inode, const Credentials&, PageTable&);
 
 }

kernel/include/kernel/Epoll.h

@@ -0,0 +1,100 @@
+#pragma once
+
+#include <BAN/HashMap.h>
+#include <kernel/FS/Inode.h>
+
+#include <sys/epoll.h>
+
+namespace Kernel
+{
+
+	class Epoll final : public Inode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<Epoll>> create();
+		~Epoll();
+
+		BAN::ErrorOr<void> ctl(int op, int fd, BAN::RefPtr<Inode> inode, epoll_event event);
+		BAN::ErrorOr<size_t> wait(BAN::Span<epoll_event> events, uint64_t waketime_ns);
+
+		void notify(BAN::RefPtr<Inode> inode, uint32_t event);
+
+	private:
+		Epoll() = default;
+
+	public:
+		ino_t ino() const override         { return 0; }
+		Mode mode() const override         { return { Mode::IRUSR | Mode::IWUSR }; }
+		nlink_t nlink() const override     { return 0; }
+		uid_t uid() const override         { return 0; }
+		gid_t gid() const override         { return 0; }
+		off_t size() const override        { return 0; }
+		timespec atime() const override    { return {}; }
+		timespec mtime() const override    { return {}; }
+		timespec ctime() const override    { return {}; }
+		blksize_t blksize() const override { return PAGE_SIZE; }
+		blkcnt_t blocks() const override   { return 0; }
+		dev_t dev() const override         { return 0; }
+		dev_t rdev() const override        { return 0; }
+
+		bool is_epoll() const override { return true; }
+
+		const FileSystem* filesystem() const override { return nullptr; }
+
+		bool can_read_impl() const override { return false; }
+		bool can_write_impl() const override { return false; }
+		bool has_error_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
+
+		BAN::ErrorOr<void> fsync_impl() override { return {}; }
+
+	private:
+		struct ListenEventList
+		{
+			ListenEventList() = default;
+
+			ListenEventList(const ListenEventList&) = delete;
+			ListenEventList& operator=(const ListenEventList&) = delete;
+
+			ListenEventList(ListenEventList&& other)
+				: events(BAN::move(other.events))
+			{}
+			ListenEventList& operator=(ListenEventList&& other)
+			{
+				events = BAN::move(other.events);
+				return *this;
+			}
+
+			BAN::HashMap<int, epoll_event> events;
+
+			bool has_fd(int fd) const
+			{
+				return events.contains(fd);
+			}
+
+			bool empty() const
+			{
+				return events.empty();
+			}
+
+			BAN::ErrorOr<void> add_fd(int fd, epoll_event event)
+			{
+				TRY(events.insert(fd, event));
+				return {};
+			}
+
+			void remove_fd(int fd)
+			{
+				events.remove(fd);
+			}
+		};
+
+	private:
+		ThreadBlocker m_thread_blocker;
+		SpinLock m_ready_lock;
+		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_ready_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_processing_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, ListenEventList> m_listening_events;
+	};
+
+}

kernel/include/kernel/FS/DevFS/FileSystem.h

@@ -22,6 +22,7 @@ namespace Kernel
 
 		void add_inode(BAN::StringView path, BAN::RefPtr<TmpInode>);
 
+		void initiate_disk_cache_drop();
 		void initiate_sync(bool should_block);
 
 	private:
@@ -37,6 +38,10 @@ namespace Kernel
 		ThreadBlocker m_sync_done;
 		ThreadBlocker m_sync_thread_blocker;
 		volatile bool m_should_sync { false };
+
+		SpinLock m_disk_cache_lock;
+		ThreadBlocker m_disk_cache_thread_blocker;
+		BAN::Atomic<bool> m_should_drop_disk_cache { false };
 	};
 
 }

kernel/include/kernel/FS/EventFD.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <kernel/FS/Inode.h>
+
+namespace Kernel
+{
+
+	class EventFD final : public Inode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<Inode>> create(uint64_t initval, bool semaphore);
+
+		ino_t ino() const override { return 0; }
+		Mode mode() const override { return { Mode::IFCHR | Mode::IRUSR | Mode::IWUSR }; }
+		nlink_t nlink() const override { return ref_count(); }
+		uid_t uid() const override { return 0; }
+		gid_t gid() const override { return 0; }
+		off_t size() const override { return 0; }
+		timespec atime() const override { return {}; }
+		timespec mtime() const override { return {}; }
+		timespec ctime() const override { return {}; }
+		blksize_t blksize() const override { return 8; }
+		blkcnt_t blocks() const override { return 0; }
+		dev_t dev() const override { return 0; }
+		dev_t rdev() const override { return 0; }
+
+		const FileSystem* filesystem() const override { return nullptr; }
+
+	protected:
+		BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
+		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
+		BAN::ErrorOr<void> fsync_impl() final override { return {}; }
+
+		bool can_read_impl() const override { return m_value > 0; }
+		bool can_write_impl() const override { return m_value < UINT64_MAX - 1; }
+		bool has_error_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
+
+	private:
+		EventFD(uint64_t initval, bool is_semaphore)
+			: m_is_semaphore(is_semaphore)
+			, m_value(initval)
+		{ }
+
+	private:
+		const bool m_is_semaphore;
+		uint64_t m_value;
+
+		ThreadBlocker m_thread_blocker;
+	};
+
+}

kernel/include/kernel/FS/Ext2/FileSystem.h

@@ -26,18 +26,32 @@ namespace Kernel
 		class BlockBufferWrapper
 		{
 			BAN_NON_COPYABLE(BlockBufferWrapper);
-			BAN_NON_MOVABLE(BlockBufferWrapper);
 
 		public:
-			BlockBufferWrapper(BAN::Span<uint8_t> buffer, bool& used)
+			BlockBufferWrapper(BAN::Span<uint8_t> buffer, void (*callback)(void*, const uint8_t*), void* argument)
 				: m_buffer(buffer)
-				, m_used(used)
-			{
-				ASSERT(m_used);
-			}
+				, m_callback(callback)
+				, m_argument(argument)
+			{ }
+			BlockBufferWrapper(BlockBufferWrapper&& other) { *this = BAN::move(other); }
 			~BlockBufferWrapper()
 			{
-				m_used = false;
+				if (m_callback == nullptr)
+					return;
+				m_callback(m_argument, m_buffer.data());
+			}
+
+			BlockBufferWrapper& operator=(BlockBufferWrapper&& other)
+			{
+				this->m_buffer = other.m_buffer;
+				this->m_callback = other.m_callback;
+				this->m_argument = other.m_argument;
+
+				other.m_buffer = {};
+				other.m_callback = nullptr;
+				other.m_argument = nullptr;
+
+				return *this;
 			}
 
 			size_t size() const { return m_buffer.size(); }
@@ -53,7 +67,8 @@ namespace Kernel
 
 		private:
 			BAN::Span<uint8_t> m_buffer;
-			bool& m_used;
+			void (*m_callback)(void*, const uint8_t*);
+			void* m_argument;
 		};
 
 	public:
@@ -62,8 +77,6 @@ namespace Kernel
 
 		virtual BAN::RefPtr<Inode> root_inode() override { return m_root_inode; }
 
-		virtual dev_t dev() const override { return m_block_device->rdev(); };
-
 	private:
 		Ext2FS(BAN::RefPtr<BlockDevice> block_device)
 			: m_block_device(block_device)
@@ -81,7 +94,7 @@ namespace Kernel
 		BAN::ErrorOr<void> sync_superblock();
 		BAN::ErrorOr<void> sync_block(uint32_t block);
 
-		BlockBufferWrapper get_block_buffer();
+		BAN::ErrorOr<BlockBufferWrapper> get_block_buffer();
 
 		BAN::ErrorOr<uint32_t> reserve_free_block(uint32_t primary_bgd);
 		BAN::ErrorOr<void> release_block(uint32_t block);
@@ -104,10 +117,13 @@ namespace Kernel
 		{
 		public:
 			BlockBufferManager() = default;
-			BlockBufferWrapper get_buffer();
+			BAN::ErrorOr<BlockBufferWrapper> get_buffer();
 
 			BAN::ErrorOr<void> initialize(size_t block_size);
 
+		private:
+			void destroy_callback(const uint8_t* buffer_ptr);
+
 		private:
 			struct BlockBuffer
 			{
@@ -115,8 +131,20 @@ namespace Kernel
 				bool used { false };
 			};
 
+			struct ThreadInfo
+			{
+				pid_t tid { 0 };
+				size_t buffers { 0 };
+			};
+
 		private:
-			BAN::Array<BlockBuffer, 10> m_buffers;
+			static constexpr size_t max_threads = 8;
+			static constexpr size_t max_buffers_per_thread = 6;
+
+			Mutex m_buffer_mutex;
+			ThreadBlocker m_buffer_blocker;
+			BAN::Array<BlockBuffer, max_threads * max_buffers_per_thread> m_buffers;
+			BAN::Array<ThreadInfo, max_threads> m_thread_infos;
 		};
 
 	private:

kernel/include/kernel/FS/Ext2/Inode.h

@@ -37,6 +37,7 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t) override;
 		virtual BAN::ErrorOr<void> create_directory_impl(BAN::StringView, mode_t, uid_t, gid_t) override;
 		virtual BAN::ErrorOr<void> link_inode_impl(BAN::StringView, BAN::RefPtr<Inode>) override;
+		virtual BAN::ErrorOr<void> rename_inode_impl(BAN::RefPtr<Inode>, BAN::StringView, BAN::StringView) override;
 		virtual BAN::ErrorOr<void> unlink_impl(BAN::StringView) override;
 
 		virtual BAN::ErrorOr<BAN::String> link_target_impl() override;
@@ -46,29 +47,32 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
 		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
+		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t) override;
+		virtual BAN::ErrorOr<void> utimens_impl(const timespec[2]) override;
 		virtual BAN::ErrorOr<void> fsync_impl() override;
 
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		// Returns maximum number of data blocks in use
 		// NOTE: the inode might have more blocks than what this suggests if it has been shrinked
 		uint32_t max_used_data_block_count() const { return size() / blksize(); }
 
-		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block(uint32_t block, uint32_t index, uint32_t depth);
-		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index(uint32_t data_block_index);
+		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block(uint32_t& block, uint32_t index, uint32_t depth, bool allocate);
+		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index(uint32_t data_block_index, bool allocate);
 
 		BAN::ErrorOr<void> link_inode_to_directory(Ext2Inode&, BAN::StringView name);
+		BAN::ErrorOr<void> remove_inode_from_directory(BAN::StringView name, bool cleanup_directory);
 		BAN::ErrorOr<bool> is_directory_empty();
 
 		BAN::ErrorOr<void> cleanup_indirect_block(uint32_t block, uint32_t depth);
 		BAN::ErrorOr<void> cleanup_default_links();
+		BAN::ErrorOr<void> cleanup_data_blocks();
 		BAN::ErrorOr<void> cleanup_from_fs();
 
-		BAN::ErrorOr<uint32_t> allocate_new_block_to_indirect_block(uint32_t& block, uint32_t index, uint32_t depth);
-		BAN::ErrorOr<uint32_t> allocate_new_block(uint32_t data_block_index);
 		BAN::ErrorOr<void> sync();
 
 		uint32_t block_group() const;
@@ -81,6 +85,26 @@ namespace Kernel
 		{}
 		static BAN::ErrorOr<BAN::RefPtr<Ext2Inode>> create(Ext2FS&, uint32_t);
 
+	private:
+		struct ScopedSync
+		{
+			ScopedSync(Ext2Inode& inode)
+				: inode(inode)
+				, inode_info(inode.m_inode)
+			{ }
+
+			~ScopedSync()
+			{
+				if (memcmp(&inode.m_inode, &inode_info, sizeof(Ext2::Inode)) == 0)
+					return;
+				if (auto ret = inode.sync(); ret.is_error())
+					dwarnln("failed to sync inode: {}", ret.error());
+			}
+
+			Ext2Inode& inode;
+			Ext2::Inode inode_info;
+		};
+
 	private:
 		Ext2FS& m_fs;
 		Ext2::Inode m_inode;

kernel/include/kernel/FS/FAT/FileSystem.h

@@ -37,8 +37,6 @@ namespace Kernel
 
 		virtual BAN::RefPtr<Inode> root_inode() override { return m_root_inode; }
 
-		virtual dev_t dev() const override { return m_block_device->rdev(); };
-
 		BAN::ErrorOr<BAN::RefPtr<FATInode>> open_inode(BAN::RefPtr<FATInode> parent, const FAT::DirectoryEntry& entry, uint32_t cluster_index, uint32_t entry_index);
 		BAN::ErrorOr<void> inode_read_cluster(BAN::RefPtr<FATInode>, size_t index, BAN::ByteSpan buffer);
 		blksize_t inode_block_size(BAN::RefPtr<const FATInode>) const;

kernel/include/kernel/FS/FAT/Inode.h

@@ -44,11 +44,13 @@ namespace Kernel
 		//virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		//virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
 		//virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
+		//virtual BAN::ErrorOr<void> utimens_impl(const timespec[2]) override;
 		virtual BAN::ErrorOr<void> fsync_impl() override { return {}; }
 
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		FATInode(FATFS& fs, const FAT::DirectoryEntry& entry, ino_t ino, uint32_t block_count)

kernel/include/kernel/FS/FileSystem.h

@@ -26,8 +26,6 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<FileSystem>> from_block_device(BAN::RefPtr<BlockDevice>);
 
 		virtual BAN::RefPtr<Inode> root_inode() = 0;
-
-		virtual dev_t dev() const = 0;
 	};
 
 }

kernel/include/kernel/FS/Inode.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <BAN/ByteSpan.h>
+#include <BAN/LinkedList.h>
 #include <BAN/RefPtr.h>
 #include <BAN/String.h>
 #include <BAN/StringView.h>
@@ -19,10 +20,9 @@
 namespace Kernel
 {
 
-	class FileSystem;
-
 	class FileBackedRegion;
-	class SharedFileData;
+	class FileSystem;
+	struct SharedFileData;
 
 	class Inode : public BAN::RefCounted<Inode>
 	{
@@ -85,6 +85,7 @@ namespace Kernel
 		virtual dev_t rdev() const = 0;
 
 		virtual bool is_device() const { return false; }
+		virtual bool is_epoll() const { return false; }
 		virtual bool is_pipe() const { return false; }
 		virtual bool is_tty() const { return false; }
 
@@ -96,6 +97,7 @@ namespace Kernel
 		BAN::ErrorOr<void> create_file(BAN::StringView, mode_t, uid_t, gid_t);
 		BAN::ErrorOr<void> create_directory(BAN::StringView, mode_t, uid_t, gid_t);
 		BAN::ErrorOr<void> link_inode(BAN::StringView, BAN::RefPtr<Inode>);
+		BAN::ErrorOr<void> rename_inode(BAN::RefPtr<Inode>, BAN::StringView, BAN::StringView);
 		BAN::ErrorOr<void> unlink(BAN::StringView);
 
 		// Link API
@@ -107,9 +109,12 @@ namespace Kernel
 		BAN::ErrorOr<void> bind(const sockaddr* address, socklen_t address_len);
 		BAN::ErrorOr<void> connect(const sockaddr* address, socklen_t address_len);
 		BAN::ErrorOr<void> listen(int backlog);
-		BAN::ErrorOr<size_t> sendto(BAN::ConstByteSpan message, const sockaddr* address, socklen_t address_len);
-		BAN::ErrorOr<size_t> recvfrom(BAN::ByteSpan buffer, sockaddr* address, socklen_t* address_len);
+		BAN::ErrorOr<size_t> sendmsg(const msghdr& message, int flags);
+		BAN::ErrorOr<size_t> recvmsg(msghdr& message, int flags);
 		BAN::ErrorOr<void> getsockname(sockaddr* address, socklen_t* address_len);
+		BAN::ErrorOr<void> getpeername(sockaddr* address, socklen_t* address_len);
+		BAN::ErrorOr<void> getsockopt(int level, int option, void* value, socklen_t* value_len);
+		BAN::ErrorOr<void> setsockopt(int level, int option, const void* value, socklen_t value_len);
 
 		// General API
 		BAN::ErrorOr<size_t> read(off_t, BAN::ByteSpan buffer);
@@ -117,23 +122,33 @@ namespace Kernel
 		BAN::ErrorOr<void> truncate(size_t);
 		BAN::ErrorOr<void> chmod(mode_t);
 		BAN::ErrorOr<void> chown(uid_t, gid_t);
+		BAN::ErrorOr<void> utimens(const timespec[2]);
 		BAN::ErrorOr<void> fsync();
 
 		// Select/Non blocking API
 		bool can_read() const;
 		bool can_write() const;
 		bool has_error() const;
+		bool has_hungup() const;
 
 		BAN::ErrorOr<long> ioctl(int request, void* arg);
 
+		BAN::ErrorOr<void> add_epoll(class Epoll*);
+		void del_epoll(class Epoll*);
+		void epoll_notify(uint32_t event);
+
+		virtual void on_close(int status_flags) { (void)status_flags; }
+		virtual void on_clone(int status_flags) { (void)status_flags; }
+
 	protected:
 		// Directory API
-		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView)				{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t)		{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t)		{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> create_directory_impl(BAN::StringView, mode_t, uid_t, gid_t)	{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> link_inode_impl(BAN::StringView, BAN::RefPtr<Inode>)			{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> unlink_impl(BAN::StringView)									{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView)							{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t)					{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t)					{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> create_directory_impl(BAN::StringView, mode_t, uid_t, gid_t)				{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> link_inode_impl(BAN::StringView, BAN::RefPtr<Inode>)						{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> rename_inode_impl(BAN::RefPtr<Inode>, BAN::StringView, BAN::StringView)	{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> unlink_impl(BAN::StringView)												{ return BAN::Error::from_errno(ENOTSUP); }
 
 		// Link API
 		virtual BAN::ErrorOr<BAN::String> link_target_impl()				{ return BAN::Error::from_errno(ENOTSUP); }
@@ -144,9 +159,12 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> connect_impl(const sockaddr*, socklen_t)							{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> listen_impl(int)													{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> bind_impl(const sockaddr*, socklen_t)							{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<size_t> sendto_impl(BAN::ConstByteSpan, const sockaddr*, socklen_t)	{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<size_t> recvfrom_impl(BAN::ByteSpan, sockaddr*, socklen_t*)			{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<size_t> recvmsg_impl(msghdr&, int)										{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<size_t> sendmsg_impl(const msghdr&, int)								{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> getsockname_impl(sockaddr*, socklen_t*)							{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*)							{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> getsockopt_impl(int, int, void*, socklen_t*)						{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> setsockopt_impl(int, int, const void*, socklen_t)				{ return BAN::Error::from_errno(ENOTSUP); }
 
 		// General API
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan)		{ return BAN::Error::from_errno(ENOTSUP); }
@@ -154,12 +172,14 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> truncate_impl(size_t)					{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> chmod_impl(mode_t)						{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t)					{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> utimens_impl(const timespec[2])			{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> fsync_impl() = 0;
 
 		// Select/Non blocking API
 		virtual bool can_read_impl() const = 0;
 		virtual bool can_write_impl() const = 0;
 		virtual bool has_error_impl() const = 0;
+		virtual bool has_hungup_impl() const = 0;
 
 		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) { return BAN::Error::from_errno(ENOTSUP); }
 
@@ -168,8 +188,12 @@ namespace Kernel
 
 	private:
 		BAN::WeakPtr<SharedFileData> m_shared_region;
+		SpinLock m_epoll_lock;
+		BAN::LinkedList<class Epoll*> m_epolls;
+		friend class Epoll;
 		friend class FileBackedRegion;
-		friend class SharedFileData;
+		friend class OpenFileDescriptorSet;
+		friend struct SharedFileData;
 		friend class TTY;
 	};
 

kernel/include/kernel/FS/Pipe.h

@@ -2,6 +2,7 @@
 
 #include <BAN/Array.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/ThreadBlocker.h>
 
 namespace Kernel
@@ -13,8 +14,9 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<Inode>> create(const Credentials&);
 
 		virtual bool is_pipe() const override { return true; }
-		void clone_writing();
-		void close_writing();
+
+		void on_close(int status_flags) override;
+		void on_clone(int status_flags) override;
 
 		virtual ino_t ino() const override { return 0; } // FIXME
 		virtual Mode mode() const override { return { Mode::IFIFO | Mode::IRUSR | Mode::IWUSR }; }
@@ -37,9 +39,10 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		virtual BAN::ErrorOr<void> fsync_impl() final override { return {}; }
 
-		virtual bool can_read_impl() const override { return m_buffer_size > 0; }
+		virtual bool can_read_impl() const override { return !m_buffer->empty(); }
 		virtual bool can_write_impl() const override { return true; }
-		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_error_impl() const override { return m_reading_count == 0; }
+		virtual bool has_hungup_impl() const override { return m_writing_count == 0; }
 
 	private:
 		Pipe(const Credentials&);
@@ -52,11 +55,10 @@ namespace Kernel
 		timespec m_ctime {};
 		ThreadBlocker m_thread_blocker;
 
-		BAN::Array<uint8_t, PAGE_SIZE> m_buffer;
-		BAN::Atomic<size_t> m_buffer_size { 0 };
-		size_t m_buffer_tail { 0 };
+		BAN::UniqPtr<ByteRingBuffer> m_buffer;
 
 		BAN::Atomic<uint32_t> m_writing_count { 1 };
+		BAN::Atomic<uint32_t> m_reading_count { 1 };
 	};
 
 }

kernel/include/kernel/FS/ProcFS/Inode.h

@@ -47,6 +47,7 @@ namespace Kernel
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		ProcROProcessInode(Process&, size_t (Process::*)(off_t, BAN::ByteSpan) const, TmpFileSystem&, const TmpInodeInfo&);
@@ -56,6 +57,31 @@ namespace Kernel
 		size_t (Process::*m_callback)(off_t, BAN::ByteSpan) const;
 	};
 
+	class ProcSymlinkProcessInode final : public TmpInode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<ProcSymlinkProcessInode>> create_new(Process& process, BAN::ErrorOr<BAN::String> (Process::*callback)() const, TmpFileSystem&, mode_t);
+		~ProcSymlinkProcessInode() = default;
+
+		virtual uid_t uid() const override { return m_process.credentials().ruid(); }
+		virtual gid_t gid() const override { return m_process.credentials().rgid(); }
+
+	protected:
+		virtual BAN::ErrorOr<BAN::String> link_target_impl() override;
+
+		virtual bool can_read_impl() const override { return false; }
+		virtual bool can_write_impl() const override { return false; }
+		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
+
+	private:
+		ProcSymlinkProcessInode(Process& process, BAN::ErrorOr<BAN::String> (Process::*)() const, TmpFileSystem&, const TmpInodeInfo&);
+
+	private:
+		Process& m_process;
+		BAN::ErrorOr<BAN::String> (Process::*m_callback)() const;
+	};
+
 	class ProcROInode final : public TmpInode
 	{
 	public:
@@ -72,6 +98,7 @@ namespace Kernel
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		ProcROInode(size_t (*callback)(off_t, BAN::ByteSpan), TmpFileSystem&, const TmpInodeInfo&);
@@ -80,4 +107,57 @@ namespace Kernel
 		size_t (*m_callback)(off_t, BAN::ByteSpan);
 	};
 
+	class ProcSymlinkInode final : public TmpInode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<ProcSymlinkInode>> create_new(BAN::ErrorOr<BAN::String> (*)(void*), void (*)(void*), void* data, TmpFileSystem&, mode_t, uid_t, gid_t);
+		~ProcSymlinkInode();
+
+	protected:
+		virtual BAN::ErrorOr<BAN::String> link_target_impl() override;
+
+		virtual bool can_read_impl() const override { return false; }
+		virtual bool can_write_impl() const override { return false; }
+		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
+
+	private:
+		ProcSymlinkInode(BAN::ErrorOr<BAN::String> (*callback)(void*), void (*destructor)(void*), void* data, TmpFileSystem&, const TmpInodeInfo&);
+
+	private:
+		BAN::ErrorOr<BAN::String> (*m_callback)(void*);
+		void (*m_destructor)(void*);
+		void* m_data;
+	};
+
+	class ProcFDDirectoryInode final : public TmpInode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<ProcFDDirectoryInode>> create_new(Process&, TmpFileSystem&, mode_t);
+		~ProcFDDirectoryInode() = default;
+
+		virtual uid_t uid() const override { return m_process.credentials().ruid(); }
+		virtual gid_t gid() const override { return m_process.credentials().rgid(); }
+
+	protected:
+		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView) override;
+		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t) override;
+		virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t) override                 { return BAN::Error::from_errno(EPERM); }
+		virtual BAN::ErrorOr<void> create_directory_impl(BAN::StringView, mode_t, uid_t, gid_t) override            { return BAN::Error::from_errno(EPERM); }
+		virtual BAN::ErrorOr<void> link_inode_impl(BAN::StringView, BAN::RefPtr<Inode>) override                    { return BAN::Error::from_errno(EPERM); }
+		virtual BAN::ErrorOr<void> rename_inode_impl(BAN::RefPtr<Inode>, BAN::StringView, BAN::StringView) override { return BAN::Error::from_errno(EPERM); }
+		virtual BAN::ErrorOr<void> unlink_impl(BAN::StringView) override                                            { return BAN::Error::from_errno(EPERM); }
+
+		virtual bool can_read_impl() const override { return false; }
+		virtual bool can_write_impl() const override { return false; }
+		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
+
+	private:
+		ProcFDDirectoryInode(Process&, TmpFileSystem&, const TmpInodeInfo&);
+
+	private:
+		Process& m_process;
+	};
+
 }

kernel/include/kernel/FS/Socket.h

@@ -51,8 +51,6 @@ namespace Kernel
 			: m_info(info)
 		{}
 
-		BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan buffer) override { return recvfrom_impl(buffer, nullptr, nullptr); }
-		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan buffer) override { return sendto_impl(buffer, nullptr, 0); }
 		BAN::ErrorOr<void> fsync_impl() final override { return {}; }
 
 	private:

kernel/include/kernel/FS/TmpFS/Definitions.h

@@ -28,7 +28,7 @@ namespace Kernel
 		// 1x singly indirect
 		// 1x doubly indirect
 		// 1x triply indirect
-		BAN::Array<paddr_t, 5> block;
+		BAN::Array<size_t, 5> block;
 		static constexpr size_t direct_block_count = 2;
 #elif ARCH(i686)
 		uint32_t __padding;
@@ -36,8 +36,8 @@ namespace Kernel
 		// 1x singly indirect
 		// 1x doubly indirect
 		// 1x triply indirect
-		BAN::Array<paddr_t, 8> block;
-		static constexpr size_t direct_block_count = 5;
+		BAN::Array<size_t, 16> block;
+		static constexpr size_t direct_block_count = 13;
 #else
 	#error
 #endif

kernel/include/kernel/FS/TmpFS/FileSystem.h

@@ -58,7 +58,7 @@ namespace Kernel
 
 		virtual BAN::RefPtr<Inode> root_inode() override { return m_root_inode; }
 
-		virtual dev_t dev() const override { return m_rdev; }
+		dev_t rdev() const { return m_rdev; }
 
 		BAN::ErrorOr<BAN::RefPtr<TmpInode>> open_inode(ino_t ino);
 
@@ -118,16 +118,8 @@ namespace Kernel
 
 	private:
 		InodeLocation find_inode(ino_t ino);
-
 		paddr_t find_block(size_t index);
 
-		template<TmpFuncs::for_each_indirect_paddr_allocating_callback F>
-		BAN::ErrorOr<void> for_each_indirect_paddr_allocating(PageInfo page_info, F callback, size_t depth);
-		template<TmpFuncs::for_each_indirect_paddr_allocating_callback F>
-		BAN::ErrorOr<BAN::Iteration> for_each_indirect_paddr_allocating_internal(PageInfo page_info, F callback, size_t depth);
-
-		paddr_t find_indirect(PageInfo root, size_t index, size_t depth);
-
 	private:
 		const dev_t m_rdev;
 
@@ -146,14 +138,14 @@ namespace Kernel
 		static constexpr size_t max_data_pages =
 			(PAGE_SIZE / sizeof(PageInfo)) *
 			(PAGE_SIZE / sizeof(PageInfo)) *
-			(PAGE_SIZE / sizeof(PageInfo));
+			(PAGE_SIZE / sizeof(PageInfo) - 1);
 
 		// We store inodes in pages with double indirection.
 		// With 64-bit pointers we can store 512^2 pages of inodes
 		// which should be enough for now.
 		// In future this should be dynamically calculated based on maximum
 		// number of pages for this file system.
-		PageInfo m_inode_pages;
+		PageInfo m_inode_pages {};
 		static constexpr size_t first_inode = 1;
 		static constexpr size_t max_inodes =
 			(PAGE_SIZE / sizeof(PageInfo)) *

kernel/include/kernel/FS/TmpFS/Inode.h

@@ -47,14 +47,22 @@ namespace Kernel
 	protected:
 		TmpInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
 
+		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
+		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t) override;
+		virtual BAN::ErrorOr<void> utimens_impl(const timespec[2]) override;
 		virtual BAN::ErrorOr<void> fsync_impl() override { return {}; }
 
 		void sync();
-		void free_all_blocks();
 		virtual BAN::ErrorOr<void> prepare_unlink() { return {}; };
 
+		void free_all_blocks();
+		void free_indirect_blocks(size_t block, uint32_t depth);
+
 		BAN::Optional<size_t> block_index(size_t data_block_index);
+		BAN::Optional<size_t> block_index_from_indirect(size_t block, size_t index, uint32_t depth);
+
 		BAN::ErrorOr<size_t> block_index_with_allocation(size_t data_block_index);
+		BAN::ErrorOr<size_t> block_index_from_indirect_with_allocation(size_t& block, size_t index, uint32_t depth);
 
 	protected:
 		TmpFileSystem& m_fs;
@@ -75,11 +83,11 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
-		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
 
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		TmpFileInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
@@ -97,11 +105,11 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override { return BAN::Error::from_errno(ENODEV); }
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override { return BAN::Error::from_errno(ENODEV); }
 		virtual BAN::ErrorOr<void> truncate_impl(size_t) override { return BAN::Error::from_errno(ENODEV); }
-		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
 
 		virtual bool can_read_impl() const override { return false; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		TmpSocketInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
@@ -115,14 +123,14 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<TmpSymlinkInode>> create_new(TmpFileSystem&, mode_t, uid_t, gid_t, BAN::StringView target);
 		~TmpSymlinkInode();
 
-		BAN::ErrorOr<void> set_link_target(BAN::StringView);
-
 	protected:
-		virtual BAN::ErrorOr<BAN::String> link_target_impl() override;
+		BAN::ErrorOr<BAN::String> link_target_impl() override;
+		BAN::ErrorOr<void> set_link_target_impl(BAN::StringView) override;
 
 		virtual bool can_read_impl() const override { return false; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		TmpSymlinkInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
@@ -148,16 +156,21 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t) override final;
 		virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t) override final;
 		virtual BAN::ErrorOr<void> create_directory_impl(BAN::StringView, mode_t, uid_t, gid_t) override final;
+		virtual BAN::ErrorOr<void> link_inode_impl(BAN::StringView, BAN::RefPtr<Inode>) override final;
+		virtual BAN::ErrorOr<void> rename_inode_impl(BAN::RefPtr<Inode>, BAN::StringView, BAN::StringView) override final;
 		virtual BAN::ErrorOr<void> unlink_impl(BAN::StringView) override;
 
 		virtual bool can_read_impl() const override { return false; }
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
 		template<TmpFuncs::for_each_valid_entry_callback F>
 		void for_each_valid_entry(F callback);
 
+		BAN::ErrorOr<void> unlink_inode(BAN::StringView, bool cleanup);
+
 		friend class TmpInode;
 	};
 

kernel/include/kernel/FS/USTARModule.h

@@ -0,0 +1,11 @@
+#pragma once
+
+#include <kernel/BootInfo.h>
+#include <kernel/FS/Inode.h>
+
+namespace Kernel
+{
+
+	BAN::ErrorOr<bool> unpack_boot_module_into_directory(BAN::RefPtr<Inode>, const BootModule&);
+
+}

kernel/include/kernel/FS/VirtualFileSystem.h

@@ -29,9 +29,6 @@ namespace Kernel
 
 		virtual BAN::RefPtr<Inode> root_inode() override { return m_root_fs->root_inode(); }
 
-		// FIXME:
-		virtual dev_t dev() const override { return 0; }
-
 		BAN::ErrorOr<void> mount(const Credentials&, BAN::StringView, BAN::StringView);
 		BAN::ErrorOr<void> mount(const Credentials&, BAN::RefPtr<FileSystem>, BAN::StringView);
 
@@ -74,13 +71,13 @@ namespace Kernel
 
 		File root_file()
 		{
-			return File(root_inode(), "/"_sv);
+			return File { root_inode(), "/"_sv };
 		}
 
-		BAN::ErrorOr<File> file_from_relative_path(const File& parent, const Credentials&, BAN::StringView, int);
-		BAN::ErrorOr<File> file_from_absolute_path(const Credentials& credentials, BAN::StringView path, int flags)
+		BAN::ErrorOr<File> file_from_relative_path(BAN::RefPtr<Inode> root_inode, const File& parent, const Credentials&, BAN::StringView, int);
+		BAN::ErrorOr<File> file_from_absolute_path(BAN::RefPtr<Inode> root_inode, const Credentials& credentials, BAN::StringView path, int flags)
 		{
-			return file_from_relative_path(root_file(), credentials, path, flags);
+			return file_from_relative_path(root_inode, File { root_inode, "/"_sv }, credentials, path, flags);
 		}
 
 	private:

kernel/include/kernel/GDT.h

@@ -128,6 +128,17 @@ namespace Kernel
 #endif
 		}
 
+#if ARCH(i686)
+		void set_fsbase(uintptr_t addr);
+		void set_gsbase(uintptr_t addr);
+#endif
+
+		static uint16_t cpu_index_offset() { return m_cpu_index_offset; }
+		void set_cpu_index(uint8_t index)
+		{
+			write_entry(m_cpu_index_offset, 0, index, 0xF2, 0x4);
+		}
+
 	private:
 		GDT() = default;
 
@@ -146,11 +157,13 @@ namespace Kernel
 
 	private:
 #if ARCH(x86_64)
-		BAN::Array<SegmentDescriptor, 7> m_gdt; // null, kernel code, kernel data, user code, user data, tss low, tss high
-		static constexpr uint16_t m_tss_offset = 0x28;
+		BAN::Array<SegmentDescriptor, 9> m_gdt; // null, kernel code, kernel data, user code (32 bit), user data, user code (64 bit), cpu-index, tss low, tss high
+		static constexpr uint16_t m_cpu_index_offset = 0x30;
+		static constexpr uint16_t m_tss_offset = 0x38;
 #elif ARCH(i686)
-		BAN::Array<SegmentDescriptor, 7> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, tss
-		static constexpr uint16_t m_tss_offset = 0x30;
+		BAN::Array<SegmentDescriptor, 10> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, cpu-index, tss
+		static constexpr uint16_t m_cpu_index_offset = 0x40;
+		static constexpr uint16_t m_tss_offset = 0x48;
 #endif
 		TaskStateSegment m_tss;
 		const GDTR m_gdtr {

kernel/include/kernel/IDT.h

@@ -18,10 +18,12 @@ namespace Kernel
 
 	constexpr uint8_t IRQ_VECTOR_BASE   = 0x20;
 	constexpr uint8_t IRQ_MSI_BASE      = 0x80;
-	constexpr uint8_t IRQ_SYSCALL       = 0xF0;
-	constexpr uint8_t IRQ_YIELD         = 0xF1;
-	constexpr uint8_t IRQ_IPI           = 0xF2;
-	constexpr uint8_t IRQ_TIMER         = 0xF3;
+	constexpr uint8_t IRQ_MSI_END       = 0xF0;
+#if ARCH(i686)
+	constexpr uint8_t IRQ_SYSCALL       = 0xF0; // hard coded in kernel/API/Syscall.h
+#endif
+	constexpr uint8_t IRQ_IPI           = 0xF1;
+	constexpr uint8_t IRQ_TIMER         = 0xF2;
 
 #if ARCH(x86_64)
 	struct GateDescriptor
@@ -75,7 +77,7 @@ namespace Kernel
 	private:
 		IDT() = default;
 
-		void register_interrupt_handler(uint8_t index, void (*handler)());
+		void register_interrupt_handler(uint8_t index, void (*handler)(), uint8_t ist = 0);
 		void register_syscall_handler(uint8_t index, void (*handler)());
 
 	private:

kernel/include/kernel/Input/InputDevice.h

@@ -15,6 +15,7 @@ namespace Kernel
 		{
 			Mouse,
 			Keyboard,
+			Joystick,
 		};
 
 	public:
@@ -31,7 +32,7 @@ namespace Kernel
 		bool can_read_impl() const override { SpinLockGuard _(m_event_lock); return m_event_count > 0; }
 		bool can_write_impl() const override { return false; }
 		bool has_error_impl() const override { return false; }
-
+		bool has_hungup_impl() const override { return false; }
 
 	private:
 		BAN::ErrorOr<size_t> read_non_block(BAN::ByteSpan);
@@ -64,7 +65,7 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<KeyboardDevice>> create(mode_t mode, uid_t uid, gid_t gid);
 
-		void notify() { m_thread_blocker.unblock(); }
+		void notify();
 
 	private:
 		KeyboardDevice(mode_t mode, uid_t uid, gid_t gid);
@@ -73,6 +74,7 @@ namespace Kernel
 		bool can_read_impl() const override;
 		bool can_write_impl() const override { return false; }
 		bool has_error_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
 
 		BAN::StringView name() const final override { return m_name; }
 		dev_t rdev() const final override { return m_rdev; }
@@ -90,7 +92,7 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<MouseDevice>> create(mode_t mode, uid_t uid, gid_t gid);
 
-		void notify() { m_thread_blocker.unblock(); }
+		void notify();
 
 	private:
 		MouseDevice(mode_t mode, uid_t uid, gid_t gid);
@@ -99,6 +101,7 @@ namespace Kernel
 		bool can_read_impl() const override;
 		bool can_write_impl() const override { return false; }
 		bool has_error_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
 
 		BAN::StringView name() const final override { return m_name; }
 		dev_t rdev() const final override { return m_rdev; }

kernel/include/kernel/Input/PS2/Controller.h

@@ -14,6 +14,15 @@ namespace Kernel::Input
 
 	class PS2Controller
 	{
+	public:
+		enum class DeviceType
+		{
+			None,
+			Unknown,
+			Keyboard,
+			Mouse,
+		};
+
 	public:
 		static BAN::ErrorOr<void> initialize(uint8_t scancode_set);
 		static PS2Controller& get();
@@ -24,10 +33,14 @@ namespace Kernel::Input
 		// Returns true, if byte is used as command, if returns false, byte is meant to device
 		bool handle_command_byte(PS2Device*, uint8_t);
 
+		uint8_t data_port() const { return m_data_port; }
+
 	private:
 		PS2Controller() = default;
 		BAN::ErrorOr<void> initialize_impl(uint8_t scancode_set);
-		BAN::ErrorOr<void> initialize_device(uint8_t, uint8_t scancode_set);
+		BAN::ErrorOr<DeviceType> identify_device(uint8_t);
+
+		void device_initialize_task(void*);
 
 		BAN::ErrorOr<uint8_t> read_byte();
 		BAN::ErrorOr<void> send_byte(uint16_t port, uint8_t byte);
@@ -59,6 +72,9 @@ namespace Kernel::Input
 		};
 
 	private:
+		uint16_t m_command_port { PS2::IOPort::COMMAND };
+		uint16_t m_data_port    { PS2::IOPort::DATA };
+
 		BAN::RefPtr<PS2Device> m_devices[2];
 
 		Mutex m_mutex;

kernel/include/kernel/Input/PS2/Device.h

@@ -22,6 +22,7 @@ namespace Kernel::Input
 
 	protected:
 		PS2Device(PS2Controller&, InputDevice::Type type);
+		virtual ~PS2Device();
 
 	protected:
 		PS2Controller& m_controller;