Kernel: Fix ByteRingBuffer->back()

No need to swap bytes of every 16 bit word in the packet, we can just do
one swap at the return

BAN/include/BAN/HashMap.h

@@ -1,49 +1,154 @@
 #pragma once
 
-#include <BAN/Hash.h>
-#include <BAN/LinkedList.h>
-#include <BAN/Vector.h>
+#include <BAN/HashSet.h>
 
 namespace BAN
 {
 
-	template<typename Key, typename T, typename HASH = BAN::hash<Key>>
+	template<typename HashSetIt, typename HashMap, typename Entry>
+	class HashMapIterator
+	{
+	public:
+		HashMapIterator() = default;
+
+		Entry& operator*()
+		{
+			return const_cast<Entry&>(m_iterator.operator*());
+		}
+		const Entry& operator*() const
+		{
+			return m_iterator.operator*();
+		}
+
+		Entry* operator->()
+		{
+			return const_cast<Entry*>(m_iterator.operator->());
+		}
+		const Entry* operator->() const
+		{
+			return m_iterator.operator->();
+		}
+
+		HashMapIterator& operator++()
+		{
+			++m_iterator;
+			return *this;
+		}
+		HashMapIterator operator++(int)
+		{
+			auto temp = *this;
+			++(*this);
+			return temp;
+		}
+
+		bool operator==(HashMapIterator other) const
+		{
+			return m_iterator == other.m_iterator;
+		}
+		bool operator!=(HashMapIterator other) const
+		{
+			return m_iterator != other.m_iterator;
+		}
+
+	private:
+		explicit HashMapIterator(HashSetIt it)
+			: m_iterator(it)
+		{ }
+
+	private:
+		HashSetIt m_iterator;
+		friend HashMap;
+	};
+
+	namespace detail
+	{
+		template<typename T, typename Key, typename HASH, typename COMP>
+		concept HashMapFindable = requires(const Key& a, const T& b) { COMP()(a, b); HASH()(b); };
+	}
+
+	template<typename Key, typename T, typename HASH = BAN::hash<Key>, typename COMP = BAN::equal<Key>>
 	class HashMap
 	{
 	public:
 		struct Entry
 		{
-			template<typename... Args>
-			Entry(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-				: key(key)
-				, value(forward<Args>(args)...)
-			{}
-
-			template<typename... Args>
-			Entry(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-				: key(BAN::move(key))
-				, value(forward<Args>(args)...)
-			{}
-
-			Key key;
+			const Key key;
 			T value;
+
+			Entry() = delete;
+			Entry& operator=(const Entry&) = delete;
+			Entry& operator=(Entry&&) = delete;
+
+			Entry(const Entry& other)
+				: key(other.key)
+				, value(other.value)
+			{ }
+
+			Entry(Entry&& other)
+				: key(BAN::move(const_cast<Key&>(other.key)))
+				, value(BAN::move(other.value))
+			{ }
+
+			template<typename... Args>
+			Entry(Key&& key, Args&&... args)
+				: key(BAN::move(key))
+				, value(BAN::forward<Args>(args)...)
+			{ }
+		};
+
+		struct EntryHash
+		{
+			constexpr bool operator()(const Key& a)
+			{
+				return HASH()(a);
+			}
+			constexpr bool operator()(const Entry& a)
+			{
+				return HASH()(a.key);
+			}
+		};
+
+		struct EntryComp
+		{
+			constexpr bool operator()(const Entry& a, const Key& b)
+			{
+				return COMP()(a.key, b);
+			}
+			constexpr bool operator()(const Entry& a, const Entry& b)
+			{
+				return COMP()(a.key, b.key);
+			}
 		};
 
 	public:
 		using size_type      = size_t;
 		using key_type       = Key;
 		using value_type     = T;
-		using iterator = IteratorDouble<Entry, Vector, LinkedList, HashMap>;
-		using const_iterator = ConstIteratorDouble<Entry, Vector, LinkedList, HashMap>;
+		using iterator       = HashMapIterator<typename HashSet<Entry, EntryHash, EntryComp>::iterator,       HashMap, Entry>;
+		using const_iterator = HashMapIterator<typename HashSet<Entry, EntryHash, EntryComp>::const_iterator, HashMap, const Entry>;
 
 	public:
 		HashMap() = default;
-		HashMap(const HashMap<Key, T, HASH>&);
-		HashMap(HashMap<Key, T, HASH>&&);
-		~HashMap();
+		~HashMap() { clear(); }
 
-		HashMap<Key, T, HASH>& operator=(const HashMap<Key, T, HASH>&);
-		HashMap<Key, T, HASH>& operator=(HashMap<Key, T, HASH>&&);
+		HashMap(const HashMap& other) { *this = other; }
+		HashMap& operator=(const HashMap& other)
+		{
+			m_hash_set = other.m_hash_set;
+			return *this;
+		}
+
+		HashMap(HashMap&& other) { *this = BAN::move(other); }
+		HashMap& operator=(HashMap&& other)
+		{
+			m_hash_set = BAN::move(other.m_hash_set);
+			return *this;
+		}
+
+		iterator begin()             { return iterator(m_hash_set.begin()); }
+		iterator end()               { return iterator(m_hash_set.end()); }
+		const_iterator begin() const { return const_iterator(m_hash_set.begin()); }
+		const_iterator end() const   { return const_iterator(m_hash_set.end()); }
 
 		ErrorOr<iterator> insert(const Key& key, const T& value)           { return emplace(key, value); }
 		ErrorOr<iterator> insert(const Key& key, T&& value)                { return emplace(key, move(value)); }
@@ -57,263 +162,100 @@ namespace BAN
 
 		template<typename... Args>
 		ErrorOr<iterator> emplace(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-		{ return emplace(Key(key), forward<Args>(args)...); }
+		{ return emplace(Key(key), BAN::forward<Args>(args)...); }
 		template<typename... Args>
-		ErrorOr<iterator> emplace(Key&&, Args&&...) requires is_constructible_v<T, Args...>;
+		ErrorOr<iterator> emplace(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
+		{
+			ASSERT(!contains(key));
+			auto it = TRY(m_hash_set.insert(Entry { BAN::move(key), T(BAN::forward<Args>(args)...) }));
+			return iterator(it);
+		}
 
 		template<typename... Args>
 		ErrorOr<iterator> emplace_or_assign(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-		{ return emplace_or_assign(Key(key), forward<Args>(args)...); }
+		{ return emplace_or_assign(Key(key), BAN::forward<Args>(args)...); }
 		template<typename... Args>
-		ErrorOr<iterator> emplace_or_assign(Key&&, Args&&...) requires is_constructible_v<T, Args...>;
-
-		iterator begin() { return iterator(m_buckets.end(), m_buckets.begin()); }
-		iterator end()   { return iterator(m_buckets.end(), m_buckets.end()); }
-		const_iterator begin() const { return const_iterator(m_buckets.end(), m_buckets.begin()); }
-		const_iterator end() const   { return const_iterator(m_buckets.end(), m_buckets.end()); }
-
-		ErrorOr<void> reserve(size_type);
-
-		void remove(const Key&);
-		void remove(iterator it);
-		void clear();
-
-		T& operator[](const Key&);
-		const T& operator[](const Key&) const;
-
-		iterator find(const Key& key);
-		const_iterator find(const Key& key) const;
-		bool contains(const Key&) const;
-
-		bool empty() const;
-		size_type size() const;
-
-	private:
-		ErrorOr<void> rebucket(size_type);
-		LinkedList<Entry>& get_bucket(const Key&);
-		const LinkedList<Entry>& get_bucket(const Key&) const;
-		Vector<LinkedList<Entry>>::iterator get_bucket_iterator(const Key&);
-		Vector<LinkedList<Entry>>::const_iterator get_bucket_iterator(const Key&) const;
-
-	private:
-		Vector<LinkedList<Entry>> m_buckets;
-		size_type m_size = 0;
-
-		friend iterator;
-	};
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::HashMap(const HashMap<Key, T, HASH>& other)
+		ErrorOr<iterator> emplace_or_assign(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
 		{
-		*this = other;
+			if (auto it = m_hash_set.find(key); it != m_hash_set.end())
+			{
+				const_cast<T&>(it->value) = T(BAN::forward<Args>(args)...);
+				return iterator(it);
 			}
 
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::HashMap(HashMap<Key, T, HASH>&& other)
-	{
-		*this = move(other);
+			auto it = TRY(m_hash_set.insert(Entry { BAN::move(key), T(BAN::forward<Args>(args)...) }));
+			return iterator(it);
 		}
 
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::~HashMap()
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		void remove(const U& key)
 		{
-		clear();
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>& HashMap<Key, T, HASH>::operator=(const HashMap<Key, T, HASH>& other)
-	{
-		clear();
-		m_buckets = other.m_buckets;
-		m_size = other.m_size;
-		return *this;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>& HashMap<Key, T, HASH>::operator=(HashMap<Key, T, HASH>&& other)
-	{
-		clear();
-		m_buckets = move(other.m_buckets);
-		m_size = other.m_size;
-		other.m_size = 0;
-		return *this;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	template<typename... Args>
-	ErrorOr<typename HashMap<Key, T, HASH>::iterator> HashMap<Key, T, HASH>::emplace(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-	{
-		ASSERT(!contains(key));
-		TRY(rebucket(m_size + 1));
-
-		auto bucket_it = get_bucket_iterator(key);
-		TRY(bucket_it->emplace_back(move(key), forward<Args>(args)...));
-		m_size++;
-
-		return iterator(m_buckets.end(), bucket_it, prev(bucket_it->end(), 1));
-	}
-
-	template<typename Key, typename T, typename HASH>
-	template<typename... Args>
-	ErrorOr<typename HashMap<Key, T, HASH>::iterator> HashMap<Key, T, HASH>::emplace_or_assign(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-	{
-		if (empty())
-			return emplace(move(key), forward<Args>(args)...);
-
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto entry_it = bucket_it->begin(); entry_it != bucket_it->end(); entry_it++)
-		{
-			if (entry_it->key != key)
-				continue;
-			entry_it->value = T(forward<Args>(args)...);
-			return iterator(m_buckets.end(), bucket_it, entry_it);
-		}
-
-		return emplace(move(key), forward<Args>(args)...);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	ErrorOr<void> HashMap<Key, T, HASH>::reserve(size_type size)
-	{
-		TRY(rebucket(size));
-		return {};
-	}
-
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::remove(const Key& key)
-	{
-		auto it = find(key);
-		if (it != end())
+			if (auto it = find(key); it != end())
 				remove(it);
 		}
 
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::remove(iterator it)
+		iterator remove(iterator it)
 		{
-		it.outer_current()->remove(it.inner_current());
-		m_size--;
+			return iterator(m_hash_set.remove(it.m_iterator));
 		}
 
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::clear()
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		iterator find(const U& key)
 		{
-		m_buckets.clear();
-		m_size = 0;
+			return iterator(m_hash_set.find(key));
 		}
 
-	template<typename Key, typename T, typename HASH>
-	T& HashMap<Key, T, HASH>::operator[](const Key& key)
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		const_iterator find(const U& key) const
 		{
-		ASSERT(!empty());
-		auto& bucket = get_bucket(key);
-		for (Entry& entry : bucket)
-			if (entry.key == key)
-				return entry.value;
-		ASSERT_NOT_REACHED();
+			return const_iterator(m_hash_set.find(key));
 		}
 
-	template<typename Key, typename T, typename HASH>
-	const T& HashMap<Key, T, HASH>::operator[](const Key& key) const
+		void clear()
 		{
-		ASSERT(!empty());
-		const auto& bucket = get_bucket(key);
-		for (const Entry& entry : bucket)
-			if (entry.key == key)
-				return entry.value;
-		ASSERT_NOT_REACHED();
+			m_hash_set.clear();
 		}
 
-	template<typename Key, typename T, typename HASH>
-	typename HashMap<Key, T, HASH>::iterator HashMap<Key, T, HASH>::find(const Key& key)
+		ErrorOr<void> reserve(size_type size)
 		{
-		if (empty())
-			return end();
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto it = bucket_it->begin(); it != bucket_it->end(); it++)
-			if (it->key == key)
-				return iterator(m_buckets.end(), bucket_it, it);
-		return end();
+			return m_hash_set.reserve(size);
 		}
 
-	template<typename Key, typename T, typename HASH>
-	typename HashMap<Key, T, HASH>::const_iterator HashMap<Key, T, HASH>::find(const Key& key) const
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		T& operator[](const U& key)
 		{
-		if (empty())
-			return end();
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto it = bucket_it->begin(); it != bucket_it->end(); it++)
-			if (it->key == key)
-				return const_iterator(m_buckets.end(), bucket_it, it);
-		return end();
+			return find(key)->value;
 		}
 
-	template<typename Key, typename T, typename HASH>
-	bool HashMap<Key, T, HASH>::contains(const Key& key) const
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		const T& operator[](const U& key) const
+		{
+			return find(key)->value;
+		}
+
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		bool contains(const U& key) const
 		{
 			return find(key) != end();
 		}
 
-	template<typename Key, typename T, typename HASH>
-	bool HashMap<Key, T, HASH>::empty() const
+		size_type capacity() const
 		{
-		return m_size == 0;
+			return m_hash_set.capacity();
 		}
 
-	template<typename Key, typename T, typename HASH>
-	typename HashMap<Key, T, HASH>::size_type HashMap<Key, T, HASH>::size() const
+		size_type size() const
 		{
-		return m_size;
+			return m_hash_set.size();
 		}
 
-	template<typename Key, typename T, typename HASH>
-	ErrorOr<void> HashMap<Key, T, HASH>::rebucket(size_type bucket_count)
+		bool empty() const
 		{
-		if (m_buckets.size() >= bucket_count)
-			return {};
-
-		size_type new_bucket_count = BAN::Math::max<size_type>(bucket_count, m_buckets.size() * 2);
-		Vector<LinkedList<Entry>> new_buckets;
-		TRY(new_buckets.resize(new_bucket_count));
-
-		for (auto& bucket : m_buckets)
-		{
-			for (auto it = bucket.begin(); it != bucket.end();)
-			{
-				size_type new_bucket_index = HASH()(it->key) % new_buckets.size();
-				it = bucket.move_element_to_other_linked_list(new_buckets[new_bucket_index], new_buckets[new_bucket_index].end(), it);
-			}
+			return m_hash_set.empty();
 		}
 
-		m_buckets = move(new_buckets);
-		return {};
-	}
-
-	template<typename Key, typename T, typename HASH>
-	LinkedList<typename HashMap<Key, T, HASH>::Entry>& HashMap<Key, T, HASH>::get_bucket(const Key& key)
-	{
-		return *get_bucket_iterator(key);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	const LinkedList<typename HashMap<Key, T, HASH>::Entry>& HashMap<Key, T, HASH>::get_bucket(const Key& key) const
-	{
-		return *get_bucket_iterator(key);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	Vector<LinkedList<typename HashMap<Key, T, HASH>::Entry>>::iterator HashMap<Key, T, HASH>::get_bucket_iterator(const Key& key)
-	{
-		ASSERT(!m_buckets.empty());
-		auto index = HASH()(key) % m_buckets.size();
-		return next(m_buckets.begin(), index);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	Vector<LinkedList<typename HashMap<Key, T, HASH>::Entry>>::const_iterator HashMap<Key, T, HASH>::get_bucket_iterator(const Key& key) const
-	{
-		ASSERT(!m_buckets.empty());
-		auto index = HASH()(key) % m_buckets.size();
-		return next(m_buckets.begin(), index);
-	}
+	private:
+		HashSet<Entry, EntryHash, EntryComp> m_hash_set;
+	};
 
 }

BAN/include/BAN/HashSet.h

@@ -2,198 +2,358 @@
 
 #include <BAN/Errors.h>
 #include <BAN/Hash.h>
-#include <BAN/Iterators.h>
-#include <BAN/LinkedList.h>
 #include <BAN/Math.h>
 #include <BAN/Move.h>
-#include <BAN/Vector.h>
+#include <BAN/New.h>
 
 namespace BAN
 {
 
-	template<typename T, typename HASH = hash<T>>
+	template<typename HashSet, typename Bucket, typename T>
+	class HashSetIterator
+	{
+	public:
+		HashSetIterator() = default;
+
+		const T& operator*() const
+		{
+			ASSERT(m_bucket);
+			return *m_bucket->element();
+		}
+
+		const T* operator->() const
+		{
+			ASSERT(m_bucket);
+			return m_bucket->element();
+		}
+
+		HashSetIterator& operator++()
+		{
+			ASSERT(m_bucket);
+			m_bucket++;
+			skip_to_valid_bucket();
+			return *this;
+		}
+		HashSetIterator operator++(int)
+		{
+			auto temp = *this;
+			++(*this);
+			return temp;
+		}
+
+		bool operator==(HashSetIterator other) const
+		{
+			return m_bucket == other.m_bucket;
+		}
+		bool operator!=(HashSetIterator other) const
+		{
+			return m_bucket != other.m_bucket;
+		}
+
+	private:
+		explicit HashSetIterator(Bucket* bucket)
+			: m_bucket(bucket)
+		{
+			if (m_bucket != nullptr)
+				skip_to_valid_bucket();
+		}
+
+		void skip_to_valid_bucket()
+		{
+			while (m_bucket->state != Bucket::USED && !m_bucket->end)
+				m_bucket++;
+			if (m_bucket->end)
+				m_bucket = nullptr;
+		}
+
+	private:
+		Bucket* m_bucket { nullptr };
+		friend HashSet;
+	};
+
+	namespace detail
+	{
+		template<typename T, typename U, typename HASH, typename COMP>
+		concept HashSetFindable = requires(const U& a, const T& b) { COMP()(a, b); HASH()(b); };
+	}
+
+	template<typename T, typename HASH = BAN::hash<T>, typename COMP = BAN::equal<T>>
 	class HashSet
 	{
+	private:
+		struct Bucket
+		{
+			static constexpr uint8_t UNUSED  = 0;
+			static constexpr uint8_t USED    = 1;
+			static constexpr uint8_t REMOVED = 2;
+
+			alignas(T) uint8_t storage[sizeof(T)];
+			hash_t hash;
+			uint8_t state       : 2;
+			uint8_t chain_start : 1;
+			uint8_t end         : 1;
+
+			T* element() { return reinterpret_cast<T*>(storage); }
+			const T* element() const { return reinterpret_cast<const T*>(storage); }
+		};
+
 	public:
 		using value_type     = T;
 		using size_type      = size_t;
-		using iterator = IteratorDouble<T, Vector, LinkedList, HashSet>;
-		using const_iterator = ConstIteratorDouble<T, Vector, LinkedList, HashSet>;
+		using iterator       = HashSetIterator<HashSet,       Bucket,       T>;
+		using const_iterator = HashSetIterator<HashSet, const Bucket, const T>;
 
 	public:
 		HashSet() = default;
-		HashSet(const HashSet&);
-		HashSet(HashSet&&);
+		~HashSet() { clear(); }
 
-		HashSet& operator=(const HashSet&);
-		HashSet& operator=(HashSet&&);
-
-		ErrorOr<void> insert(const T&);
-		ErrorOr<void> insert(T&&);
-		void remove(const T&);
-		void clear();
-
-		ErrorOr<void> reserve(size_type);
-
-		iterator begin() { return iterator(m_buckets.end(), m_buckets.begin()); }
-		iterator end()   { return iterator(m_buckets.end(), m_buckets.end()); }
-		const_iterator begin() const { return const_iterator(m_buckets.end(), m_buckets.begin()); }
-		const_iterator end() const   { return const_iterator(m_buckets.end(), m_buckets.end()); }
-
-		bool contains(const T&) const;
-
-		size_type size() const;
-		bool empty() const;
-
-	private:
-		ErrorOr<void> rebucket(size_type);
-		LinkedList<T>& get_bucket(const T&);
-		const LinkedList<T>& get_bucket(const T&) const;
-
-	private:
-		Vector<LinkedList<T>> m_buckets;
-		size_type m_size = 0;
-	};
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>::HashSet(const HashSet& other)
-		: m_buckets(other.m_buckets)
-		, m_size(other.m_size)
-	{
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>::HashSet(HashSet&& other)
-		: m_buckets(move(other.m_buckets))
-		, m_size(other.m_size)
-	{
-		other.clear();
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>& HashSet<T, HASH>::operator=(const HashSet& other)
+		HashSet(const HashSet& other) { *this = other; }
+		HashSet& operator=(const HashSet& other)
 		{
 			clear();
+
+			MUST(reserve(other.size()));
+			for (auto& bucket : other)
+				MUST(insert(bucket));
+
+			return *this;
+		}
+
+		HashSet(HashSet&& other) { *this = BAN::move(other); }
+		HashSet& operator=(HashSet&& other)
+		{
+			clear();
+
 			m_buckets  = other.m_buckets;
+			m_capacity = other.m_capacity;
 			m_size     = other.m_size;
+			m_removed  = other.m_removed;
+
+			other.m_buckets  = nullptr;
+			other.m_capacity = 0;
+			other.m_size     = 0;
+			other.m_removed  = 0;
+
 			return *this;
 		}
 
-	template<typename T, typename HASH>
-	HashSet<T, HASH>& HashSet<T, HASH>::operator=(HashSet&& other)
+		iterator begin()             { return iterator(m_buckets); }
+		iterator end()               { return iterator(nullptr); }
+		const_iterator begin() const { return const_iterator(m_buckets); }
+		const_iterator end() const   { return const_iterator(nullptr); }
+
+		ErrorOr<iterator> insert(const T& value)
 		{
-		clear();
-		m_buckets = move(other.m_buckets);
-		m_size = other.m_size;
-		other.clear();
-		return *this;
+			return insert(T(value));
 		}
 
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::insert(const T& key)
+		ErrorOr<iterator> insert(T&& value)
 		{
-		return insert(move(T(key)));
+			if (should_rehash_with_size(m_size + 1))
+				TRY(rehash(m_size * 2));
+			return insert_impl(BAN::move(value), HASH()(value));
 		}
 
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::insert(T&& key)
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		void remove(const U& value)
 		{
-		if (!empty() && get_bucket(key).contains(key))
-			return {};
-
-		TRY(rebucket(m_size + 1));
-		TRY(get_bucket(key).push_back(move(key)));
-		m_size++;
-		return {};
+			if (auto it = find(value); it != end())
+				remove(it);
 		}
 
-	template<typename T, typename HASH>
-	void HashSet<T, HASH>::remove(const T& key)
+		iterator remove(iterator it)
 		{
-		if (empty()) return;
-		auto& bucket = get_bucket(key);
-		for (auto it = bucket.begin(); it != bucket.end(); it++)
-		{
-			if (*it == key)
-			{
-				bucket.remove(it);
+			auto& bucket = *it.m_bucket;
+			bucket.element()->~T();
+			bucket.state = Bucket::REMOVED;
 			m_size--;
-				break;
-			}
-		}
+			m_removed++;
+			return iterator(&bucket);
 		}
 
-	template<typename T, typename HASH>
-	void HashSet<T, HASH>::clear()
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		iterator find(const U& value)
 		{
-		m_buckets.clear();
+			return iterator(const_cast<Bucket*>(find_impl(value).m_bucket));
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		const_iterator find(const U& value) const
+		{
+			return find_impl(value);
+		}
+
+		void clear()
+		{
+			if (m_buckets == nullptr)
+				return;
+
+			for (size_type i = 0; i < m_capacity; i++)
+				if (m_buckets[i].state == Bucket::USED)
+					m_buckets[i].element()->~T();
+
+			BAN::deallocator(m_buckets);
+			m_buckets  = nullptr;
+			m_capacity = 0;
 			m_size     = 0;
+			m_removed  = 0;
 		}
 
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::reserve(size_type size)
+		ErrorOr<void> reserve(size_type size)
 		{
-		TRY(rebucket(size));
+			if (should_rehash_with_size(size))
+				TRY(rehash(size * 2));
 			return {};
 		}
 
-	template<typename T, typename HASH>
-	bool HashSet<T, HASH>::contains(const T& key) const
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		bool contains(const U& value) const
 		{
-		if (empty()) return false;
-		return get_bucket(key).contains(key);
+			return find(value) != end();
 		}
 
-	template<typename T, typename HASH>
-	typename HashSet<T, HASH>::size_type HashSet<T, HASH>::size() const
+		size_type capacity() const
+		{
+			return m_capacity;
+		}
+
+		size_type size() const
 		{
 			return m_size;
 		}
 
-	template<typename T, typename HASH>
-	bool HashSet<T, HASH>::empty() const
+		bool empty() const
 		{
 			return m_size == 0;
 		}
 
-	template<typename T, typename HASH>
-	ErrorOr<void> HashSet<T, HASH>::rebucket(size_type bucket_count)
+	private:
+		ErrorOr<void> rehash(size_type new_capacity)
 		{
-		if (m_buckets.size() >= bucket_count)
-			return {};
+			new_capacity = BAN::Math::max<size_t>(16, BAN::Math::max(new_capacity, m_size + 1));
+			new_capacity = BAN::Math::round_up_to_power_of_two(new_capacity);
 
-		size_type new_bucket_count = Math::max<size_type>(bucket_count, m_buckets.size() * 2);
-		Vector<LinkedList<T>> new_buckets;
-		if (new_buckets.resize(new_bucket_count).is_error())
-			return Error::from_errno(ENOMEM);
+			void* new_buckets = BAN::allocator((new_capacity + 1) * sizeof(Bucket));
+			if (new_buckets == nullptr)
+				return BAN::Error::from_errno(ENOMEM);
+			memset(new_buckets, 0, (new_capacity + 1) * sizeof(Bucket));
 
-		for (auto& bucket : m_buckets)
+			Bucket* old_buckets = m_buckets;
+			const size_type old_capacity = m_capacity;
+
+			m_buckets  = static_cast<Bucket*>(new_buckets);
+			m_capacity = new_capacity;
+			m_size     = 0;
+			m_removed  = 0;
+
+			for (size_type i = 0; i < old_capacity; i++)
 			{
-			for (auto it = bucket.begin(); it != bucket.end();)
-			{
-				size_type new_bucket_index = HASH()(*it) % new_buckets.size();
-				it = bucket.move_element_to_other_linked_list(new_buckets[new_bucket_index], new_buckets[new_bucket_index].end(), it);
-			}
+				auto& old_bucket = old_buckets[i];
+				if (old_bucket.state != Bucket::USED)
+					continue;
+				insert_impl(BAN::move(*old_bucket.element()), old_bucket.hash);
+				old_bucket.element()->~T();
 			}
 
-		m_buckets = move(new_buckets);
+			m_buckets[m_capacity].end = true;
+
+			BAN::deallocator(old_buckets);
+
 			return {};
 		}
 
-	template<typename T, typename HASH>
-	LinkedList<T>& HashSet<T, HASH>::get_bucket(const T& key)
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		const_iterator find_impl(const U& value) const
 		{
-		ASSERT(!m_buckets.empty());
-		size_type index = HASH()(key) % m_buckets.size();
-		return m_buckets[index];
+			if (m_capacity == 0)
+				return end();
+
+			bool first = true;
+			const hash_t orig_hash = HASH()(value);
+			for (auto hash = orig_hash;; hash = get_next_hash_in_chain(hash, orig_hash), first = false)
+			{
+				auto& bucket = m_buckets[hash & (m_capacity - 1)];
+				if (bucket.state == Bucket::USED && bucket.hash == orig_hash && COMP()(*bucket.element(), value))
+					return const_iterator(&bucket);
+				if (bucket.state == Bucket::UNUSED)
+					return end();
+				if (!first && bucket.chain_start)
+					return end();
+			}
 		}
 
-	template<typename T, typename HASH>
-	const LinkedList<T>& HashSet<T, HASH>::get_bucket(const T& key) const
+		iterator insert_impl(T&& value, hash_t orig_hash)
 		{
-		ASSERT(!m_buckets.empty());
-		size_type index = HASH()(key) % m_buckets.size();
-		return m_buckets[index];
+			ASSERT(!should_rehash_with_size(m_size + 1));
+
+			Bucket* target = nullptr;
+
+			bool first = true;
+			for (auto hash = orig_hash;; hash = get_next_hash_in_chain(hash, orig_hash), first = false)
+			{
+				auto& bucket = m_buckets[hash & (m_capacity - 1)];
+
+				if (!first)
+					bucket.chain_start = false;
+
+				if (bucket.state == Bucket::USED)
+				{
+					if (bucket.hash != orig_hash || !COMP()(*bucket.element(), value))
+						continue;
+					target = &bucket;
+					break;
 				}
 
+				if (target == nullptr)
+					target = &bucket;
+
+				if (bucket.state == Bucket::UNUSED)
+					break;
+			}
+
+			switch (target->state)
+			{
+				case Bucket::USED:
+					target->element()->~T();
+					break;
+				case Bucket::REMOVED:
+					m_removed--;
+					[[fallthrough]];
+				case Bucket::UNUSED:
+					m_size++;
+					break;
+			}
+
+			target->chain_start = first && target->state == Bucket::UNUSED;
+			target->hash = orig_hash;
+			target->state = Bucket::USED;
+
+			new (target->element()) T(BAN::move(value));
+
+			return iterator(target);
+		}
+
+		bool should_rehash_with_size(size_type size) const
+		{
+			if (m_capacity < 16)
+				return true;
+			if (size + m_removed > m_capacity / 4 * 3)
+				return true;
+			return false;
+		}
+
+		hash_t get_next_hash_in_chain(hash_t prev_hash, hash_t orig_hash) const
+		{
+			// TODO: does this even provide better performance than `return prev_hash + 1`
+			//       when using "good" hash functions
+			return prev_hash * 1103515245 + (orig_hash | 1);
+		}
+
+	private:
+		Bucket* m_buckets { nullptr };
+		size_type m_capacity { 0 };
+		size_type m_size     { 0 };
+		size_type m_removed  { 0 };
+	};
+
 }

BAN/include/BAN/Math.h

@@ -65,6 +65,22 @@ namespace BAN::Math
 		return (x & (x - 1)) == 0;
 	}
 
+	template<integral T> requires(sizeof(T) <= 8)
+	inline constexpr T round_up_to_power_of_two(T x)
+	{
+		x--;
+		x |= x >> 1;
+		x |= x >> 2;
+		x |= x >> 4;
+		if constexpr(sizeof(T) >= 2)
+			x |= x >> 8;
+		if constexpr(sizeof(T) >= 4)
+			x |= x >> 16;
+		if constexpr(sizeof(T) >= 8)
+			x |= x >> 32;
+		return x + 1;
+	}
+
 	template<integral T>
 	__attribute__((always_inline))
 	inline constexpr bool will_multiplication_overflow(T a, T b)

BAN/include/BAN/RefPtr.h

@@ -2,9 +2,9 @@
 
 #include <BAN/Atomic.h>
 #include <BAN/Errors.h>
+#include <BAN/Hash.h>
 #include <BAN/Move.h>
 #include <BAN/NoCopyMove.h>
-#include <stdint.h>
 
 namespace BAN
 {
@@ -129,14 +129,9 @@ namespace BAN
 			return *this;
 		}
 
-		T* ptr() { ASSERT(!empty()); return m_pointer; }
-		const T* ptr() const { ASSERT(!empty()); return m_pointer; }
-
-		T& operator*() { return *ptr(); }
-		const T& operator*() const { return *ptr(); }
-
-		T* operator->() { return ptr(); }
-		const T* operator->() const { return ptr(); }
+		T* ptr() const { return m_pointer; }
+		T& operator*() const { ASSERT(!empty()); return *ptr(); }
+		T* operator->() const { ASSERT(!empty()); return ptr(); }
 
 		bool operator==(RefPtr other) const { return m_pointer == other.m_pointer; }
 		bool operator!=(RefPtr other) const { return m_pointer != other.m_pointer; }
@@ -158,4 +153,13 @@ namespace BAN
 		friend class RefPtr;
 	};
 
+	template<typename T>
+	struct hash<RefPtr<T>>
+	{
+		constexpr hash_t operator()(const RefPtr<T>& ptr) const
+		{
+			return hash<T*>()(ptr.ptr());
+		}
+	};
+
 }

BAN/include/BAN/UniqPtr.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <BAN/Errors.h>
+#include <BAN/Hash.h>
 #include <BAN/NoCopyMove.h>
 
 namespace BAN
@@ -53,32 +54,12 @@ namespace BAN
 			return *this;
 		}
 
-		T& operator*()
-		{
-			ASSERT(m_pointer);
-			return *m_pointer;
-		}
+		T* ptr() const { return m_pointer; }
+		T& operator*() const { ASSERT(!empty()); return *ptr(); }
+		T* operator->() const { ASSERT(!empty()); return ptr(); }
 
-		const T& operator*() const
-		{
-			ASSERT(m_pointer);
-			return *m_pointer;
-		}
-
-		T* operator->()
-		{
-			ASSERT(m_pointer);
-			return m_pointer;
-		}
-
-		const T* operator->() const
-		{
-			ASSERT(m_pointer);
-			return m_pointer;
-		}
-
-		T* ptr() { return m_pointer; }
-		const T* ptr() const { return m_pointer; }
+		bool empty() const { return m_pointer == nullptr; }
+		explicit operator bool() const { return m_pointer; }
 
 		void clear()
 		{
@@ -87,8 +68,6 @@ namespace BAN
 			m_pointer = nullptr;
 		}
 
-		operator bool() const { return m_pointer != nullptr; }
-
 	private:
 		T* m_pointer = nullptr;
 
@@ -96,4 +75,13 @@ namespace BAN
 		friend class UniqPtr;
 	};
 
+	template<typename T>
+	struct hash<UniqPtr<T>>
+	{
+		constexpr hash_t operator()(const UniqPtr<T>& ptr) const
+		{
+			return hash<T*>()(ptr.ptr());
+		}
+	};
+
 }

kernel/arch/i686/PageTable.cpp

@@ -16,6 +16,8 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
@@ -24,7 +26,7 @@ namespace Kernel
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
 
-	static bool s_is_post_heap_done = false;
+	static bool s_is_initialized = false;
 
 	static PageTable* s_kernel = nullptr;
 	static bool s_has_nxe = false;
@@ -33,6 +35,28 @@ namespace Kernel
 
 	static paddr_t s_global_pdpte = 0;
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
+	static uint64_t* allocate_zeroed_page_aligned_page()
+	{
+		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
+		ASSERT(page);
+		memset(page, 0, PAGE_SIZE);
+		return (uint64_t*)page;
+	}
+
+	template<typename T>
+	static paddr_t V2P(const T vaddr)
+	{
+		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+	}
+
+	template<typename T>
+	static uint64_t* P2V(const T paddr)
+	{
+		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
+	}
+
 	static inline PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
@@ -51,31 +75,22 @@ namespace Kernel
 		return result;
 	}
 
-	void PageTable::initialize_pre_heap()
+	void PageTable::initialize_fast_page()
+	{
+		s_fast_page_pt = g_boot_fast_page_pt;
+	}
+
+	static void detect_cpu_features()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_pat())
 			s_has_pat = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->initialize_kernel();
-		s_kernel->initial_load();
 	}
 
-	void PageTable::initialize_post_heap()
-	{
-		s_is_post_heap_done = true;
-	}
-
-	void PageTable::initial_load()
+	void PageTable::enable_cpu_features()
 	{
 		if (s_has_nxe)
 		{
@@ -116,8 +131,56 @@ namespace Kernel
 			"movl %%eax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel);
+
+		auto* pdpt = allocate_zeroed_page_aligned_page();
+		ASSERT(pdpt);
+
+		s_kernel->m_highest_paging_struct = V2P(pdpt);
+		s_kernel->map_kernel_memory();
+
+		PageTable::with_fast_page(s_kernel->m_highest_paging_struct, [] {
+			s_global_pdpte = PageTable::fast_page_as_sized<paddr_t>(3);
+		});
+
+		// update fast page pt
+		{
+			constexpr vaddr_t vaddr = fast_page();
+			constexpr uint16_t pdpte = (vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						auto* vaddr = allocate_zeroed_page_aligned_page();
+						ASSERT(vaddr);
+						table[entry] = V2P(vaddr);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pdpt = s_kernel->m_highest_paging_struct;
+			const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd, pde, Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -131,40 +194,12 @@ namespace Kernel
 		return true;
 	}
 
-	static uint64_t* allocate_zeroed_page_aligned_page()
+	void PageTable::map_kernel_memory()
 	{
-		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-		ASSERT(page);
-		memset(page, 0, PAGE_SIZE);
-		return (uint64_t*)page;
-	}
-
-	template<typename T>
-	static paddr_t V2P(const T vaddr)
-	{
-		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-	}
-
-	template<typename T>
-	static uint64_t* P2V(const T paddr)
-	{
-		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-	}
-
-	void PageTable::initialize_kernel()
-	{
-		ASSERT(s_global_pdpte == 0);
-		s_global_pdpte = V2P(allocate_zeroed_page_aligned_page());
-
-		map_kernel_memory();
-
-		prepare_fast_page();
-
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
-		ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0);
 		map_range_at(
 			V2P(g_kernel_start),
-			(vaddr_t)g_kernel_start,
+			reinterpret_cast<vaddr_t>(g_kernel_start),
 			g_kernel_end - g_kernel_start,
 			Flags::Present
 		);
@@ -172,7 +207,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		map_range_at(
 			V2P(g_kernel_execute_start),
-			(vaddr_t)g_kernel_execute_start,
+			reinterpret_cast<vaddr_t>(g_kernel_execute_start),
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
 		);
@@ -180,7 +215,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		map_range_at(
 			V2P(g_kernel_writable_start),
-			(vaddr_t)g_kernel_writable_start,
+			reinterpret_cast<vaddr_t>(g_kernel_writable_start),
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
 		);
@@ -188,70 +223,34 @@ namespace Kernel
 		// Map userspace memory
 		map_range_at(
 			V2P(g_userspace_start),
-			(vaddr_t)g_userspace_start,
+			reinterpret_cast<vaddr_t>(g_userspace_start),
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
-
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
-		ASSERT(pdpt[pdpte] & Flags::Present);
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		ASSERT(pt[pte] == 0);
-		pt[pte] = Flags::Reserved;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
-		ASSERT(paddr);
-		ASSERT(paddr % PAGE_SIZE == 0);
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
 
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
+		ASSERT(!(*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
 
-		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
-
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
+		ASSERT((*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = 0;
 
-		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = Flags::Reserved;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -260,25 +259,23 @@ namespace Kernel
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
-		return page_table;
+
+		uint64_t* pdpt = allocate_zeroed_page_aligned_page();
+		if (pdpt == nullptr)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
 		}
 
-	void PageTable::map_kernel_memory()
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pdpte);
+		page_table->m_highest_paging_struct = V2P(pdpt);
 
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = V2P(kmalloc(32, 32, true));
-		ASSERT(m_highest_paging_struct);
-
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
 		pdpt[0] = 0;
 		pdpt[1] = 0;
 		pdpt[2] = 0;
 		pdpt[3] = s_global_pdpte | Flags::Present;
 		static_assert(KERNEL_OFFSET == 0xC0000000);
+
+		return page_table;
 	}
 
 	PageTable::~PageTable()
@@ -318,7 +315,7 @@ namespace Kernel
 		const bool is_userspace = (vaddr < KERNEL_OFFSET);
 		if (is_userspace && this != &PageTable::current())
 			;
-		else if (pages <= 32 || !s_is_post_heap_done)
+		else if (pages <= 32 || !s_is_initialized)
 		{
 			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
 				asm volatile("invlpg (%0)" :: "r"(vaddr));

kernel/arch/i686/Signal.S

@@ -19,7 +19,7 @@ signal_trampoline:
 	pushl %eax
 	pushl %ebp
 
-	movl 80(%esp), %eax
+	movl 84(%esp), %eax
 	pushl %eax; addl $4, (%esp)
 	pushl (%eax)
 

kernel/arch/i686/boot.S

@@ -98,8 +98,7 @@ bananboot_end:
 boot_pdpt:
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
-	.quad 0
-	.quad 0
+	.skip 2 * 8
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
 .align 4096
@@ -112,13 +111,16 @@ boot_pd:
 	.endr
 boot_pts:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.rept 512
 			.long i + (PG_READ_WRITE | PG_PRESENT)
 			.long 0
 			.set i, i + 0x1000
 		.endr
 	.endr
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -274,7 +276,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -284,21 +286,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
+ap_stack_paddr:
+	.skip 4
+ap_stack_vaddr:
+	.skip 4
+ap_prepare_paging:
+	.skip 4
+ap_page_table:
+	.skip 4
+ap_ready:
 	.skip 4
-ap_stack_loaded:
-	.skip 1
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -307,8 +315,7 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
+	movl AP_REL(ap_stack_paddr), %esp
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
 	leal V2P(enable_tsc),        %ecx; call *%ecx
@@ -316,24 +323,28 @@ ap_protected_mode:
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_flush_gdt)
+	ljmpl $0x08, $AP_REL(ap_flush_gdt)
 
 ap_flush_gdt:
-	# move stack pointer to higher half
-	movl %esp, %esp
-	addl $KERNEL_OFFSET, %esp
-
-	# jump to higher half
-	leal ap_higher_half, %ecx
+	movl $ap_higher_half, %ecx
 	jmp *%ecx
 
 ap_higher_half:
+	movl AP_REL(ap_prepare_paging), %eax
+	call *%eax
+
+	# load AP's initial values
+	movl AP_REL(ap_stack_vaddr), %esp
+	movl AP_REL(ap_page_table), %eax
+	movl $1, AP_REL(ap_ready)
+	movl %eax, %cr3
+
 	# clear rbp for stacktrace
 	xorl %ebp, %ebp
 
 1:	pause
 	cmpb $0, g_ap_startup_done
-	jz 1b
+	je 1b
 
 	lock incb g_ap_running_count
 

kernel/arch/i686/linker.ld

@@ -15,17 +15,17 @@ SECTIONS
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{

kernel/arch/x86_64/PageTable.cpp

@@ -2,7 +2,6 @@
 #include <kernel/CPUID.h>
 #include <kernel/Lock/SpinLock.h>
 #include <kernel/Memory/Heap.h>
-#include <kernel/Memory/kmalloc.h>
 #include <kernel/Memory/PageTable.h>
 
 extern uint8_t g_kernel_start[];
@@ -17,13 +16,15 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
 	SpinLock PageTable::s_fast_page_lock;
 
 	static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000;
-	static bool s_is_post_heap_done = false;
+	static bool s_is_initialized = false;
 
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
@@ -35,6 +36,8 @@ namespace Kernel
 
 	static paddr_t s_global_pml4_entries[512] { 0 };
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
 	static constexpr inline bool is_canonical(uintptr_t addr)
 	{
 		constexpr uintptr_t mask = 0xFFFF800000000000;
@@ -54,34 +57,6 @@ namespace Kernel
 		return addr;
 	}
 
-	struct FuncsKmalloc
-	{
-		static paddr_t allocate_zeroed_page_aligned_page()
-		{
-			void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-			ASSERT(page);
-			memset(page, 0, PAGE_SIZE);
-			return kmalloc_paddr_of(reinterpret_cast<vaddr_t>(page)).value();
-		}
-
-		static void unallocate_page(paddr_t paddr)
-		{
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-		}
-
-		static uint64_t* P2V(paddr_t paddr)
-		{
-			return reinterpret_cast<uint64_t*>(paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-		}
-	};
-
-	struct FuncsHHDM
-	{
 	static paddr_t allocate_zeroed_page_aligned_page()
 	{
 		const paddr_t paddr = Heap::get().take_free_page();
@@ -95,27 +70,14 @@ namespace Kernel
 		Heap::get().release_page(paddr);
 	}
 
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			ASSERT(vaddr >= s_hhdm_offset);
-			ASSERT(vaddr < KERNEL_OFFSET);
-			return vaddr - s_hhdm_offset;
-		}
-
 	static uint64_t* P2V(paddr_t paddr)
 	{
 		ASSERT(paddr != 0);
 		ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset));
 		return reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
 	}
-	};
 
-	static paddr_t   (*allocate_zeroed_page_aligned_page)() = &FuncsKmalloc::allocate_zeroed_page_aligned_page;
-	static void      (*unallocate_page)(paddr_t)            = &FuncsKmalloc::unallocate_page;
-	static paddr_t   (*V2P)(vaddr_t)                        = &FuncsKmalloc::V2P;
-	static uint64_t* (*P2V)(paddr_t)                        = &FuncsKmalloc::P2V;
-
-	static inline PageTable::flags_t parse_flags(uint64_t entry)
+	static PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
 
@@ -137,7 +99,7 @@ namespace Kernel
 	//   0: 4 KiB
 	//   1: 2 MiB
 	//   2: 1 GiB
-	static void init_map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
+	static void map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
 	{
 		ASSERT(0 <= page_size && page_size <= 2);
 
@@ -184,7 +146,7 @@ namespace Kernel
 		const uint64_t noexec_flag = s_has_nxe ? (static_cast<uint64_t>(1) << 63) : 0;
 
 		const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, noexec_flag);
-		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags;
+		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags | noexec_flag;
 
 		paddr_t lowest_paddr = pdpt;
 		uint16_t lowest_entry = pdpte;
@@ -207,23 +169,11 @@ namespace Kernel
 		});
 	}
 
-	static void init_map_hhdm(paddr_t pml4)
+	static void initialize_hhdm(paddr_t pml4)
 	{
 		for (const auto& entry : g_boot_info.memory_map_entries)
 		{
-			bool should_map = false;
-			switch (entry.type)
-			{
-				case MemoryMapEntry::Type::Available:
-					should_map = true;
-					break;
-				case MemoryMapEntry::Type::ACPIReclaim:
-				case MemoryMapEntry::Type::ACPINVS:
-				case MemoryMapEntry::Type::Reserved:
-					should_map = false;
-					break;
-			}
-			if (!should_map)
+			if (entry.type != MemoryMapEntry::Type::Available)
 				continue;
 
 			constexpr size_t one_gib = 1024 * 1024 * 1024;
@@ -235,156 +185,39 @@ namespace Kernel
 			{
 				if (s_has_gib && paddr % one_gib == 0 && paddr + one_gib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 2);
+					map_hhdm_page(pml4, paddr, 2);
 					paddr += one_gib;
 				}
 				else if (paddr % two_mib == 0 && paddr + two_mib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 1);
+					map_hhdm_page(pml4, paddr, 1);
 					paddr += two_mib;
 				}
 				else
 				{
-					init_map_hhdm_page(pml4, paddr, 0);
+					map_hhdm_page(pml4, paddr, 0);
 					paddr += PAGE_SIZE;
 				}
 			}
 		}
 	}
 
-	static paddr_t copy_page_from_kmalloc_to_heap(paddr_t kmalloc_paddr)
+	void PageTable::initialize_fast_page()
 	{
-		const paddr_t heap_paddr = Heap::get().take_free_page();
-		ASSERT(heap_paddr);
-
-		const vaddr_t kmalloc_vaddr = kmalloc_vaddr_of(kmalloc_paddr).value();
-
-		PageTable::with_fast_page(heap_paddr, [kmalloc_vaddr] {
-			memcpy(PageTable::fast_page_as_ptr(), reinterpret_cast<void*>(kmalloc_vaddr), PAGE_SIZE);
-		});
-
-		return heap_paddr;
+		s_fast_page_pt = g_boot_fast_page_pt;
 	}
 
-	static void copy_paging_structure_to_heap(uint64_t* old_table, uint64_t* new_table, int depth)
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t old_entry = old_table[index];
-			if (old_entry == 0)
-			{
-				new_table[index] = 0;
-				continue;
-			}
-
-			const paddr_t old_paddr = old_entry & page_addr_mask;
-			const paddr_t new_paddr = copy_page_from_kmalloc_to_heap(old_paddr);
-			new_table[index] = new_paddr | (old_entry & page_flag_mask);
-
-			uint64_t* next_old_table = reinterpret_cast<uint64_t*>(old_paddr + s_hhdm_offset);
-			uint64_t* next_new_table = reinterpret_cast<uint64_t*>(new_paddr + s_hhdm_offset);
-			copy_paging_structure_to_heap(next_old_table, next_new_table, depth - 1);
-		}
-	}
-
-	static void free_kmalloc_paging_structure(uint64_t* table, int depth)
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t entry = table[index];
-			if (entry == 0)
-				continue;
-
-			const paddr_t paddr = entry & page_addr_mask;
-
-			uint64_t* next_table = reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
-			free_kmalloc_paging_structure(next_table, depth - 1);
-
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-	}
-
-	void PageTable::initialize_pre_heap()
+	static void detect_cpu_features()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_1gib_pages())
 			s_has_gib = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-		s_kernel->prepare_fast_page();
-		s_kernel->initialize_kernel();
-
-		for (auto pml4e : s_global_pml4_entries)
-			ASSERT(pml4e == 0);
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		s_global_pml4_entries[511] = pml4[511];
 	}
 
-	void PageTable::initialize_post_heap()
-	{
-		ASSERT(s_kernel);
-
-		init_map_hhdm(s_kernel->m_highest_paging_struct);
-
-		const paddr_t old_pml4_paddr = s_kernel->m_highest_paging_struct;
-		const paddr_t new_pml4_paddr = copy_page_from_kmalloc_to_heap(old_pml4_paddr);
-
-		uint64_t* old_pml4 = reinterpret_cast<uint64_t*>(kmalloc_vaddr_of(old_pml4_paddr).value());
-		uint64_t* new_pml4 = reinterpret_cast<uint64_t*>(new_pml4_paddr + s_hhdm_offset);
-
-		const paddr_t old_pdpt_paddr = old_pml4[511] & s_page_addr_mask;
-		const paddr_t new_pdpt_paddr = Heap::get().take_free_page();
-		ASSERT(new_pdpt_paddr);
-
-		uint64_t* old_pdpt = reinterpret_cast<uint64_t*>(old_pdpt_paddr + s_hhdm_offset);
-		uint64_t* new_pdpt = reinterpret_cast<uint64_t*>(new_pdpt_paddr + s_hhdm_offset);
-		copy_paging_structure_to_heap(old_pdpt, new_pdpt, 2);
-
-		new_pml4[511] = new_pdpt_paddr | (old_pml4[511] & s_page_flag_mask);
-		s_global_pml4_entries[511] = new_pml4[511];
-
-		s_kernel->m_highest_paging_struct = new_pml4_paddr;
-		s_kernel->load();
-
-		free_kmalloc_paging_structure(old_pdpt, 2);
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pdpt_paddr).value()));
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pml4_paddr).value()));
-
-		allocate_zeroed_page_aligned_page = &FuncsHHDM::allocate_zeroed_page_aligned_page;
-		unallocate_page                   = &FuncsHHDM::unallocate_page;
-		V2P                               = &FuncsHHDM::V2P;
-		P2V                               = &FuncsHHDM::P2V;
-
-		s_is_post_heap_done = true;
-
-		// This is a hack to unmap fast page. fast page pt is copied
-		// while it is mapped, so we need to manually unmap it
-		SpinLockGuard _(s_fast_page_lock);
-		unmap_fast_page();
-	}
-
-	void PageTable::initial_load()
+	void PageTable::enable_cpu_features()
 	{
 		if (s_has_nxe)
 		{
@@ -423,8 +256,63 @@ namespace Kernel
 			"movq %%rax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		const paddr_t boot_pml4_paddr = ({
+			paddr_t paddr;
+			asm volatile("movq %%cr3, %0" : "=r"(paddr));
+			paddr;
+		});
+
+		initialize_hhdm(boot_pml4_paddr);
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel != nullptr);
+
+		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		ASSERT(s_kernel->m_highest_paging_struct);
+
+		uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+		s_kernel->map_kernel_memory();
+		s_global_pml4_entries[511] = pml4[511];
+
+		// update fast page pt
+		{
+			constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+			constexpr uint16_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+			constexpr uint16_t pdpte = (uc_vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (uc_vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						table[entry] = allocate_zeroed_page_aligned_page();
+						ASSERT(table[entry]);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pml4 = s_kernel->m_highest_paging_struct;
+			const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, Flags::ReadWrite | Flags::Present);
+			const paddr_t pd   = get_or_allocate_entry(pdpt, pdpte, Flags::ReadWrite | Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd,   pde,   Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -440,12 +328,12 @@ namespace Kernel
 		return true;
 	}
 
-	void PageTable::initialize_kernel()
+	void PageTable::map_kernel_memory()
 	{
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
 		const vaddr_t kernel_start = reinterpret_cast<vaddr_t>(g_kernel_start);
 		map_range_at(
-			V2P(kernel_start),
+			kernel_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_start,
 			g_kernel_end - g_kernel_start,
 			Flags::Present
@@ -454,7 +342,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		const vaddr_t kernel_execute_start = reinterpret_cast<vaddr_t>(g_kernel_execute_start);
 		map_range_at(
-			V2P(kernel_execute_start),
+			kernel_execute_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_execute_start,
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
@@ -463,7 +351,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		const vaddr_t kernel_writable_start = reinterpret_cast<vaddr_t>(g_kernel_writable_start);
 		map_range_at(
-			V2P(kernel_writable_start),
+			kernel_writable_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_writable_start,
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
@@ -472,114 +360,58 @@ namespace Kernel
 		// Map userspace memory
 		const vaddr_t userspace_start = reinterpret_cast<vaddr_t>(g_userspace_start);
 		map_range_at(
-			V2P(userspace_start),
+			userspace_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			userspace_start,
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
-
-		uint64_t* pml4 = P2V(m_highest_paging_struct);
-		ASSERT(!(pml4[pml4e] & Flags::Present));
-		pml4[pml4e] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		ASSERT(!(pdpt[pdpte] & Flags::Present));
-		pdpt[pdpte] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		ASSERT(pt[pte] == 0);
-		pt[pte] = Flags::Reserved;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
-		ASSERT(paddr);
-		ASSERT(paddr % PAGE_SIZE == 0);
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
 
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
+		ASSERT(!(*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = paddr | Flags::ReadWrite | Flags::Present;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
-
+		ASSERT(s_fast_page_pt);
 		ASSERT(s_fast_page_lock.current_processor_has_lock());
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
+		ASSERT((*s_fast_page_pt & Flags::Present));
+		s_fast_page_pt[0] = 0;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = Flags::Reserved;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
+		asm volatile("invlpg (%0)" :: "r"(fast_page()));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
 	{
 		SpinLockGuard _(s_kernel->m_lock);
+
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
+
+		page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		if (page_table->m_highest_paging_struct == 0)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
+		}
+
+		uint64_t* pml4 = P2V(page_table->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+
 		return page_table;
 	}
 
-	void PageTable::map_kernel_memory()
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pml4_entries[511]);
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-
-		PageTable::with_fast_page(m_highest_paging_struct, [] {
-			for (size_t i = 0; i < 512; i++)
-			{
-				if (s_global_pml4_entries[i] == 0)
-					continue;
-				ASSERT(i >= 256);
-				PageTable::fast_page_as_sized<uint64_t>(i) = s_global_pml4_entries[i];
-			}
-		});
-	}
-
 	PageTable::~PageTable()
 	{
 		if (m_highest_paging_struct == 0)
@@ -624,7 +456,7 @@ namespace Kernel
 		const bool is_userspace = (vaddr < KERNEL_OFFSET);
 		if (is_userspace && this != &PageTable::current())
 			;
-		else if (pages <= 32 || !s_is_post_heap_done)
+		else if (pages <= 32 || !s_is_initialized)
 		{
 			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
 				asm volatile("invlpg (%0)" :: "r"(vaddr));

kernel/arch/x86_64/boot.S

@@ -97,27 +97,25 @@ bananboot_end:
 .align 4096
 boot_pml4:
 	.quad V2P(boot_pdpt_lo) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 510
-		.quad 0
-	.endr
+	.skip 510 * 8
 	.quad V2P(boot_pdpt_hi) + (PG_READ_WRITE | PG_PRESENT)
 boot_pdpt_lo:
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 511
-		.quad 0
-	.endr
+	.skip 511 * 8
 boot_pdpt_hi:
-	.rept 510
-		.quad 0
-	.endr
+	.skip 510 * 8
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.quad 0
+	.skip 8
 boot_pd:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.quad i + (PG_PAGE_SIZE | PG_READ_WRITE | PG_PRESENT)
 		.set i, i + 0x200000
 	.endr
+	.quad V2P(g_boot_fast_page_pt) + (PG_READ_WRITE | PG_PRESENT)
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -273,7 +271,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -283,21 +281,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
-	.skip 4
-ap_stack_loaded:
-	.skip 1
+ap_stack_paddr:
+	.skip 8
+ap_stack_vaddr:
+	.skip 8
+ap_prepare_paging:
+	.skip 8
+ap_page_table:
+	.skip 8
+ap_ready:
+	.skip 8
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -306,8 +310,7 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
+	movl AP_REL(ap_stack_paddr), %esp
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
 	leal V2P(enable_tsc),        %ecx; call *%ecx
@@ -315,28 +318,34 @@ ap_protected_mode:
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_long_mode)
+	ljmpl $0x08, $AP_REL(ap_long_mode)
 
 .code64
 ap_long_mode:
-	# move stack pointer to higher half
-	movl %esp, %esp
-	addq $KERNEL_OFFSET, %rsp
+	movq $ap_higher_half, %rax
+	jmp *%rax
+
+ap_higher_half:
+	movq AP_REL(ap_prepare_paging), %rax
+	call *%rax
+
+	# load AP's initial values
+	movq AP_REL(ap_stack_vaddr), %rsp
+	movq AP_REL(ap_page_table), %rax
+	movq $1, AP_REL(ap_ready)
+	movq %rax, %cr3
 
 	# clear rbp for stacktrace
 	xorq %rbp, %rbp
 
-	xorb %al, %al
 1:	pause
-	cmpb %al, g_ap_startup_done
-	jz 1b
+	cmpb $0, g_ap_startup_done
+	je 1b
 
 	lock incb g_ap_running_count
 
-	# jump to ap_main in higher half
-	movabsq $ap_main, %rcx
-	call *%rcx
-	jmp V2P(system_halt)
+	call ap_main
+	jmp system_halt
 
 ap_gdt:
 	.quad 0x0000000000000000 # null descriptor

kernel/arch/x86_64/linker.ld

@@ -15,17 +15,17 @@ SECTIONS
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{

kernel/include/kernel/API/SharedPage.h

@@ -12,7 +12,7 @@ namespace Kernel::API
 
 	struct SharedPage
 	{
-		uint8_t __sequence[0x100];
+		uint16_t gdt_cpu_offset;
 
 		uint32_t features;
 

kernel/include/kernel/API/Syscall.h

@@ -14,7 +14,7 @@
 	#define _kas_globbers
 #endif
 
-#define _kas_argument_var(index, value) register long _kas_a##index asm(_ban_stringify(_ban_get(index, _kas_arguments))) = (long)value;
+#define _kas_argument_var(index, value) register long _kas_a##index asm(_ban_stringify(_ban_get(index, _kas_arguments))) = (long)(value);
 #define _kas_dummy_var(index, value) register long _kas_d##index asm(#value);
 #define _kas_input(index, _) "r"(_kas_a##index)
 #define _kas_output(index, _) , "=r"(_kas_d##index)

kernel/include/kernel/Epoll.h

@@ -1,12 +1,8 @@
 #pragma once
 
-#include <BAN/Array.h>
-#include <BAN/CircularQueue.h>
 #include <BAN/HashMap.h>
-#include <BAN/HashSet.h>
 #include <kernel/FS/Inode.h>
 
-#include <limits.h>
 #include <sys/epoll.h>
 
 namespace Kernel
@@ -53,57 +49,52 @@ namespace Kernel
 		BAN::ErrorOr<void> fsync_impl() override { return {}; }
 
 	private:
-		struct InodeRefPtrHash
-		{
-			BAN::hash_t operator()(const BAN::RefPtr<Inode>& inode)
-			{
-				return BAN::hash<const Inode*>()(inode.ptr());
-			}
-		};
-
 		struct ListenEventList
 		{
-			BAN::Array<epoll_event, OPEN_MAX> events;
-			uint32_t bitmap[(OPEN_MAX + 31) / 32] {};
+			ListenEventList() = default;
+
+			ListenEventList(const ListenEventList&) = delete;
+			ListenEventList& operator=(const ListenEventList&) = delete;
+
+			ListenEventList(ListenEventList&& other)
+				: events(BAN::move(other.events))
+			{}
+			ListenEventList& operator=(ListenEventList&& other)
+			{
+				events = BAN::move(other.events);
+				return *this;
+			}
+
+			BAN::HashMap<int, epoll_event> events;
 
 			bool has_fd(int fd) const
 			{
-				// For some reason having (fd < 0 || ...) makes GCC 15.1.0
-				// think bitmap access can be out of bounds...
-				if (static_cast<size_t>(fd) >= events.size())
-					return false;
-				return bitmap[fd / 32] & (1u << (fd % 32));
+				return events.contains(fd);
 			}
 
 			bool empty() const
 			{
-				for (auto val : bitmap)
-					if (val != 0)
-						return false;
-				return true;
+				return events.empty();
 			}
 
-			void add_fd(int fd, epoll_event event)
+			BAN::ErrorOr<void> add_fd(int fd, epoll_event event)
 			{
-				ASSERT(!has_fd(fd));
-				bitmap[fd / 32] |= (1u << (fd % 32));
-				events[fd] = event;
+				TRY(events.insert(fd, event));
+				return {};
 			}
 
 			void remove_fd(int fd)
 			{
-				ASSERT(has_fd(fd));
-				bitmap[fd / 32] &= ~(1u << (fd % 32));
-				events[fd] = {};
+				events.remove(fd);
 			}
 		};
 
 	private:
 		ThreadBlocker m_thread_blocker;
 		SpinLock m_ready_lock;
-		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t,        InodeRefPtrHash> m_ready_events;
-		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t,        InodeRefPtrHash> m_processing_events;
-		BAN::HashMap<BAN::RefPtr<Inode>, ListenEventList, InodeRefPtrHash> m_listening_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_ready_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_processing_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, ListenEventList> m_listening_events;
 	};
 
 }

kernel/include/kernel/GDT.h

@@ -133,6 +133,12 @@ namespace Kernel
 		void set_gsbase(uintptr_t addr);
 #endif
 
+		static uint16_t cpu_index_offset() { return m_cpu_index_offset; }
+		void set_cpu_index(uint8_t index)
+		{
+			write_entry(m_cpu_index_offset, 0, index, 0xF2, 0x4);
+		}
+
 	private:
 		GDT() = default;
 
@@ -151,11 +157,13 @@ namespace Kernel
 
 	private:
 #if ARCH(x86_64)
-		BAN::Array<SegmentDescriptor, 8> m_gdt; // null, kernel code, kernel data, user code (32 bit), user data, user code (64 bit), tss low, tss high
-		static constexpr uint16_t m_tss_offset = 0x30;
+		BAN::Array<SegmentDescriptor, 9> m_gdt; // null, kernel code, kernel data, user code (32 bit), user data, user code (64 bit), cpu-index, tss low, tss high
+		static constexpr uint16_t m_cpu_index_offset = 0x30;
+		static constexpr uint16_t m_tss_offset = 0x38;
 #elif ARCH(i686)
-		BAN::Array<SegmentDescriptor, 9> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, tss
-		static constexpr uint16_t m_tss_offset = 0x40;
+		BAN::Array<SegmentDescriptor, 10> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, cpu-index, tss
+		static constexpr uint16_t m_cpu_index_offset = 0x40;
+		static constexpr uint16_t m_tss_offset = 0x48;
 #endif
 		TaskStateSegment m_tss;
 		const GDTR m_gdtr {

kernel/include/kernel/Memory/ByteRingBuffer.h

@@ -51,7 +51,7 @@ namespace Kernel
 		uint8_t back() const
 		{
 			ASSERT(!empty());
-			return reinterpret_cast<const uint8_t*>(m_vaddr)[m_tail + m_size];
+			return reinterpret_cast<const uint8_t*>(m_vaddr)[m_tail + m_size - 1];
 		}
 
 		bool empty() const { return m_size == 0; }

kernel/include/kernel/Memory/DMARegion.h

@@ -8,7 +8,7 @@ namespace Kernel
 	class DMARegion
 	{
 	public:
-		static BAN::ErrorOr<BAN::UniqPtr<DMARegion>> create(size_t size);
+		static BAN::ErrorOr<BAN::UniqPtr<DMARegion>> create(size_t size, PageTable::MemoryType type = PageTable::MemoryType::Uncached);
 		~DMARegion();
 
 		size_t size() const { return m_size; }

kernel/include/kernel/Memory/MemoryRegion.h

@@ -9,12 +9,6 @@
 namespace Kernel
 {
 
-	struct AddressRange
-	{
-		vaddr_t start;
-		vaddr_t end;
-	};
-
 	class MemoryRegion
 	{
 		BAN_NON_COPYABLE(MemoryRegion);

kernel/include/kernel/Memory/PageTable.h

@@ -46,13 +46,22 @@ namespace Kernel
 		};
 
 	public:
-		static void initialize_pre_heap();
-		static void initialize_post_heap();
+		static void initialize_fast_page();
+		static void initialize_and_load();
+
+		static void enable_cpu_features();
 
 		static PageTable& kernel();
 		static PageTable& current() { return *reinterpret_cast<PageTable*>(Processor::get_current_page_table()); }
 
-		static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; }
+		static constexpr vaddr_t fast_page()
+		{
+#if ARCH(x86_64)
+			return 0xffffffffbfe00000;
+#elif ARCH(i686)
+			return 0xffe00000;
+#endif
+		}
 
 		template<with_fast_page_callback F>
 		static void with_fast_page(paddr_t paddr, F callback)
@@ -121,7 +130,6 @@ namespace Kernel
 		vaddr_t reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX);
 
 		void load();
-		void initial_load();
 
 		void invalidate_page(vaddr_t addr, bool send_smp_message) { invalidate_range(addr, 1, send_smp_message); }
 		void invalidate_range(vaddr_t addr, size_t pages, bool send_smp_message);
@@ -129,14 +137,14 @@ namespace Kernel
 		InterruptState lock() const { return m_lock.lock(); }
 		void unlock(InterruptState state) const { m_lock.unlock(state); }
 
+		paddr_t paddr() const { return m_highest_paging_struct; }
+
 		void debug_dump();
 
 	private:
 		PageTable() = default;
 		uint64_t get_page_data(vaddr_t) const;
-		void initialize_kernel();
 		void map_kernel_memory();
-		void prepare_fast_page();
 
 		static void map_fast_page(paddr_t);
 		static void unmap_fast_page();

kernel/include/kernel/Memory/Types.h

@@ -23,4 +23,10 @@ namespace Kernel
 	using vaddr_t = uintptr_t;
 	using paddr_t = uint64_t;
 
+	struct AddressRange
+	{
+		vaddr_t start;
+		vaddr_t end;
+	};
+
 }

kernel/include/kernel/Memory/VirtualRange.h

@@ -14,42 +14,27 @@ namespace Kernel
 		BAN_NON_MOVABLE(VirtualRange);
 
 	public:
-		// Create virtual range to fixed virtual address
-		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr(PageTable&, vaddr_t, size_t, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages);
-		// Create virtual range to virtual address range
-		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr_range(PageTable&, vaddr_t vaddr_start, vaddr_t vaddr_end, size_t, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages);
+		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr_range(PageTable&, AddressRange address_range, size_t, PageTable::flags_t flags, bool add_guard_pages);
 		~VirtualRange();
 
 		vaddr_t vaddr() const { return m_vaddr + (m_has_guard_pages ? PAGE_SIZE : 0); }
 		size_t size() const { return m_size - (m_has_guard_pages ? 2 * PAGE_SIZE : 0); }
 		PageTable::flags_t flags() const { return m_flags; }
 
-		paddr_t paddr_of(vaddr_t vaddr) const
-		{
-			ASSERT(vaddr % PAGE_SIZE == 0);
-			const size_t index = (vaddr - this->vaddr()) / PAGE_SIZE;
-			ASSERT(index < m_paddrs.size());
-			const paddr_t paddr = m_paddrs[index];
-			ASSERT(paddr);
-			return paddr;
-		}
+		paddr_t paddr_of(vaddr_t vaddr) const { return m_page_table.physical_address_of(vaddr & PAGE_ADDR_MASK); }
 
 		bool contains(vaddr_t address) const { return vaddr() <= address && address < vaddr() + size(); }
 
-		BAN::ErrorOr<bool> allocate_page_for_demand_paging(vaddr_t address);
-
 	private:
-		VirtualRange(PageTable&, bool preallocated, bool has_guard_pages, vaddr_t, size_t, PageTable::flags_t);
+		VirtualRange(PageTable&, bool has_guard_pages, vaddr_t, size_t, PageTable::flags_t);
 		BAN::ErrorOr<void> initialize();
 
 	private:
 		PageTable&               m_page_table;
-		const bool               m_preallocated;
 		const bool               m_has_guard_pages;
 		const vaddr_t            m_vaddr;
 		const size_t             m_size;
 		const PageTable::flags_t m_flags;
-		BAN::Vector<paddr_t>     m_paddrs;
 		SpinLock                 m_lock;
 
 		friend class BAN::UniqPtr<VirtualRange>;

kernel/include/kernel/Memory/kmalloc.h

@@ -13,4 +13,3 @@ void* kmalloc(size_t size, size_t align, bool force_identity_map = false);
 void kfree(void*);
 
 BAN::Optional<Kernel::paddr_t> kmalloc_paddr_of(Kernel::vaddr_t);
-BAN::Optional<Kernel::vaddr_t> kmalloc_vaddr_of(Kernel::paddr_t);

kernel/include/kernel/Processor.h

@@ -2,6 +2,7 @@
 
 #include <BAN/Atomic.h>
 #include <BAN/ForwardList.h>
+#include <BAN/Math.h>
 
 #include <kernel/API/SharedPage.h>
 #include <kernel/Arch.h>
@@ -58,6 +59,8 @@ namespace Kernel
 		static Processor& create(ProcessorID id);
 		static Processor& initialize();
 
+		void allocate_stack();
+
 		static ProcessorID current_id() { return read_gs_sized<ProcessorID>(offsetof(Processor, m_id)); }
 		static uint8_t current_index()  { return read_gs_sized<uint8_t>(offsetof(Processor, m_index)); }
 		static ProcessorID id_from_index(size_t index);
@@ -100,11 +103,8 @@ namespace Kernel
 				handle_smp_messages();
 		}
 
-		static uintptr_t current_stack_bottom() { return read_gs_sized<uintptr_t>(offsetof(Processor, m_stack)); }
-		static uintptr_t current_stack_top()	{ return current_stack_bottom() + s_stack_size; }
-
-		uintptr_t stack_bottom() const	{ return reinterpret_cast<uintptr_t>(m_stack); }
-		uintptr_t stack_top() const		{ return stack_bottom() + s_stack_size; }
+		vaddr_t stack_top_vaddr() const { return m_stack_vaddr + s_stack_size; }
+		paddr_t stack_top_paddr() const { return m_stack_paddr + s_stack_size; }
 
 		static void set_thread_syscall_stack(vaddr_t vaddr) { write_gs_sized<vaddr_t>(offsetof(Processor, m_thread_syscall_stack), vaddr); }
 
@@ -140,11 +140,7 @@ namespace Kernel
 		static void disable_sse()
 		{
 			uintptr_t dummy;
-#if ARCH(x86_64)
-			asm volatile("movq %%cr0, %0; orq $0x08, %0; movq %0, %%cr0" : "=r"(dummy));
-#elif ARCH(i686)
-			asm volatile("movl %%cr0, %0; orl $0x08, %0; movl %0, %%cr0" : "=r"(dummy));
-#endif
+			asm volatile("mov %%cr0, %0; or $0x08, %0; mov %0, %%cr0" : "=r"(dummy));
 		}
 
 		static void enable_sse()
@@ -169,35 +165,17 @@ namespace Kernel
 		}
 
 		template<typename T>
-		static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8)
+		static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8 && BAN::Math::is_power_of_two(sizeof(T)))
 		{
-#define __ASM_INPUT(operation) asm volatile(operation " %%gs:%a[offset], %[result]" : [result]"=r"(result) : [offset]"ir"(offset))
-			T result;
-			if constexpr(sizeof(T) == 8)
-				__ASM_INPUT("movq");
-			if constexpr(sizeof(T) == 4)
-				__ASM_INPUT("movl");
-			if constexpr(sizeof(T) == 2)
-				__ASM_INPUT("movw");
-			if constexpr(sizeof(T) == 1)
-				__ASM_INPUT("movb");
-			return result;
-#undef __ASM_INPUT
+			T value;
+			asm volatile("mov %%gs:%a[offset], %[value]" : [value]"=r"(value) : [offset]"ir"(offset));
+			return value;
 		}
 
 		template<typename T>
-		static void write_gs_sized(uintptr_t offset, T value) requires(sizeof(T) <= 8)
+		static void write_gs_sized(uintptr_t offset, T value) requires(sizeof(T) <= 8 && BAN::Math::is_power_of_two(sizeof(T)))
 		{
-#define __ASM_INPUT(operation) asm volatile(operation " %[value], %%gs:%a[offset]" :: [value]"r"(value), [offset]"ir"(offset) : "memory")
-			if constexpr(sizeof(T) == 8)
-				__ASM_INPUT("movq");
-			if constexpr(sizeof(T) == 4)
-				__ASM_INPUT("movl");
-			if constexpr(sizeof(T) == 2)
-				__ASM_INPUT("movw");
-			if constexpr(sizeof(T) == 1)
-				__ASM_INPUT("movb");
-#undef __ASM_INPUT
+			asm volatile("mov %[value], %%gs:%a[offset]" :: [value]"r"(value), [offset]"ir"(offset) : "memory");
 		}
 
 	private:
@@ -215,8 +193,9 @@ namespace Kernel
 
 		Thread* m_sse_thread { nullptr };
 
-		static constexpr size_t s_stack_size { 4096 };
-		void* m_stack { nullptr };
+		static constexpr size_t s_stack_size { PAGE_SIZE };
+		vaddr_t m_stack_vaddr { 0 };
+		paddr_t m_stack_paddr { 0 };
 
 		GDT* m_gdt { nullptr };
 		IDT* m_idt { nullptr };

kernel/include/kernel/SchedulerQueueNode.h

@@ -22,8 +22,9 @@ namespace Kernel
 
 		uint64_t wake_time_ns { static_cast<uint64_t>(-1) };
 
-		SpinLock blocker_lock;
-		ThreadBlocker* blocker { nullptr };
+		BAN::Atomic<ThreadBlocker*> blocker { nullptr };
+		SchedulerQueueNode* block_chain_prev { nullptr };
+		SchedulerQueueNode* block_chain_next { nullptr };
 
 		ProcessorID processor_id { PROCESSOR_NONE };
 		bool blocked { false };

kernel/include/kernel/Thread.h

@@ -178,9 +178,7 @@ namespace Kernel
 		bool                       m_is_userspace         { false };
 		bool                       m_delete_process       { false };
 
-		bool                       m_has_custom_fsbase    { false };
 		vaddr_t                    m_fsbase               { 0 };
-		bool                       m_has_custom_gsbase    { false };
 		vaddr_t                    m_gsbase               { 0 };
 
 		SchedulerQueue::Node*      m_scheduler_node       { nullptr };

kernel/include/kernel/ThreadBlocker.h

@@ -15,7 +15,6 @@ namespace Kernel
 		void block_with_wake_time_ns(uint64_t wake_time_ns, BaseMutex*);
 		void unblock();
 
-
 		void block_with_timeout_ms(uint64_t timeout_ms, BaseMutex* mutex)
 		{
 			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(timeout_ms, 1'000'000));
@@ -29,14 +28,12 @@ namespace Kernel
 
 	private:
 		void add_thread_to_block_queue(SchedulerQueue::Node*);
-		void remove_blocked_thread(SchedulerQueue::Node*);
+		void remove_thread_from_block_queue(SchedulerQueue::Node*);
 
 	private:
+		SchedulerQueue::Node* m_block_chain { nullptr };
 		SpinLock m_lock;
 
-		SchedulerQueue::Node* m_block_chain[32] {};
-		size_t m_block_chain_length { 0 };
-
 		friend class Scheduler;
 	};
 

kernel/kernel/APIC.cpp

@@ -293,9 +293,25 @@ namespace Kernel
 			dprintln("Trying to enable processor (lapic id {})", processor.apic_id);
 
 			auto& proc = Kernel::Processor::create(ProcessorID(processor.apic_id));
+			proc.allocate_stack();
+
+			struct ap_init_info_t
+			{
+				uintptr_t stack_paddr;
+				uintptr_t stack_vaddr;
+				uintptr_t prepare_paging;
+				uintptr_t page_table;
+				uintptr_t ready;
+			};
+
 			PageTable::with_fast_page(ap_init_paddr, [&] {
-				PageTable::fast_page_as_sized<uint32_t>(2) = kmalloc_paddr_of(proc.stack_top()).value();
-				PageTable::fast_page_as_sized<uint8_t>(13) = 0;
+				PageTable::fast_page_as<ap_init_info_t>(8) = {
+					.stack_paddr    = static_cast<uintptr_t>(proc.stack_top_paddr()),
+					.stack_vaddr    = proc.stack_top_vaddr(),
+					.prepare_paging = reinterpret_cast<uintptr_t>(&PageTable::enable_cpu_features),
+					.page_table     = static_cast<uintptr_t>(PageTable::kernel().paddr()),
+					.ready          = 0,
+				};
 			});
 
 			write_to_local_apic(LAPIC_ERROR_REG, 0x00);
@@ -334,12 +350,9 @@ namespace Kernel
 
 			// give processor upto 100 * 100 us + 200 us to boot
 			PageTable::with_fast_page(ap_init_paddr, [&] {
-				for (int i = 0; i < 100; i++)
-				{
-					if (__atomic_load_n(&PageTable::fast_page_as_sized<uint8_t>(13), __ATOMIC_SEQ_CST))
+				for (int i = 0; i < 100; i++, udelay(100))
+					if (__atomic_load_n(&PageTable::fast_page_as<ap_init_info_t>(8).ready, __ATOMIC_SEQ_CST))
 						break;
-					udelay(100);
-				}
 			});
 
 			initialized_aps++;

kernel/kernel/Debug.cpp

@@ -7,6 +7,8 @@
 #include <kernel/Terminal/TTY.h>
 #include <kernel/Timer/Timer.h>
 
+#include <BAN/ScopeGuard.h>
+
 #include <LibDEFLATE/Compressor.h>
 #include <LibQR/QRCode.h>
 
@@ -14,6 +16,13 @@
 
 bool g_disable_debug = false;
 
+extern "C" bool safe_user_memcpy(void*, const void*, size_t);
+
+namespace Kernel
+{
+	extern bool g_safe_user_alloc_nonexisting;
+}
+
 namespace Debug
 {
 
@@ -45,48 +54,25 @@ namespace Debug
 
 		SpinLockGuard _(s_debug_lock);
 
-		const stackframe* frame = reinterpret_cast<const stackframe*>(bp);
-
-		void* first_ip = frame->ip;
-		void* last_ip = 0;
-		bool first = true;
+		const bool temp = g_safe_user_alloc_nonexisting;
+		g_safe_user_alloc_nonexisting = false;
+		BAN::ScopeGuard alloc_restore([temp] { g_safe_user_alloc_nonexisting = temp; });
 
 		BAN::Formatter::print(Debug::putchar, "\e[36mStack trace:\r\n");
 
 		if (ip != 0)
 			BAN::Formatter::print(Debug::putchar, "    {}\r\n", reinterpret_cast<void*>(ip));
 
-		while (frame)
+		stackframe frame;
+		if (!safe_user_memcpy(&frame, reinterpret_cast<void*>(bp), sizeof(stackframe)))
+			;
+		else for (size_t depth = 0; depth < 64; depth++)
 		{
-			if (!PageTable::is_valid_pointer((vaddr_t)frame))
-			{
-				derrorln("invalid pointer {H}", (vaddr_t)frame);
+			BAN::Formatter::print(Debug::putchar, "    {}\r\n", reinterpret_cast<void*>(frame.ip));
+			if (frame.bp == nullptr || !safe_user_memcpy(&frame, frame.bp, sizeof(stackframe)))
 				break;
 		}
 
-			if (PageTable::current().is_page_free((vaddr_t)frame & PAGE_ADDR_MASK))
-			{
-				derrorln("    {} not mapped", frame);
-				break;
-			}
-
-			BAN::Formatter::print(Debug::putchar, "    {}\r\n", (void*)frame->ip);
-
-			if (!first && frame->ip == first_ip)
-			{
-				derrorln("looping kernel panic :(");
-				break;
-			}
-			else if (!first && frame->ip == last_ip)
-			{
-				derrorln("repeating stack trace");
-				break;
-			}
-
-			last_ip = frame->ip;
-			frame = frame->bp;
-			first = false;
-		}
 		BAN::Formatter::print(Debug::putchar, "\e[m");
 	}
 

kernel/kernel/Device/FramebufferDevice.cpp

@@ -84,10 +84,10 @@ namespace Kernel
 
 		m_video_buffer = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET, UINTPTR_MAX,
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			BAN::Math::div_round_up<size_t>(m_width * m_height * (BANAN_FB_BPP / 8), PAGE_SIZE) * PAGE_SIZE,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, false
+			false
 		));
 
 		return {};

kernel/kernel/Epoll.cpp

@@ -16,7 +16,7 @@ namespace Kernel
 
 	Epoll::~Epoll()
 	{
-		for (auto [inode, _] : m_listening_events)
+		for (auto& [inode, _] : m_listening_events)
 			inode->del_epoll(this);
 	}
 
@@ -44,7 +44,7 @@ namespace Kernel
 
 				if (!contains_inode)
 					TRY(inode->add_epoll(this));
-				it->value.add_fd(fd, event);
+				TRY(it->value.add_fd(fd, event));
 
 				SpinLockGuard _(m_ready_lock);
 				auto ready_it = m_ready_events.find(inode);
@@ -144,9 +144,8 @@ namespace Kernel
 
 					{
 						uint32_t listen_mask = EPOLLHUP | EPOLLERR;
-						for (size_t fd = 0; fd < listen.events.size(); fd++)
-							if (listen.has_fd(fd))
-								listen_mask |= listen.events[fd].events;
+						for (const auto& [_, events] : listen.events)
+							listen_mask |= events.events;
 						events &= listen_mask;
 					}
 
@@ -171,11 +170,10 @@ namespace Kernel
 
 #undef REMOVE_IT
 
-					for (size_t fd = 0; fd < listen.events.size() && event_count < event_span.size(); fd++)
+					for (auto& [_, listen_event] : listen.events)
 					{
-						if (!listen.has_fd(fd))
-							continue;
-						auto& listen_event = listen.events[fd];
+						if (event_count >= event_span.size())
+							break;
 
 						const auto new_events = (listen_event.events | EPOLLHUP | EPOLLERR) & events;
 						if (new_events == 0)

kernel/kernel/GDT.cpp

@@ -32,6 +32,8 @@ namespace Kernel
 		gdt->write_entry(0x38, 0x00000000, 0x00000, 0xF2, 0xC); // gsbase
 #endif
 
+		gdt->write_entry(m_cpu_index_offset, 0, 0, 0, 0);
+
 		gdt->write_tss();
 
 		return gdt;

kernel/kernel/IDT.cpp

@@ -191,6 +191,8 @@ namespace Kernel
 		},
 	};
 
+	bool g_safe_user_alloc_nonexisting { true };
+
 	extern "C" void cpp_isr_handler(uint32_t isr, uint32_t error, InterruptStack* interrupt_stack, const Registers* regs)
 	{
 		if (g_paniced)
@@ -214,6 +216,16 @@ namespace Kernel
 				PageFaultError page_fault_error;
 				page_fault_error.raw = error;
 
+				const uint8_t* ip = reinterpret_cast<const uint8_t*>(interrupt_stack->ip);
+
+				if (!g_safe_user_alloc_nonexisting) for (const auto& safe_user : s_safe_user_page_faults)
+				{
+					if (ip < safe_user.ip_start || ip >= safe_user.ip_end)
+						continue;
+					interrupt_stack->ip = reinterpret_cast<vaddr_t>(safe_user.ip_fault);
+					return;
+				}
+
 				Processor::set_interrupt_state(InterruptState::Enabled);
 				auto result = Process::current().allocate_page_for_demand_paging(regs->cr2, page_fault_error.write, page_fault_error.instruction);
 				Processor::set_interrupt_state(InterruptState::Disabled);
@@ -234,8 +246,7 @@ namespace Kernel
 				if (result.value())
 					return;
 
-				const uint8_t* ip = reinterpret_cast<const uint8_t*>(interrupt_stack->ip);
-				for (const auto& safe_user : s_safe_user_page_faults)
+				if (g_safe_user_alloc_nonexisting) for (const auto& safe_user : s_safe_user_page_faults)
 				{
 					if (ip < safe_user.ip_start || ip >= safe_user.ip_end)
 						continue;
@@ -282,7 +293,7 @@ namespace Kernel
 
 		const char* process_name = (tid && Thread::current().has_process())
 			? Process::current().name()
-			: nullptr;
+			: "";
 
 #if ARCH(x86_64)
 		dwarnln(

kernel/kernel/Memory/DMARegion.cpp

@@ -5,7 +5,7 @@
 namespace Kernel
 {
 
-	BAN::ErrorOr<BAN::UniqPtr<DMARegion>> DMARegion::create(size_t size)
+	BAN::ErrorOr<BAN::UniqPtr<DMARegion>> DMARegion::create(size_t size, PageTable::MemoryType type)
 	{
 		size_t needed_pages = BAN::Math::div_round_up<size_t>(size, PAGE_SIZE);
 
@@ -26,7 +26,7 @@ namespace Kernel
 		vaddr_guard.disable();
 		paddr_guard.disable();
 
-		PageTable::kernel().map_range_at(paddr, vaddr, size, PageTable::Flags::ReadWrite | PageTable::Flags::Present, PageTable::MemoryType::Uncached);
+		PageTable::kernel().map_range_at(paddr, vaddr, size, PageTable::Flags::ReadWrite | PageTable::Flags::Present, type);
 
 		return BAN::UniqPtr<DMARegion>::adopt(region_ptr);
 	}

kernel/kernel/Memory/FileBackedRegion.cpp

@@ -67,8 +67,12 @@ namespace Kernel
 		ASSERT(success);
 
 		for (size_t i = 0; i < pages.size(); i++)
-			if (pages[i])
+		{
+			if (pages[i] == 0)
+				continue;
 			sync(i);
+			Heap::get().release_page(pages[i]);
+		}
 
 		mutex.unlock();
 	}

kernel/kernel/Memory/VirtualRange.cpp

@@ -1,67 +1,47 @@
+#include <BAN/ScopeGuard.h>
 #include <kernel/Memory/Heap.h>
 #include <kernel/Memory/VirtualRange.h>
 
 namespace Kernel
 {
 
-	BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> VirtualRange::create_to_vaddr(PageTable& page_table, vaddr_t vaddr, size_t size, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages)
-	{
-		ASSERT(size % PAGE_SIZE == 0);
-		ASSERT(vaddr % PAGE_SIZE == 0);
-		ASSERT(vaddr > 0);
-
-		if (add_guard_pages)
-		{
-			vaddr -= PAGE_SIZE;
-			size += 2 * PAGE_SIZE;
-		}
-
-		auto result = TRY(BAN::UniqPtr<VirtualRange>::create(page_table, preallocate_pages, add_guard_pages, vaddr, size, flags));
-		ASSERT(page_table.reserve_range(vaddr, size));
-		TRY(result->initialize());
-
-		return result;
-	}
-
-	BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> VirtualRange::create_to_vaddr_range(PageTable& page_table, vaddr_t vaddr_start, vaddr_t vaddr_end, size_t size, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages)
+	BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> VirtualRange::create_to_vaddr_range(PageTable& page_table, AddressRange address_range, size_t size, PageTable::flags_t flags, bool add_guard_pages)
 	{
 		if (add_guard_pages)
 			size += 2 * PAGE_SIZE;
 
 		ASSERT(size % PAGE_SIZE == 0);
-		ASSERT(vaddr_start > 0);
-		ASSERT(vaddr_start + size <= vaddr_end);
 
 		// Align vaddr range to page boundaries
-		if (size_t rem = vaddr_start % PAGE_SIZE)
-			vaddr_start += PAGE_SIZE - rem;
-		if (size_t rem = vaddr_end % PAGE_SIZE)
-			vaddr_end -= rem;
-		ASSERT(vaddr_start < vaddr_end);
-		ASSERT(vaddr_end - vaddr_start + 1 >= size / PAGE_SIZE);
+		if (const size_t rem = address_range.start % PAGE_SIZE)
+			address_range.start += PAGE_SIZE - rem;
+		if (const size_t rem = address_range.end % PAGE_SIZE)
+			address_range.end -= rem;
 
-		const vaddr_t vaddr = page_table.reserve_free_contiguous_pages(size / PAGE_SIZE, vaddr_start, vaddr_end);
+		const vaddr_t vaddr = page_table.reserve_free_contiguous_pages(size / PAGE_SIZE, address_range.start, address_range.end);
 		if (vaddr == 0)
 			return BAN::Error::from_errno(ENOMEM);
-		ASSERT(vaddr >= vaddr_start);
-		ASSERT(vaddr + size <= vaddr_end);
 
-		auto result_or_error = BAN::UniqPtr<VirtualRange>::create(page_table, preallocate_pages, add_guard_pages, vaddr, size, flags);
-		if (result_or_error.is_error())
-		{
+		BAN::ScopeGuard vaddr_cleaner([&page_table, vaddr, size] {
 			page_table.unmap_range(vaddr, size);
-			return result_or_error.release_error();
-		}
+		});
 
-		auto result = result_or_error.release_value();
+		auto result = TRY(BAN::UniqPtr<VirtualRange>::create(
+			page_table,
+			add_guard_pages,
+			vaddr,
+			size,
+			flags)
+		);
 		TRY(result->initialize());
 
+		vaddr_cleaner.disable();
+
 		return result;
 	}
 
-	VirtualRange::VirtualRange(PageTable& page_table, bool preallocated, bool has_guard_pages, vaddr_t vaddr, size_t size, PageTable::flags_t flags)
+	VirtualRange::VirtualRange(PageTable& page_table, bool has_guard_pages, vaddr_t vaddr, size_t size, PageTable::flags_t flags)
 		: m_page_table(page_table)
-		, m_preallocated(preallocated)
 		, m_has_guard_pages(has_guard_pages)
 		, m_vaddr(vaddr)
 		, m_size(size)
@@ -71,70 +51,26 @@ namespace Kernel
 	VirtualRange::~VirtualRange()
 	{
 		ASSERT(m_vaddr);
-		m_page_table.unmap_range(m_vaddr, m_size);
-
-		for (paddr_t paddr : m_paddrs)
-			if (paddr != 0)
+		for (size_t off = 0; off < size(); off += PAGE_SIZE)
+			if (const auto paddr = m_page_table.physical_address_of(vaddr() + off))
 				Heap::get().release_page(paddr);
+		m_page_table.unmap_range(m_vaddr, m_size);
 	}
 
 	BAN::ErrorOr<void> VirtualRange::initialize()
 	{
-		TRY(m_paddrs.resize(size() / PAGE_SIZE, 0));
-
-		if (!m_preallocated)
-			return {};
-
 		const size_t page_count = size() / PAGE_SIZE;
 		for (size_t i = 0; i < page_count; i++)
 		{
-			m_paddrs[i] = Heap::get().take_free_page();
-			if (m_paddrs[i] == 0)
+			const auto paddr = Heap::get().take_free_page();
+			if (paddr == 0)
 				return BAN::Error::from_errno(ENOMEM);
-			m_page_table.map_page_at(m_paddrs[i], vaddr() + i * PAGE_SIZE, m_flags);
-		}
-
-		if (&PageTable::current() == &m_page_table || &PageTable::kernel() == &m_page_table)
-			memset(reinterpret_cast<void*>(vaddr()), 0, size());
-		else
-		{
-			const size_t page_count = size() / PAGE_SIZE;
-			for (size_t i = 0; i < page_count; i++)
-			{
-				PageTable::with_fast_page(m_paddrs[i], [&] {
+			PageTable::with_fast_page(paddr, [] {
 				memset(PageTable::fast_page_as_ptr(), 0, PAGE_SIZE);
 			});
+			m_page_table.map_page_at(paddr, vaddr() + i * PAGE_SIZE, m_flags);
 		}
-		}
-
 		return {};
 	}
 
-	BAN::ErrorOr<bool> VirtualRange::allocate_page_for_demand_paging(vaddr_t vaddr)
-	{
-		ASSERT(contains(vaddr));
-		vaddr &= PAGE_ADDR_MASK;
-
-		if (m_preallocated)
-			return false;
-
-		SpinLockGuard _(m_lock);
-
-		const size_t index = (vaddr - this->vaddr()) / PAGE_SIZE;
-		if (m_paddrs[index])
-			return false;
-
-		m_paddrs[index] = Heap::get().take_free_page();
-		if (m_paddrs[index] == 0)
-			return BAN::Error::from_errno(ENOMEM);
-
-		PageTable::with_fast_page(m_paddrs[index], []{
-			memset(PageTable::fast_page_as_ptr(), 0x00, PAGE_SIZE);
-		});
-
-		m_page_table.map_page_at(m_paddrs[index], vaddr, m_flags);
-
-		return true;
-	}
-
 }

kernel/kernel/Memory/kmalloc.cpp

@@ -436,12 +436,3 @@ BAN::Optional<Kernel::paddr_t> kmalloc_paddr_of(Kernel::vaddr_t vaddr)
 		return {};
 	return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
 }
-
-BAN::Optional<Kernel::vaddr_t> kmalloc_vaddr_of(Kernel::paddr_t paddr)
-{
-	using namespace Kernel;
-	const vaddr_t vaddr = paddr + KERNEL_OFFSET - g_boot_info.kernel_paddr;
-	if (!is_kmalloc_vaddr(vaddr))
-		return {};
-	return vaddr;
-}

kernel/kernel/Networking/E1000/E1000.cpp

@@ -173,7 +173,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> E1000::initialize_rx()
 	{
-		m_rx_buffer_region = TRY(DMARegion::create(E1000_RX_BUFFER_SIZE * E1000_RX_DESCRIPTOR_COUNT));
+		m_rx_buffer_region = TRY(DMARegion::create(E1000_RX_BUFFER_SIZE * E1000_RX_DESCRIPTOR_COUNT, PageTable::MemoryType::Normal));
 		m_rx_descriptor_region = TRY(DMARegion::create(sizeof(e1000_rx_desc) * E1000_RX_DESCRIPTOR_COUNT));
 
 		auto* rx_descriptors = reinterpret_cast<volatile e1000_rx_desc*>(m_rx_descriptor_region->vaddr());
@@ -207,7 +207,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> E1000::initialize_tx()
 	{
-		m_tx_buffer_region = TRY(DMARegion::create(E1000_TX_BUFFER_SIZE * E1000_TX_DESCRIPTOR_COUNT));
+		m_tx_buffer_region = TRY(DMARegion::create(E1000_TX_BUFFER_SIZE * E1000_TX_DESCRIPTOR_COUNT, PageTable::MemoryType::Normal));
 		m_tx_descriptor_region = TRY(DMARegion::create(sizeof(e1000_tx_desc) * E1000_TX_DESCRIPTOR_COUNT));
 
 		auto* tx_descriptors = reinterpret_cast<volatile e1000_tx_desc*>(m_tx_descriptor_region->vaddr());
@@ -304,7 +304,7 @@ namespace Kernel
 		// FIXME: there isnt really any reason to wait for transmission
 		write32(REG_TDT, (tx_current + 1) % E1000_TX_DESCRIPTOR_COUNT);
 		while (descriptor.status == 0)
-			continue;
+			Processor::pause();
 
 		dprintln_if(DEBUG_E1000, "sent {} bytes", packet_size);
 

kernel/kernel/Networking/Loopback.cpp

@@ -14,11 +14,10 @@ namespace Kernel
 
 		loopback->m_buffer = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET,
-			BAN::numeric_limits<vaddr_t>::max(),
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			buffer_size * buffer_count,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, false
+			false
 		));
 
 		auto* thread = TRY(Thread::create_kernel([](void* loopback_ptr) {

kernel/kernel/Networking/NetworkLayer.cpp

@@ -18,16 +18,20 @@ namespace Kernel
 
 			const uint16_t* buffer_u16 = reinterpret_cast<const uint16_t*>(buffer.data());
 			for (size_t j = 0; j < buffer.size() / 2; j++)
-				checksum += BAN::host_to_network_endian(buffer_u16[j]);
-			if (buffer.size() % 2 == 0)
-				continue;
+				checksum += buffer_u16[j];
+
+			if (buffer.size() % 2)
+			{
+				// NOTE: we only allow last buffer to be odd-length
 				ASSERT(i == buffers.size() - 1);
-			checksum += buffer[buffer.size() - 1] << 8;
+				checksum += buffer[buffer.size() - 1];
+			}
 		}
 
 		while (checksum >> 16)
-			checksum = (checksum >> 16) + (checksum & 0xFFFF);
-		return ~(uint16_t)checksum;
+			checksum = (checksum & 0xFFFF) + (checksum >> 16);
+
+		return BAN::host_to_network_endian<uint16_t>(~checksum);
 	}
 
 }

kernel/kernel/Networking/RTL8169/RTL8169.cpp

@@ -107,7 +107,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> RTL8169::initialize_rx()
 	{
-		m_rx_buffer_region = TRY(DMARegion::create(m_rx_descriptor_count * s_buffer_size));
+		m_rx_buffer_region = TRY(DMARegion::create(m_rx_descriptor_count * s_buffer_size, PageTable::MemoryType::Normal));
 		m_rx_descriptor_region = TRY(DMARegion::create(m_rx_descriptor_count * sizeof(RTL8169Descriptor)));
 
 		for (size_t i = 0; i < m_rx_descriptor_count; i++)
@@ -144,7 +144,7 @@ namespace Kernel
 
 	BAN::ErrorOr<void> RTL8169::initialize_tx()
 	{
-		m_tx_buffer_region = TRY(DMARegion::create(m_tx_descriptor_count * s_buffer_size));
+		m_tx_buffer_region = TRY(DMARegion::create(m_tx_descriptor_count * s_buffer_size, PageTable::MemoryType::Normal));
 		m_tx_descriptor_region = TRY(DMARegion::create(m_tx_descriptor_count * sizeof(RTL8169Descriptor)));
 
 		for (size_t i = 0; i < m_tx_descriptor_count; i++)

kernel/kernel/Networking/UDPSocket.cpp

@@ -14,11 +14,10 @@ namespace Kernel
 		auto socket = TRY(BAN::RefPtr<UDPSocket>::create(network_layer, info));
 		socket->m_packet_buffer = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET,
-			~(uintptr_t)0,
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			packet_buffer_size,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, false
+			false
 		));
 		return socket;
 	}

kernel/kernel/Networking/UNIX/Socket.cpp

@@ -14,15 +14,7 @@
 namespace Kernel
 {
 
-	struct UnixSocketHash
-	{
-		BAN::hash_t operator()(const BAN::RefPtr<Inode>& socket)
-		{
-			return BAN::hash<const Inode*>{}(socket.ptr());
-		}
-	};
-
-	static BAN::HashMap<BAN::RefPtr<Inode>, BAN::WeakPtr<UnixDomainSocket>, UnixSocketHash> s_bound_sockets;
+	static BAN::HashMap<BAN::RefPtr<Inode>, BAN::WeakPtr<UnixDomainSocket>> s_bound_sockets;
 	static Mutex s_bound_socket_lock;
 
 	static constexpr size_t s_packet_buffer_size = 0x10000;
@@ -50,11 +42,10 @@ namespace Kernel
 		auto socket = TRY(BAN::RefPtr<UnixDomainSocket>::create(socket_type, info));
 		socket->m_packet_buffer = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET,
-			~(uintptr_t)0,
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			s_packet_buffer_size,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, false
+			false
 		));
 		return socket;
 	}

kernel/kernel/Process.cpp

@@ -34,6 +34,8 @@
 #include <sys/sysmacros.h>
 #include <sys/wait.h>
 
+#pragma GCC diagnostic ignored "-Wstack-usage="
+
 namespace Kernel
 {
 
@@ -3213,6 +3215,9 @@ namespace Kernel
 				return BAN::Error::from_errno(ENOSYS);
 		}
 
+		if (op == FUTEX_WAIT && BAN::atomic_load(*addr) != val)
+			return BAN::Error::from_errno(EAGAIN);
+
 		uint64_t wake_time_ns = BAN::numeric_limits<uint64_t>::max();
 
 		if (user_abstime != nullptr)
@@ -3234,9 +3239,6 @@ namespace Kernel
 			}
 		}
 
-		if (op == FUTEX_WAIT && BAN::atomic_load(*addr) != val)
-			return BAN::Error::from_errno(EAGAIN);
-
 		futex_t* futex;
 
 		{
@@ -3312,7 +3314,6 @@ namespace Kernel
 	BAN::ErrorOr<long> Process::sys_set_fsbase(void* addr)
 	{
 		auto& thread = Thread::current();
-		thread.m_has_custom_fsbase = true;
 		thread.set_fsbase(reinterpret_cast<vaddr_t>(addr));
 		Processor::load_fsbase();
 		return 0;
@@ -3326,7 +3327,6 @@ namespace Kernel
 	BAN::ErrorOr<long> Process::sys_set_gsbase(void* addr)
 	{
 		auto& thread = Thread::current();
-		thread.m_has_custom_gsbase = true;
 		thread.set_gsbase(reinterpret_cast<vaddr_t>(addr));
 		Processor::load_gsbase();
 		return 0;
@@ -3355,9 +3355,8 @@ namespace Kernel
 	{
 		LockGuard _(m_process_lock);
 
-		// main thread cannot call pthread_exit
-		if (&Thread::current() == m_threads.front())
-			return BAN::Error::from_errno(EINVAL);
+		if (m_threads.size() == 1)
+			return sys_exit(0);
 
 		TRY(m_exited_pthreads.emplace_back(Thread::current().tid(), value));
 
@@ -3397,6 +3396,7 @@ namespace Kernel
 		{
 			if (auto ret = check_thread(); ret.has_value())
 			{
+				if (user_value != nullptr)
 					TRY(write_to_user(user_value, &ret.value(), sizeof(void*)));
 				return 0;
 			}

kernel/kernel/Processor.cpp

@@ -73,9 +73,6 @@ namespace Kernel
 		ASSERT(processor.m_id == PROCESSOR_NONE);
 		processor.m_id = id;
 
-		processor.m_stack = kmalloc(s_stack_size, 4096, true);
-		ASSERT(processor.m_stack);
-
 		processor.m_gdt = GDT::create(&processor);
 		ASSERT(processor.m_gdt);
 
@@ -157,6 +154,21 @@ namespace Kernel
 		return processor;
 	}
 
+	// NOTE: I don't like this being a separate function but we need heap and page tables for this :)
+	void Processor::allocate_stack()
+	{
+		ASSERT(m_stack_paddr == 0);
+		ASSERT(m_stack_vaddr == 0);
+
+		m_stack_paddr = Heap::get().take_free_page();
+		ASSERT(m_stack_paddr);
+
+		m_stack_vaddr = PageTable::kernel().reserve_free_page(KERNEL_OFFSET);
+		ASSERT(m_stack_vaddr);
+
+		PageTable::kernel().map_page_at(m_stack_paddr, m_stack_vaddr, PageTable::ReadWrite | PageTable::Present);
+	}
+
 	void Processor::initialize_smp()
 	{
 		const auto processor_id = current_id();
@@ -205,8 +217,7 @@ namespace Kernel
 		memset(reinterpret_cast<void*>(s_shared_page_vaddr), 0, PAGE_SIZE);
 
 		auto& shared_page = *reinterpret_cast<volatile API::SharedPage*>(s_shared_page_vaddr);
-		for (size_t i = 0; i <= 0xFF; i++)
-			shared_page.__sequence[i] = i;
+		shared_page.gdt_cpu_offset = GDT::cpu_index_offset();
 		shared_page.features = 0;
 
 		ASSERT(Processor::count() + sizeof(Kernel::API::SharedPage) <= PAGE_SIZE);
@@ -565,7 +576,7 @@ namespace Kernel
 		if (!scheduler().is_idle())
 			Thread::current().set_cpu_time_stop();
 
-		asm_yield_trampoline(Processor::current_stack_top());
+		asm_yield_trampoline(processor_info.stack_top_vaddr());
 
 		processor_info.m_start_ns = SystemTimer::get().ns_since_boot();
 

kernel/kernel/Scheduler.cpp

@@ -307,15 +307,14 @@ namespace Kernel
 
 	void Scheduler::wake_up_sleeping_threads()
 	{
+		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
+
 		const uint64_t current_ns = SystemTimer::get().ns_since_boot();
 		while (!m_block_queue.empty() && current_ns >= m_block_queue.front()->wake_time_ns)
 		{
 			auto* node = m_block_queue.pop_front();
-			{
-				SpinLockGuard _(node->blocker_lock);
-				if (node->blocker)
-					node->blocker->remove_blocked_thread(node);
-			}
+			if (auto* blocker = node->blocker.load())
+				blocker->remove_thread_from_block_queue(node);
 			node->blocked = false;
 			update_most_loaded_node_queue(node, &m_run_queue);
 			m_run_queue.add_thread_to_back(node);
@@ -368,11 +367,8 @@ namespace Kernel
 				return;
 			if (node != m_current)
 				m_block_queue.remove_node(node);
-			{
-				SpinLockGuard _(node->blocker_lock);
-				if (node->blocker)
-					node->blocker->remove_blocked_thread(node);
-			}
+			if (auto* blocker = node->blocker.load())
+				blocker->remove_thread_from_block_queue(node);
 			node->blocked = false;
 			if (node != m_current)
 				m_run_queue.add_thread_to_back(node);
@@ -665,11 +661,8 @@ namespace Kernel
 		m_current->blocked = true;
 		m_current->wake_time_ns = wake_time_ns;
 
-		{
-			SpinLockGuard _(m_current->blocker_lock);
 		if (blocker)
 			blocker->add_thread_to_block_queue(m_current);
-		}
 
 		update_most_loaded_node_queue(m_current, &m_block_queue);
 

kernel/kernel/Terminal/PseudoTerminal.cpp

@@ -42,10 +42,10 @@ namespace Kernel
 
 		auto pts_master_buffer = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET, static_cast<vaddr_t>(-1),
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			16 * PAGE_SIZE,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, false
+			false
 		));
 		auto pts_master = TRY(BAN::RefPtr<PseudoTerminalMaster>::create(BAN::move(pts_master_buffer), mode, uid, gid));
 		DevFileSystem::get().remove_from_cache(pts_master);

kernel/kernel/Thread.cpp

@@ -171,11 +171,10 @@ namespace Kernel
 		// Initialize stack and registers
 		thread->m_kernel_stack = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET,
-			~(uintptr_t)0,
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			kernel_stack_size,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, true
+			true
 		));
 
 		// Initialize stack for returning
@@ -208,10 +207,10 @@ namespace Kernel
 
 		thread->m_kernel_stack = TRY(VirtualRange::create_to_vaddr_range(
 			page_table,
-			s_user_stack_addr_start, USERSPACE_END,
+			{ s_user_stack_addr_start, USERSPACE_END },
 			kernel_stack_size,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, true
+			true
 		));
 
 		auto userspace_stack = TRY(MemoryBackedRegion::create(
@@ -304,22 +303,7 @@ namespace Kernel
 	{
 		if (!is_userspace() || !has_process())
 			return;
-
-#if ARCH(x86_64)
-		if (m_has_custom_gsbase)
-			return;
-#elif ARCH(i686)
-		if (m_has_custom_fsbase)
-			return;
-#endif
-
-		const vaddr_t vaddr = process().shared_page_vaddr() + Processor::current_index();
-
-#if ARCH(x86_64)
-		set_gsbase(vaddr);
-#elif ARCH(i686)
-		set_fsbase(vaddr);
-#endif
+		Processor::gdt().set_cpu_index(Processor::current_index());
 	}
 
 	BAN::ErrorOr<Thread*> Thread::pthread_create(entry_t entry, void* arg)
@@ -356,10 +340,10 @@ namespace Kernel
 
 		thread->m_kernel_stack = TRY(VirtualRange::create_to_vaddr_range(
 			new_process->page_table(),
-			s_user_stack_addr_start, USERSPACE_END,
+			{ s_user_stack_addr_start, USERSPACE_END },
 			kernel_stack_size,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, true
+			true
 		));
 
 		// NOTE: copy [sp, stack_end] so fork return works

kernel/kernel/ThreadBlocker.cpp

@@ -22,61 +22,65 @@ namespace Kernel
 	}
 
 	void ThreadBlocker::unblock()
-	{
-		decltype(m_block_chain) temp_block_chain;
-		size_t temp_block_chain_length { 0 };
-
 	{
 		SpinLockGuard _(m_lock);
-			for (size_t i = 0; i < m_block_chain_length; i++)
-				temp_block_chain[i] = m_block_chain[i];
-			temp_block_chain_length = m_block_chain_length;
-			m_block_chain_length = 0;
+
+		for (auto* node = m_block_chain; node;)
+		{
+			auto* next = node->block_chain_next;
+
+			ASSERT(node->blocked);
+			ASSERT(node->blocker == this);
+
+			node->blocker.store(nullptr);
+			node->block_chain_prev = nullptr;
+			node->block_chain_next = nullptr;
+
+			Processor::scheduler().unblock_thread(node);
+
+			node = next;
 		}
 
-		for (size_t i = 0; i < temp_block_chain_length; i++)
-			Processor::scheduler().unblock_thread(temp_block_chain[i]);
+		m_block_chain = nullptr;
 	}
 
 	void ThreadBlocker::add_thread_to_block_queue(SchedulerQueue::Node* node)
 	{
-		ASSERT(node->blocker_lock.current_processor_has_lock());
-
 		SpinLockGuard _(m_lock);
 
-		ASSERT(m_block_chain_length < sizeof(m_block_chain) / sizeof(m_block_chain[0]));
-
-		ASSERT(node);
 		ASSERT(node->blocked);
 		ASSERT(node->blocker == nullptr);
 
-		for (size_t i = 0 ; i < m_block_chain_length; i++)
-			ASSERT(m_block_chain[i] != node);
-		m_block_chain[m_block_chain_length++] = node;
+		node->blocker.store(this);
+		node->block_chain_prev = nullptr;
+		node->block_chain_next = m_block_chain;
 
-		node->blocker = this;
+		if (m_block_chain)
+			m_block_chain->block_chain_prev = node;
+		m_block_chain = node;
 	}
 
-	void ThreadBlocker::remove_blocked_thread(SchedulerQueue::Node* node)
+	void ThreadBlocker::remove_thread_from_block_queue(SchedulerQueue::Node* node)
 	{
-		ASSERT(node->blocker_lock.current_processor_has_lock());
-
 		SpinLockGuard _(m_lock);
 
-		ASSERT(node);
+		// NOTE: this is possible if we got here while another
+		//       core was doing an unblock on this blocker
+		if (node->blocker.load() != this)
+			return;
 		ASSERT(node->blocked);
-		ASSERT(node->blocker == this);
 
-		for (size_t i = 0 ; i < m_block_chain_length; i++)
-		{
-			if (m_block_chain[i] != node)
-				continue;
-			for (size_t j = i + 1; j < m_block_chain_length; j++)
-				m_block_chain[j - 1] = m_block_chain[j];
-			m_block_chain_length--;
-		}
+		if (node->block_chain_prev)
+			node->block_chain_prev->block_chain_next = node->block_chain_next;
+		if (node->block_chain_next)
+			node->block_chain_next->block_chain_prev = node->block_chain_prev;
 
-		node->blocker = nullptr;
+		if (node == m_block_chain)
+			m_block_chain = node->block_chain_next;
+
+		node->blocker.store(nullptr);
+		node->block_chain_prev = nullptr;
+		node->block_chain_next = nullptr;
 	}
 
 }

kernel/kernel/kernel.cpp

@@ -126,19 +126,20 @@ extern "C" void kernel_main(uint32_t boot_magic, uint32_t boot_info)
 	parse_boot_info(boot_magic, boot_info);
 	dprintln("boot info parsed");
 
-	Processor::create(PROCESSOR_NONE);
+	auto& processor = Processor::create(PROCESSOR_NONE);
 	Processor::initialize();
 	dprintln("BSP initialized");
 
-	PageTable::initialize_pre_heap();
-	PageTable::kernel().initial_load();
-	dprintln("PageTable stage1 initialized");
+	PageTable::initialize_fast_page();
+	dprintln("fast page initialized");
 
 	Heap::initialize();
-	dprintln("Heap initialzed");
+	dprintln("Heap initialized");
 
-	PageTable::initialize_post_heap();
-	dprintln("PageTable stage2 initialized");
+	PageTable::initialize_and_load();
+	dprintln("PageTable initialized");
+
+	processor.allocate_stack();
 
 	parse_command_line();
 	dprintln("command line parsed, root='{}', console='{}'", cmdline.root, cmdline.console);
@@ -270,7 +271,7 @@ extern "C" void ap_main()
 	using namespace Kernel;
 
 	Processor::initialize();
-	PageTable::kernel().initial_load();
+
 	InterruptController::get().enable();
 
 	Processor::wait_until_processors_ready();

ports/ca-certificates/build.sh

@@ -1,8 +1,8 @@
 #!/bin/bash ../install.sh
 
 NAME='ca-certificates'
-VERSION='2025-12-02'
-DOWNLOAD_URL="https://curl.se/ca/cacert-$VERSION.pem#f1407d974c5ed87d544bd931a278232e13925177e239fca370619aba63c757b4"
+VERSION='2026.03.19'
+DOWNLOAD_URL="https://curl.se/ca/cacert-${VERSION//./-}.pem#b6e66569cc3d438dd5abe514d0df50005d570bfc96c14dca8f768d020cb96171"
 
 configure() {
 	:
@@ -13,7 +13,10 @@ build() {
 }
 
 install() {
-	mkdir -p "$BANAN_SYSROOT/etc/ssl/certs"
-	cp -v "../cacert-$VERSION.pem" "$BANAN_SYSROOT/etc/ssl/certs/ca-certificates.crt"
-	ln -svf "certs/ca-certificates.crt" "$BANAN_SYSROOT/etc/ssl/cert.pem"
+	rm -rf "$BANAN_SYSROOT/etc/cacert/extracted"
+	mkdir -p "$BANAN_SYSROOT/etc/cacert/extracted"
+
+	cp -vf "../cacert-${VERSION//./-}.pem" "$BANAN_SYSROOT/etc/cacert/cacert.pem"
+	awk '/-----BEGIN CERTIFICATE-----/ {c=1;n++} c {print > sprintf("cert%03d.pem", n)} /-----END CERTIFICATE-----/ {c=0}' "../cacert-${VERSION//./-}.pem"
+	mv cert*.pem "$BANAN_SYSROOT/etc/cacert/extracted/"
 }

ports/curl/build.sh

@@ -3,7 +3,7 @@
 NAME='curl'
 VERSION='8.17.0'
 DOWNLOAD_URL="https://curl.se/download/curl-$VERSION.tar.xz#955f6e729ad6b3566260e8fef68620e76ba3c31acf0a18524416a185acf77992"
-DEPENDENCIES=('ca-certificates' 'openssl' 'zlib' 'zstd')
+DEPENDENCIES=('openssl' 'zlib' 'zstd')
 CONFIG_SUB=('config.sub')
 CONFIGURE_OPTIONS=(
 	'--disable-threaded-resolver'
@@ -16,6 +16,6 @@ CONFIGURE_OPTIONS=(
 	'--with-zlib'
 	'--with-zstd'
 	'--without-libpsl'
-	'--with-ca-bundle=/etc/ssl/certs/ca-certificates.crt'
-	'--without-ca-path'
+	'--with-ca-path=/etc/ssl/certs'
+	'--with-ca-bundle=/etc/ssl/certs/ca-bundle.crt'
 )

ports/openssl/build.sh

@@ -3,9 +3,24 @@
 NAME='openssl'
 VERSION='3.6.0'
 DOWNLOAD_URL="https://github.com/openssl/openssl/releases/download/openssl-$VERSION/openssl-$VERSION.tar.gz#b6a5f44b7eb69e3fa35dbf15524405b44837a481d43d81daddde3ff21fcbb8e9"
-DEPENDENCIES=('zlib')
+DEPENDENCIES=('ca-certificates' 'zlib')
 MAKE_INSTALL_TARGETS=('install_sw' 'install_ssldirs')
 
 configure() {
 	./Configure --prefix=/usr --openssldir=/etc/ssl -DOPENSSL_USE_IPV6=0 no-asm no-tests banan_os-generic threads zlib
 }
+
+post_install() {
+	rm -f "$BANAN_SYSROOT/etc/ssl/certs"/*
+
+	ln -svf "../cacert/cacert.pem" "$BANAN_SYSROOT/etc/ssl/cert.pem"
+	ln -svf "../../cacert/cacert.pem" "$BANAN_SYSROOT/etc/ssl/certs/ca-certificates.crt"
+	ln -svf "../../cacert/cacert.pem" "$BANAN_SYSROOT/etc/ssl/certs/ca-bundle.crt"
+
+	openssl rehash "$BANAN_SYSROOT/etc/cacert/extracted"
+	find "$BANAN_SYSROOT/etc/cacert/extracted" -type l -print0 |
+	while IFS= read -r -d '' link; do
+		ln -s "../../cacert/extracted/$(readlink "$link")" "$BANAN_SYSROOT/etc/ssl/certs/${link##*/}"
+		rm "$link"
+	done
+}

userspace/libraries/CMakeLists.txt

@@ -3,14 +3,11 @@ set(USERSPACE_LIBRARIES
 	LibC
 	LibClipboard
 	LibDEFLATE
-	LibDL
 	LibELF
 	LibFont
 	LibGUI
 	LibImage
 	LibInput
-	LibMath
-	LibPthread
 	LibQR
 )
 

userspace/libraries/LibC/CMakeLists.txt

@@ -76,7 +76,7 @@ set(LIBC_SOURCES
 add_library(objlibc OBJECT ${LIBC_SOURCES})
 target_compile_definitions(objlibc PRIVATE __arch=${BANAN_ARCH} __is_libc)
 
-target_compile_options(objlibc PRIVATE -O2 -g -Wstack-usage=512 -fno-exceptions -fno-rtti -fpic)
+target_compile_options(objlibc PRIVATE -O2 -g -Wstack-usage=8192 -fno-exceptions -fno-rtti -fpic)
 target_compile_options(objlibc PUBLIC -Wall -Wextra -Werror -Wno-error=stack-usage=)
 
 function(add_crtx crtx)
@@ -105,6 +105,13 @@ target_link_options(libc-shared PRIVATE -nolibc -nostdlib++)
 install(TARGETS libc-static OPTIONAL)
 install(TARGETS libc-shared OPTIONAL)
 
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \"libc.a\"  \"${CMAKE_INSTALL_LIBDIR}/libdl.a\")")
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \"libc.so\" \"${CMAKE_INSTALL_LIBDIR}/libdl.so\")")
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \"libc.a\"  \"${CMAKE_INSTALL_LIBDIR}/libm.a\")")
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \"libc.so\" \"${CMAKE_INSTALL_LIBDIR}/libm.so\")")
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \"libc.a\"  \"${CMAKE_INSTALL_LIBDIR}/libpthread.a\")")
+install(CODE "execute_process(COMMAND ${CMAKE_COMMAND} -E create_symlink \"libc.so\" \"${CMAKE_INSTALL_LIBDIR}/libpthread.so\")")
+
 set_target_properties(libc-static PROPERTIES OUTPUT_NAME libc)
 set_target_properties(libc-shared PROPERTIES OUTPUT_NAME libc)
 

userspace/libraries/LibC/include/limits.h

@@ -111,7 +111,7 @@ __BEGIN_DECLS
 #define MAX_CANON                _POSIX_MAX_CANON
 #define MAX_INPUT                _POSIX_MAX_INPUT
 #define NAME_MAX                 255
-#define PATH_MAX                 _POSIX_PATH_MAX
+#define PATH_MAX                 4096
 #define PIPE_BUF                 PAGE_SIZE
 //#define POSIX_ALLOC_SIZE_MIN
 //#define POSIX_REC_INCR_XFER_SIZE

userspace/libraries/LibC/include/pthread.h

@@ -169,6 +169,12 @@ void		pthread_testcancel(void);
 void		pthread_cleanup_pop(int execute);
 void		pthread_cleanup_push(void (*routine)(void*), void* arg);
 
+#define _pthread_equal(t1, t2) ((t1) == (t2))
+#define pthread_equal(t1, t2) _pthread_equal(t1, t2)
+
+#define _pthread_self() (_get_uthread()->id)
+#define pthread_self() _pthread_self()
+
 #define _pthread_testcancel() do { \
 		struct uthread* uthread = _get_uthread(); \
 		if (__builtin_expect(uthread->cancel_state == PTHREAD_CANCEL_ENABLE, 1)) \

userspace/libraries/LibC/include/stdlib.h

@@ -56,7 +56,7 @@ int					clearenv(void);
 div_t				div(int numer, int denom);
 double				drand48(void);
 double				erand48(unsigned short xsubi[3]);
-void				exit(int status);
+void				exit(int status) __attribute__((__noreturn__));
 void				free(void* ptr);
 char*				getenv(const char* name);
 int					getsubopt(char** optionp, char* const* keylistp, char** valuep);

userspace/libraries/LibC/printf_impl.cpp

@@ -274,6 +274,8 @@ static void floating_point_to_maybe_exponent_string(char* buffer, T value, bool
 
 extern "C" int printf_impl(const char* format, va_list arguments, int (*putc_fun)(int, void*), void* data)
 {
+	char conversion[4096];
+
 	int written = 0;
 	while (*format)
 	{
@@ -383,9 +385,6 @@ extern "C" int printf_impl(const char* format, va_list arguments, int (*putc_fun
 				format--;
 			format++;
 
-			// FIXME: this should be thread-local to keep
-			//        satisfy multithreaded requirement
-			static char conversion[4096];
 			const char* string = nullptr;
 
 			int length = -1;

userspace/libraries/LibC/pthread.cpp

@@ -489,29 +489,29 @@ void pthread_exit(void* value_ptr)
 	ASSERT_NOT_REACHED();
 }
 
+#undef pthread_equal
 int pthread_equal(pthread_t t1, pthread_t t2)
 {
-	return t1 == t2;
+	return _pthread_equal(t1, t2);
 }
+#define pthread_equal(t1, t2) _pthread_equal(t1, t2)
+
+#undef pthread_self
+pthread_t pthread_self(void)
+{
+	return _pthread_self();
+}
+#define pthread_self() _pthread_self()
 
 int pthread_join(pthread_t thread, void** value_ptr)
 {
-	pthread_testcancel();
-
-	errno = 0;
-	while (syscall(SYS_PTHREAD_JOIN, thread, value_ptr) == -1 && errno == EINTR)
-	{
+	do {
 		pthread_testcancel();
 		errno = 0;
-	}
+	} while (syscall(SYS_PTHREAD_JOIN, thread, value_ptr) == -1 && errno == EINTR);
 	return errno;
 }
 
-pthread_t pthread_self(void)
-{
-	return _get_uthread()->id;
-}
-
 int pthread_once(pthread_once_t* once_control, void (*init_routine)(void))
 {
 	static_assert(PTHREAD_ONCE_INIT == 0);
@@ -671,6 +671,7 @@ void pthread_testcancel(void)
 {
 	_pthread_testcancel();
 }
+#define pthread_testcancel() _pthread_testcancel()
 
 int pthread_getschedparam(pthread_t thread, int* __restrict policy, struct sched_param* __restrict param)
 {

userspace/libraries/LibC/sched.cpp

@@ -27,12 +27,7 @@ int sched_getcpu(void)
 {
 	if (g_shared_page == nullptr)
 		return -1;
-
-	uint8_t cpu;
-#if defined(__x86_64__)
-	asm volatile("movb %%gs:0, %0" : "=r"(cpu));
-#elif defined(__i686__)
-	asm volatile("movb %%fs:0, %0" : "=q"(cpu));
-#endif
-	return cpu;
+	uint16_t limit;
+	asm volatile("lsl %1, %0" : "=r"(limit) : "r"(g_shared_page->gdt_cpu_offset));
+	return limit;
 }

userspace/libraries/LibC/stdlib.cpp

@@ -37,7 +37,6 @@ void exit(int status)
 	__cxa_finalize(nullptr);
 	fflush(nullptr);
 	_exit(status);
-	ASSERT_NOT_REACHED();
 }
 
 void _Exit(int status)
@@ -828,35 +827,40 @@ void qsort(void* base, size_t nel, size_t width, int (*compar)(const void*, cons
 
 // Constants and algorithm from https://en.wikipedia.org/wiki/Permuted_congruential_generator
 
-static uint64_t s_rand_state = 0x4d595df4d0f33173;
 static constexpr uint64_t s_rand_multiplier = 6364136223846793005;
 static constexpr uint64_t s_rand_increment = 1442695040888963407;
+static uint64_t s_rand_state;
 
 static constexpr uint32_t rotr32(uint32_t x, unsigned r)
 {
 	return x >> r | x << (-r & 31);
 }
 
-int rand_r(unsigned* seed)
+template<BAN::integral T>
+static inline int rand_impl(T& state)
 {
-	uint64_t x = *seed | (static_cast<uint64_t>(*seed) << 32);
-	unsigned count = (unsigned)(x >> 59);
+	uint64_t x;
+	if constexpr (sizeof(T) == 8) x = state;
+	if constexpr (sizeof(T) == 4) x = state * 0x0000000100000001;
+	if constexpr (sizeof(T) == 2) x = state * 0x0001000100010001;
+	if constexpr (sizeof(T) == 1) x = state * 0x0101010101010101;
 
-	*seed = x * s_rand_multiplier + s_rand_increment;
+	const unsigned count = x >> 59;
+
+	state = x * s_rand_multiplier + s_rand_increment;
 	x ^= x >> 18;
 
 	return rotr32(x >> 27, count) % RAND_MAX;
 }
 
+int rand_r(unsigned* seed)
+{
+	return rand_impl(*seed);
+}
+
 int rand(void)
 {
-	uint64_t x = s_rand_state;
-	unsigned count = (unsigned)(x >> 59);
-
-	s_rand_state = x * s_rand_multiplier + s_rand_increment;
-	x ^= x >> 18;
-
-	return rotr32(x >> 27, count) % RAND_MAX;
+	return rand_impl(s_rand_state);
 }
 
 void srand(unsigned int seed)
@@ -953,4 +957,5 @@ void init_default_random()
 {
 	static char buffer[128];
 	initstate(1, buffer, 128);
+	srand(1);
 }

userspace/libraries/LibC/time.cpp

@@ -30,13 +30,9 @@ int clock_gettime(clockid_t clock_id, struct timespec* tp)
 
 	const auto get_cpu =
 		[]() -> uint8_t {
-			uint8_t cpu;
-#if defined(__x86_64__)
-			asm volatile("movb %%gs:0, %0" : "=r"(cpu));
-#elif defined(__i686__)
-			asm volatile("movb %%fs:0, %0" : "=q"(cpu));
-#endif
-			return cpu;
+			uint16_t limit;
+			asm volatile("lsl %1, %0" : "=r"(limit) : "r"(g_shared_page->gdt_cpu_offset));
+			return limit;
 		};
 
 	for (;;)

userspace/libraries/LibDL/CMakeLists.txt

@@ -1,18 +0,0 @@
-set(SOURCES
-	dummy.cpp
-)
-
-add_library(libdl-static STATIC ${SOURCES})
-add_library(libdl-shared SHARED ${SOURCES})
-
-target_link_options(libdl-static PRIVATE -nolibc)
-target_link_options(libdl-shared PRIVATE -nolibc)
-
-banan_link_library(libdl-static libc)
-banan_link_library(libdl-shared libc)
-
-set_target_properties(libdl-static PROPERTIES OUTPUT_NAME libdl)
-set_target_properties(libdl-shared PROPERTIES OUTPUT_NAME libdl)
-
-install(TARGETS libdl-static OPTIONAL)
-install(TARGETS libdl-shared OPTIONAL)

userspace/libraries/LibInput/KeyboardLayout.cpp

@@ -19,20 +19,14 @@
 namespace LibInput
 {
 
-	struct StringViewLower
+	struct StringViewLowerComp
 	{
-		BAN::StringView value;
-
-		StringViewLower(BAN::StringView sv)
-			: value(sv)
-		{ }
-
-		bool operator==(const StringViewLower& other) const
+		constexpr bool operator()(const BAN::StringView& a, const BAN::StringView& b) const
 		{
-			if (value.size() != other.value.size())
+			if (a.size() != b.size())
 				return false;
-			for (size_t i = 0; i < value.size(); i++)
-				if (tolower(value[i]) != tolower(other.value[i]))
+			for (size_t i = 0; i < a.size(); i++)
+				if (tolower(a[i]) != tolower(b[i]))
 					return false;
 			return true;
 		}
@@ -40,16 +34,16 @@ namespace LibInput
 
 	struct StringViewLowerHash
 	{
-		BAN::hash_t operator()(const StringViewLower& value) const
+		BAN::hash_t operator()(const BAN::StringView& str) const
 		{
 			constexpr BAN::hash_t FNV_offset_basis = 0x811c9dc5;
 			constexpr BAN::hash_t FNV_prime = 0x01000193;
 
 			BAN::hash_t hash = FNV_offset_basis;
-			for (size_t i = 0; i < value.value.size(); i++)
+			for (size_t i = 0; i < str.size(); i++)
 			{
 				hash *= FNV_prime;
-				hash ^= (uint8_t)tolower(value.value[i]);
+				hash ^= (uint8_t)tolower(str[i]);
 			}
 
 			return hash;
@@ -113,7 +107,7 @@ namespace LibInput
 		return keycode;
 	}
 
-	static BAN::HashMap<StringViewLower, Key, StringViewLowerHash> s_name_to_key;
+	static BAN::HashMap<BAN::StringView, Key, StringViewLowerHash, StringViewLowerComp> s_name_to_key;
 	static BAN::ErrorOr<void> initialize_name_to_key();
 
 	static BAN::Optional<Key> parse_key(BAN::StringView name)

userspace/libraries/LibMath/CMakeLists.txt

@@ -1,18 +0,0 @@
-set(SOURCES
-	dummy.cpp
-)
-
-add_library(libmath-static STATIC ${SOURCES})
-add_library(libmath-shared SHARED ${SOURCES})
-
-target_link_options(libmath-static PRIVATE -nolibc)
-target_link_options(libmath-shared PRIVATE -nolibc)
-
-banan_link_library(libmath-static libc)
-banan_link_library(libmath-shared libc)
-
-set_target_properties(libmath-static PROPERTIES OUTPUT_NAME libm)
-set_target_properties(libmath-shared PROPERTIES OUTPUT_NAME libm)
-
-install(TARGETS libmath-static OPTIONAL)
-install(TARGETS libmath-shared OPTIONAL)

userspace/libraries/LibPthread/CMakeLists.txt

@@ -1,18 +0,0 @@
-set(SOURCES
-	dummy.cpp
-)
-
-add_library(libpthread-static STATIC ${SOURCES})
-add_library(libpthread-shared SHARED ${SOURCES})
-
-target_link_options(libpthread-static PRIVATE -nolibc)
-target_link_options(libpthread-shared PRIVATE -nolibc)
-
-banan_link_library(libpthread-static libc)
-banan_link_library(libpthread-shared libc)
-
-set_target_properties(libpthread-static PROPERTIES OUTPUT_NAME libpthread)
-set_target_properties(libpthread-shared PROPERTIES OUTPUT_NAME libpthread)
-
-install(TARGETS libpthread-static OPTIONAL)
-install(TARGETS libpthread-shared OPTIONAL)

userspace/programs/resolver/main.cpp

@@ -3,6 +3,7 @@
 #include <BAN/Endianness.h>
 #include <BAN/HashMap.h>
 #include <BAN/IPv4.h>
+#include <BAN/LinkedList.h>
 #include <BAN/String.h>
 #include <BAN/StringView.h>
 #include <BAN/Vector.h>