BuildSystem: Fix GCC compilation with new host GCC

Newer GCCs fail to compile GCC 15.2.0 because of how u8"" literals are
handled. This can be bypassed by compiling with -fno-char8_t

BAN/include/BAN/HashMap.h

@@ -1,49 +1,156 @@
 #pragma once
 
-#include <BAN/Hash.h>
+#include <BAN/HashSet.h>
-#include <BAN/LinkedList.h>
-#include <BAN/Vector.h>
 
 namespace BAN
 {
 
-	template<typename Key, typename T, typename HASH = BAN::hash<Key>>
+	template<typename HashSetIt, typename HashMap, typename Entry>
+	class HashMapIterator
+	{
+	public:
+		HashMapIterator() = default;
+
+		Entry& operator*()
+		{
+			return const_cast<Entry&>(m_iterator.operator*());
+		}
+		const Entry& operator*() const
+		{
+			return m_iterator.operator*();
+		}
+
+		Entry* operator->()
+		{
+			return const_cast<Entry*>(m_iterator.operator->());
+		}
+		const Entry* operator->() const
+		{
+			return m_iterator.operator->();
+		}
+
+		HashMapIterator& operator++()
+		{
+			++m_iterator;
+			return *this;
+		}
+		HashMapIterator operator++(int)
+		{
+			auto temp = *this;
+			++(*this);
+			return temp;
+		}
+
+		bool operator==(HashMapIterator other) const
+		{
+			return m_iterator == other.m_iterator;
+		}
+		bool operator!=(HashMapIterator other) const
+		{
+			return m_iterator != other.m_iterator;
+		}
+
+	private:
+		explicit HashMapIterator(HashSetIt it)
+			: m_iterator(it)
+		{ }
+
+	private:
+		HashSetIt m_iterator;
+		friend HashMap;
+	};
+
+	namespace detail
+	{
+		template<typename T, typename Key, typename HASH, typename COMP>
+		concept HashMapFindable = requires(const Key& a, const T& b) { COMP()(a, b); HASH()(b); };
+	}
+
+	template<typename Key, typename T, typename HASH = BAN::hash<Key>, typename COMP = BAN::equal<Key>>
 	class HashMap
 	{
 	public:
 		struct Entry
 		{
-			template<typename... Args>
+			const Key key;
-			Entry(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-				: key(key)
-				, value(forward<Args>(args)...)
-			{}
-
-			template<typename... Args>
-			Entry(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-				: key(BAN::move(key))
-				, value(forward<Args>(args)...)
-			{}
-
-			Key key;
 			T value;
+
+			Entry() = delete;
+			Entry& operator=(const Entry&) = delete;
+			Entry& operator=(Entry&&) = delete;
+
+			Entry(const Entry& other)
+				: key(other.key)
+				, value(other.value)
+			{ }
+
+			Entry(Entry&& other)
+				: key(BAN::move(const_cast<Key&>(other.key)))
+				, value(BAN::move(other.value))
+			{ }
+
+			template<typename... Args>
+			Entry(Key&& key, Args&&... args)
+				: key(BAN::move(key))
+				, value(BAN::forward<Args>(args)...)
+			{ }
+		};
+
+		struct EntryHash
+		{
+			template<detail::HashMapFindable<Key, HASH, COMP> U>
+			constexpr bool operator()(const U& a)
+			{
+				return HASH()(a);
+			}
+			constexpr bool operator()(const Entry& a)
+			{
+				return HASH()(a.key);
+			}
+		};
+
+		struct EntryComp
+		{
+			template<detail::HashMapFindable<Key, HASH, COMP> U>
+			constexpr bool operator()(const Entry& a, const U& b)
+			{
+				return COMP()(a.key, b);
+			}
+			constexpr bool operator()(const Entry& a, const Entry& b)
+			{
+				return COMP()(a.key, b.key);
+			}
 		};
 
 	public:
 		using size_type      = size_t;
 		using key_type       = Key;
 		using value_type     = T;
-		using iterator = IteratorDouble<Entry, Vector, LinkedList, HashMap>;
+		using iterator       = HashMapIterator<typename HashSet<Entry, EntryHash, EntryComp>::iterator,       HashMap, Entry>;
-		using const_iterator = ConstIteratorDouble<Entry, Vector, LinkedList, HashMap>;
+		using const_iterator = HashMapIterator<typename HashSet<Entry, EntryHash, EntryComp>::const_iterator, HashMap, const Entry>;
 
 	public:
 		HashMap() = default;
-		HashMap(const HashMap<Key, T, HASH>&);
+		~HashMap() { clear(); }
-		HashMap(HashMap<Key, T, HASH>&&);
-		~HashMap();
 
-		HashMap<Key, T, HASH>& operator=(const HashMap<Key, T, HASH>&);
+		HashMap(const HashMap& other) { *this = other; }
-		HashMap<Key, T, HASH>& operator=(HashMap<Key, T, HASH>&&);
+		HashMap& operator=(const HashMap& other)
+		{
+			m_hash_set = other.m_hash_set;
+			return *this;
+		}
+
+		HashMap(HashMap&& other) { *this = BAN::move(other); }
+		HashMap& operator=(HashMap&& other)
+		{
+			m_hash_set = BAN::move(other.m_hash_set);
+			return *this;
+		}
+
+		iterator begin()             { return iterator(m_hash_set.begin()); }
+		iterator end()               { return iterator(m_hash_set.end()); }
+		const_iterator begin() const { return const_iterator(m_hash_set.begin()); }
+		const_iterator end() const   { return const_iterator(m_hash_set.end()); }
 
 		ErrorOr<iterator> insert(const Key& key, const T& value)           { return emplace(key, value); }
 		ErrorOr<iterator> insert(const Key& key, T&& value)                { return emplace(key, move(value)); }
@@ -57,263 +164,100 @@ namespace BAN
 
 		template<typename... Args>
 		ErrorOr<iterator> emplace(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-		{ return emplace(Key(key), forward<Args>(args)...); }
+		{ return emplace(Key(key), BAN::forward<Args>(args)...); }
 		template<typename... Args>
-		ErrorOr<iterator> emplace(Key&&, Args&&...) requires is_constructible_v<T, Args...>;
+		ErrorOr<iterator> emplace(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
+		{
+			ASSERT(!contains(key));
+			auto it = TRY(m_hash_set.insert(Entry { BAN::move(key), T(BAN::forward<Args>(args)...) }));
+			return iterator(it);
+		}
 
 		template<typename... Args>
 		ErrorOr<iterator> emplace_or_assign(const Key& key, Args&&... args) requires is_constructible_v<T, Args...>
-		{ return emplace_or_assign(Key(key), forward<Args>(args)...); }
+		{ return emplace_or_assign(Key(key), BAN::forward<Args>(args)...); }
 		template<typename... Args>
-		ErrorOr<iterator> emplace_or_assign(Key&&, Args&&...) requires is_constructible_v<T, Args...>;
+		ErrorOr<iterator> emplace_or_assign(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-
-		iterator begin() { return iterator(m_buckets.end(), m_buckets.begin()); }
-		iterator end()   { return iterator(m_buckets.end(), m_buckets.end()); }
-		const_iterator begin() const { return const_iterator(m_buckets.end(), m_buckets.begin()); }
-		const_iterator end() const   { return const_iterator(m_buckets.end(), m_buckets.end()); }
-
-		ErrorOr<void> reserve(size_type);
-
-		void remove(const Key&);
-		void remove(iterator it);
-		void clear();
-
-		T& operator[](const Key&);
-		const T& operator[](const Key&) const;
-
-		iterator find(const Key& key);
-		const_iterator find(const Key& key) const;
-		bool contains(const Key&) const;
-
-		bool empty() const;
-		size_type size() const;
-
-	private:
-		ErrorOr<void> rebucket(size_type);
-		LinkedList<Entry>& get_bucket(const Key&);
-		const LinkedList<Entry>& get_bucket(const Key&) const;
-		Vector<LinkedList<Entry>>::iterator get_bucket_iterator(const Key&);
-		Vector<LinkedList<Entry>>::const_iterator get_bucket_iterator(const Key&) const;
-
-	private:
-		Vector<LinkedList<Entry>> m_buckets;
-		size_type m_size = 0;
-
-		friend iterator;
-	};
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>::HashMap(const HashMap<Key, T, HASH>& other)
 		{
-		*this = other;
+			if (auto it = m_hash_set.find(key); it != m_hash_set.end())
+			{
+				const_cast<T&>(it->value) = T(BAN::forward<Args>(args)...);
+				return iterator(it);
 			}
 
-	template<typename Key, typename T, typename HASH>
+			auto it = TRY(m_hash_set.insert(Entry { BAN::move(key), T(BAN::forward<Args>(args)...) }));
-	HashMap<Key, T, HASH>::HashMap(HashMap<Key, T, HASH>&& other)
+			return iterator(it);
-	{
-		*this = move(other);
 		}
 
-	template<typename Key, typename T, typename HASH>
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
-	HashMap<Key, T, HASH>::~HashMap()
+		void remove(const U& key)
 		{
-		clear();
+			if (auto it = find(key); it != end())
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>& HashMap<Key, T, HASH>::operator=(const HashMap<Key, T, HASH>& other)
-	{
-		clear();
-		m_buckets = other.m_buckets;
-		m_size = other.m_size;
-		return *this;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	HashMap<Key, T, HASH>& HashMap<Key, T, HASH>::operator=(HashMap<Key, T, HASH>&& other)
-	{
-		clear();
-		m_buckets = move(other.m_buckets);
-		m_size = other.m_size;
-		other.m_size = 0;
-		return *this;
-	}
-
-	template<typename Key, typename T, typename HASH>
-	template<typename... Args>
-	ErrorOr<typename HashMap<Key, T, HASH>::iterator> HashMap<Key, T, HASH>::emplace(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-	{
-		ASSERT(!contains(key));
-		TRY(rebucket(m_size + 1));
-
-		auto bucket_it = get_bucket_iterator(key);
-		TRY(bucket_it->emplace_back(move(key), forward<Args>(args)...));
-		m_size++;
-
-		return iterator(m_buckets.end(), bucket_it, prev(bucket_it->end(), 1));
-	}
-
-	template<typename Key, typename T, typename HASH>
-	template<typename... Args>
-	ErrorOr<typename HashMap<Key, T, HASH>::iterator> HashMap<Key, T, HASH>::emplace_or_assign(Key&& key, Args&&... args) requires is_constructible_v<T, Args...>
-	{
-		if (empty())
-			return emplace(move(key), forward<Args>(args)...);
-
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto entry_it = bucket_it->begin(); entry_it != bucket_it->end(); entry_it++)
-		{
-			if (entry_it->key != key)
-				continue;
-			entry_it->value = T(forward<Args>(args)...);
-			return iterator(m_buckets.end(), bucket_it, entry_it);
-		}
-
-		return emplace(move(key), forward<Args>(args)...);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	ErrorOr<void> HashMap<Key, T, HASH>::reserve(size_type size)
-	{
-		TRY(rebucket(size));
-		return {};
-	}
-
-	template<typename Key, typename T, typename HASH>
-	void HashMap<Key, T, HASH>::remove(const Key& key)
-	{
-		auto it = find(key);
-		if (it != end())
 				remove(it);
 		}
 
-	template<typename Key, typename T, typename HASH>
+		iterator remove(iterator it)
-	void HashMap<Key, T, HASH>::remove(iterator it)
 		{
-		it.outer_current()->remove(it.inner_current());
+			return iterator(m_hash_set.remove(it.m_iterator));
-		m_size--;
 		}
 
-	template<typename Key, typename T, typename HASH>
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
-	void HashMap<Key, T, HASH>::clear()
+		iterator find(const U& key)
 		{
-		m_buckets.clear();
+			return iterator(m_hash_set.find(key));
-		m_size = 0;
 		}
 
-	template<typename Key, typename T, typename HASH>
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
-	T& HashMap<Key, T, HASH>::operator[](const Key& key)
+		const_iterator find(const U& key) const
 		{
-		ASSERT(!empty());
+			return const_iterator(m_hash_set.find(key));
-		auto& bucket = get_bucket(key);
-		for (Entry& entry : bucket)
-			if (entry.key == key)
-				return entry.value;
-		ASSERT_NOT_REACHED();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		void clear()
-	const T& HashMap<Key, T, HASH>::operator[](const Key& key) const
 		{
-		ASSERT(!empty());
+			m_hash_set.clear();
-		const auto& bucket = get_bucket(key);
-		for (const Entry& entry : bucket)
-			if (entry.key == key)
-				return entry.value;
-		ASSERT_NOT_REACHED();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		ErrorOr<void> reserve(size_type size)
-	typename HashMap<Key, T, HASH>::iterator HashMap<Key, T, HASH>::find(const Key& key)
 		{
-		if (empty())
+			return m_hash_set.reserve(size);
-			return end();
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto it = bucket_it->begin(); it != bucket_it->end(); it++)
-			if (it->key == key)
-				return iterator(m_buckets.end(), bucket_it, it);
-		return end();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
-	typename HashMap<Key, T, HASH>::const_iterator HashMap<Key, T, HASH>::find(const Key& key) const
+		T& operator[](const U& key)
 		{
-		if (empty())
+			return find(key)->value;
-			return end();
-		auto bucket_it = get_bucket_iterator(key);
-		for (auto it = bucket_it->begin(); it != bucket_it->end(); it++)
-			if (it->key == key)
-				return const_iterator(m_buckets.end(), bucket_it, it);
-		return end();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
-	bool HashMap<Key, T, HASH>::contains(const Key& key) const
+		const T& operator[](const U& key) const
+		{
+			return find(key)->value;
+		}
+
+		template<detail::HashMapFindable<Key, HASH, COMP> U>
+		bool contains(const U& key) const
 		{
 			return find(key) != end();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		size_type capacity() const
-	bool HashMap<Key, T, HASH>::empty() const
 		{
-		return m_size == 0;
+			return m_hash_set.capacity();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		size_type size() const
-	typename HashMap<Key, T, HASH>::size_type HashMap<Key, T, HASH>::size() const
 		{
-		return m_size;
+			return m_hash_set.size();
 		}
 
-	template<typename Key, typename T, typename HASH>
+		bool empty() const
-	ErrorOr<void> HashMap<Key, T, HASH>::rebucket(size_type bucket_count)
 		{
-		if (m_buckets.size() >= bucket_count)
+			return m_hash_set.empty();
-			return {};
-
-		size_type new_bucket_count = BAN::Math::max<size_type>(bucket_count, m_buckets.size() * 2);
-		Vector<LinkedList<Entry>> new_buckets;
-		TRY(new_buckets.resize(new_bucket_count));
-
-		for (auto& bucket : m_buckets)
-		{
-			for (auto it = bucket.begin(); it != bucket.end();)
-			{
-				size_type new_bucket_index = HASH()(it->key) % new_buckets.size();
-				it = bucket.move_element_to_other_linked_list(new_buckets[new_bucket_index], new_buckets[new_bucket_index].end(), it);
-			}
 		}
 
-		m_buckets = move(new_buckets);
+	private:
-		return {};
+		HashSet<Entry, EntryHash, EntryComp> m_hash_set;
-	}
+	};
-
-	template<typename Key, typename T, typename HASH>
-	LinkedList<typename HashMap<Key, T, HASH>::Entry>& HashMap<Key, T, HASH>::get_bucket(const Key& key)
-	{
-		return *get_bucket_iterator(key);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	const LinkedList<typename HashMap<Key, T, HASH>::Entry>& HashMap<Key, T, HASH>::get_bucket(const Key& key) const
-	{
-		return *get_bucket_iterator(key);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	Vector<LinkedList<typename HashMap<Key, T, HASH>::Entry>>::iterator HashMap<Key, T, HASH>::get_bucket_iterator(const Key& key)
-	{
-		ASSERT(!m_buckets.empty());
-		auto index = HASH()(key) % m_buckets.size();
-		return next(m_buckets.begin(), index);
-	}
-
-	template<typename Key, typename T, typename HASH>
-	Vector<LinkedList<typename HashMap<Key, T, HASH>::Entry>>::const_iterator HashMap<Key, T, HASH>::get_bucket_iterator(const Key& key) const
-	{
-		ASSERT(!m_buckets.empty());
-		auto index = HASH()(key) % m_buckets.size();
-		return next(m_buckets.begin(), index);
-	}
 
 }

BAN/include/BAN/HashSet.h

@@ -2,198 +2,358 @@
 
 #include <BAN/Errors.h>
 #include <BAN/Hash.h>
-#include <BAN/Iterators.h>
-#include <BAN/LinkedList.h>
 #include <BAN/Math.h>
 #include <BAN/Move.h>
-#include <BAN/Vector.h>
+#include <BAN/New.h>
 
 namespace BAN
 {
 
-	template<typename T, typename HASH = hash<T>>
+	template<typename HashSet, typename Bucket, typename T>
+	class HashSetIterator
+	{
+	public:
+		HashSetIterator() = default;
+
+		const T& operator*() const
+		{
+			ASSERT(m_bucket);
+			return *m_bucket->element();
+		}
+
+		const T* operator->() const
+		{
+			ASSERT(m_bucket);
+			return m_bucket->element();
+		}
+
+		HashSetIterator& operator++()
+		{
+			ASSERT(m_bucket);
+			m_bucket++;
+			skip_to_valid_bucket();
+			return *this;
+		}
+		HashSetIterator operator++(int)
+		{
+			auto temp = *this;
+			++(*this);
+			return temp;
+		}
+
+		bool operator==(HashSetIterator other) const
+		{
+			return m_bucket == other.m_bucket;
+		}
+		bool operator!=(HashSetIterator other) const
+		{
+			return m_bucket != other.m_bucket;
+		}
+
+	private:
+		explicit HashSetIterator(Bucket* bucket)
+			: m_bucket(bucket)
+		{
+			if (m_bucket != nullptr)
+				skip_to_valid_bucket();
+		}
+
+		void skip_to_valid_bucket()
+		{
+			while (m_bucket->state != Bucket::USED && !m_bucket->end)
+				m_bucket++;
+			if (m_bucket->end)
+				m_bucket = nullptr;
+		}
+
+	private:
+		Bucket* m_bucket { nullptr };
+		friend HashSet;
+	};
+
+	namespace detail
+	{
+		template<typename T, typename U, typename HASH, typename COMP>
+		concept HashSetFindable = requires(const U& a, const T& b) { COMP()(a, b); HASH()(b); };
+	}
+
+	template<typename T, typename HASH = BAN::hash<T>, typename COMP = BAN::equal<T>>
 	class HashSet
 	{
+	private:
+		struct Bucket
+		{
+			static constexpr uint8_t UNUSED  = 0;
+			static constexpr uint8_t USED    = 1;
+			static constexpr uint8_t REMOVED = 2;
+
+			alignas(T) uint8_t storage[sizeof(T)];
+			hash_t hash;
+			uint8_t state       : 2;
+			uint8_t chain_start : 1;
+			uint8_t end         : 1;
+
+			T* element() { return reinterpret_cast<T*>(storage); }
+			const T* element() const { return reinterpret_cast<const T*>(storage); }
+		};
+
 	public:
 		using value_type     = T;
 		using size_type      = size_t;
-		using iterator = IteratorDouble<T, Vector, LinkedList, HashSet>;
+		using iterator       = HashSetIterator<HashSet,       Bucket,       T>;
-		using const_iterator = ConstIteratorDouble<T, Vector, LinkedList, HashSet>;
+		using const_iterator = HashSetIterator<HashSet, const Bucket, const T>;
 
 	public:
 		HashSet() = default;
-		HashSet(const HashSet&);
+		~HashSet() { clear(); }
-		HashSet(HashSet&&);
 
-		HashSet& operator=(const HashSet&);
+		HashSet(const HashSet& other) { *this = other; }
-		HashSet& operator=(HashSet&&);
+		HashSet& operator=(const HashSet& other)
-
-		ErrorOr<void> insert(const T&);
-		ErrorOr<void> insert(T&&);
-		void remove(const T&);
-		void clear();
-
-		ErrorOr<void> reserve(size_type);
-
-		iterator begin() { return iterator(m_buckets.end(), m_buckets.begin()); }
-		iterator end()   { return iterator(m_buckets.end(), m_buckets.end()); }
-		const_iterator begin() const { return const_iterator(m_buckets.end(), m_buckets.begin()); }
-		const_iterator end() const   { return const_iterator(m_buckets.end(), m_buckets.end()); }
-
-		bool contains(const T&) const;
-
-		size_type size() const;
-		bool empty() const;
-
-	private:
-		ErrorOr<void> rebucket(size_type);
-		LinkedList<T>& get_bucket(const T&);
-		const LinkedList<T>& get_bucket(const T&) const;
-
-	private:
-		Vector<LinkedList<T>> m_buckets;
-		size_type m_size = 0;
-	};
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>::HashSet(const HashSet& other)
-		: m_buckets(other.m_buckets)
-		, m_size(other.m_size)
-	{
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>::HashSet(HashSet&& other)
-		: m_buckets(move(other.m_buckets))
-		, m_size(other.m_size)
-	{
-		other.clear();
-	}
-
-	template<typename T, typename HASH>
-	HashSet<T, HASH>& HashSet<T, HASH>::operator=(const HashSet& other)
 		{
 			clear();
+
+			MUST(reserve(other.size()));
+			for (auto& bucket : other)
+				MUST(insert(bucket));
+
+			return *this;
+		}
+
+		HashSet(HashSet&& other) { *this = BAN::move(other); }
+		HashSet& operator=(HashSet&& other)
+		{
+			clear();
+
 			m_buckets  = other.m_buckets;
+			m_capacity = other.m_capacity;
 			m_size     = other.m_size;
+			m_removed  = other.m_removed;
+
+			other.m_buckets  = nullptr;
+			other.m_capacity = 0;
+			other.m_size     = 0;
+			other.m_removed  = 0;
+
 			return *this;
 		}
 
-	template<typename T, typename HASH>
+		iterator begin()             { return iterator(m_buckets); }
-	HashSet<T, HASH>& HashSet<T, HASH>::operator=(HashSet&& other)
+		iterator end()               { return iterator(nullptr); }
+		const_iterator begin() const { return const_iterator(m_buckets); }
+		const_iterator end() const   { return const_iterator(nullptr); }
+
+		ErrorOr<iterator> insert(const T& value)
 		{
-		clear();
+			return insert(T(value));
-		m_buckets = move(other.m_buckets);
-		m_size = other.m_size;
-		other.clear();
-		return *this;
 		}
 
-	template<typename T, typename HASH>
+		ErrorOr<iterator> insert(T&& value)
-	ErrorOr<void> HashSet<T, HASH>::insert(const T& key)
 		{
-		return insert(move(T(key)));
+			if (should_rehash_with_size(m_size + 1))
+				TRY(rehash(m_size * 2));
+			return insert_impl(BAN::move(value), HASH()(value));
 		}
 
-	template<typename T, typename HASH>
+		template<detail::HashSetFindable<T, HASH, COMP> U>
-	ErrorOr<void> HashSet<T, HASH>::insert(T&& key)
+		void remove(const U& value)
 		{
-		if (!empty() && get_bucket(key).contains(key))
+			if (auto it = find(value); it != end())
-			return {};
+				remove(it);
-
-		TRY(rebucket(m_size + 1));
-		TRY(get_bucket(key).push_back(move(key)));
-		m_size++;
-		return {};
 		}
 
-	template<typename T, typename HASH>
+		iterator remove(iterator it)
-	void HashSet<T, HASH>::remove(const T& key)
 		{
-		if (empty()) return;
+			auto& bucket = *it.m_bucket;
-		auto& bucket = get_bucket(key);
+			bucket.element()->~T();
-		for (auto it = bucket.begin(); it != bucket.end(); it++)
+			bucket.state = Bucket::REMOVED;
-		{
-			if (*it == key)
-			{
-				bucket.remove(it);
 			m_size--;
-				break;
+			m_removed++;
-			}
+			return iterator(&bucket);
-		}
 		}
 
-	template<typename T, typename HASH>
+		template<detail::HashSetFindable<T, HASH, COMP> U>
-	void HashSet<T, HASH>::clear()
+		iterator find(const U& value)
 		{
-		m_buckets.clear();
+			return iterator(const_cast<Bucket*>(find_impl(value).m_bucket));
+		}
+
+		template<detail::HashSetFindable<T, HASH, COMP> U>
+		const_iterator find(const U& value) const
+		{
+			return find_impl(value);
+		}
+
+		void clear()
+		{
+			if (m_buckets == nullptr)
+				return;
+
+			for (size_type i = 0; i < m_capacity; i++)
+				if (m_buckets[i].state == Bucket::USED)
+					m_buckets[i].element()->~T();
+
+			BAN::deallocator(m_buckets);
+			m_buckets  = nullptr;
+			m_capacity = 0;
 			m_size     = 0;
+			m_removed  = 0;
 		}
 
-	template<typename T, typename HASH>
+		ErrorOr<void> reserve(size_type size)
-	ErrorOr<void> HashSet<T, HASH>::reserve(size_type size)
 		{
-		TRY(rebucket(size));
+			if (should_rehash_with_size(size))
+				TRY(rehash(size * 2));
 			return {};
 		}
 
-	template<typename T, typename HASH>
+		template<detail::HashSetFindable<T, HASH, COMP> U>
-	bool HashSet<T, HASH>::contains(const T& key) const
+		bool contains(const U& value) const
 		{
-		if (empty()) return false;
+			return find(value) != end();
-		return get_bucket(key).contains(key);
 		}
 
-	template<typename T, typename HASH>
+		size_type capacity() const
-	typename HashSet<T, HASH>::size_type HashSet<T, HASH>::size() const
+		{
+			return m_capacity;
+		}
+
+		size_type size() const
 		{
 			return m_size;
 		}
 
-	template<typename T, typename HASH>
+		bool empty() const
-	bool HashSet<T, HASH>::empty() const
 		{
 			return m_size == 0;
 		}
 
-	template<typename T, typename HASH>
+	private:
-	ErrorOr<void> HashSet<T, HASH>::rebucket(size_type bucket_count)
+		ErrorOr<void> rehash(size_type new_capacity)
 		{
-		if (m_buckets.size() >= bucket_count)
+			new_capacity = BAN::Math::max<size_t>(16, BAN::Math::max(new_capacity, m_size + 1));
-			return {};
+			new_capacity = BAN::Math::round_up_to_power_of_two(new_capacity);
 
-		size_type new_bucket_count = Math::max<size_type>(bucket_count, m_buckets.size() * 2);
+			void* new_buckets = BAN::allocator((new_capacity + 1) * sizeof(Bucket));
-		Vector<LinkedList<T>> new_buckets;
+			if (new_buckets == nullptr)
-		if (new_buckets.resize(new_bucket_count).is_error())
+				return BAN::Error::from_errno(ENOMEM);
-			return Error::from_errno(ENOMEM);
+			memset(new_buckets, 0, (new_capacity + 1) * sizeof(Bucket));
 
-		for (auto& bucket : m_buckets)
+			Bucket* old_buckets = m_buckets;
+			const size_type old_capacity = m_capacity;
+
+			m_buckets  = static_cast<Bucket*>(new_buckets);
+			m_capacity = new_capacity;
+			m_size     = 0;
+			m_removed  = 0;
+
+			for (size_type i = 0; i < old_capacity; i++)
 			{
-			for (auto it = bucket.begin(); it != bucket.end();)
+				auto& old_bucket = old_buckets[i];
-			{
+				if (old_bucket.state != Bucket::USED)
-				size_type new_bucket_index = HASH()(*it) % new_buckets.size();
+					continue;
-				it = bucket.move_element_to_other_linked_list(new_buckets[new_bucket_index], new_buckets[new_bucket_index].end(), it);
+				insert_impl(BAN::move(*old_bucket.element()), old_bucket.hash);
-			}
+				old_bucket.element()->~T();
 			}
 
-		m_buckets = move(new_buckets);
+			m_buckets[m_capacity].end = true;
+
+			BAN::deallocator(old_buckets);
+
 			return {};
 		}
 
-	template<typename T, typename HASH>
+		template<detail::HashSetFindable<T, HASH, COMP> U>
-	LinkedList<T>& HashSet<T, HASH>::get_bucket(const T& key)
+		const_iterator find_impl(const U& value) const
 		{
-		ASSERT(!m_buckets.empty());
+			if (m_capacity == 0)
-		size_type index = HASH()(key) % m_buckets.size();
+				return end();
-		return m_buckets[index];
+
+			bool first = true;
+			const hash_t orig_hash = HASH()(value);
+			for (auto hash = orig_hash;; hash = get_next_hash_in_chain(hash, orig_hash), first = false)
+			{
+				auto& bucket = m_buckets[hash & (m_capacity - 1)];
+				if (bucket.state == Bucket::USED && bucket.hash == orig_hash && COMP()(*bucket.element(), value))
+					return const_iterator(&bucket);
+				if (bucket.state == Bucket::UNUSED)
+					return end();
+				if (!first && bucket.chain_start)
+					return end();
+			}
 		}
 
-	template<typename T, typename HASH>
+		iterator insert_impl(T&& value, hash_t orig_hash)
-	const LinkedList<T>& HashSet<T, HASH>::get_bucket(const T& key) const
 		{
-		ASSERT(!m_buckets.empty());
+			ASSERT(!should_rehash_with_size(m_size + 1));
-		size_type index = HASH()(key) % m_buckets.size();
+
-		return m_buckets[index];
+			Bucket* target = nullptr;
+
+			bool first = true;
+			for (auto hash = orig_hash;; hash = get_next_hash_in_chain(hash, orig_hash), first = false)
+			{
+				auto& bucket = m_buckets[hash & (m_capacity - 1)];
+
+				if (!first)
+					bucket.chain_start = false;
+
+				if (bucket.state == Bucket::USED)
+				{
+					if (bucket.hash != orig_hash || !COMP()(*bucket.element(), value))
+						continue;
+					target = &bucket;
+					break;
 				}
 
+				if (target == nullptr)
+					target = &bucket;
+
+				if (bucket.state == Bucket::UNUSED)
+					break;
+			}
+
+			switch (target->state)
+			{
+				case Bucket::USED:
+					target->element()->~T();
+					break;
+				case Bucket::REMOVED:
+					m_removed--;
+					[[fallthrough]];
+				case Bucket::UNUSED:
+					m_size++;
+					break;
+			}
+
+			target->chain_start = first && target->state == Bucket::UNUSED;
+			target->hash = orig_hash;
+			target->state = Bucket::USED;
+
+			new (target->element()) T(BAN::move(value));
+
+			return iterator(target);
+		}
+
+		bool should_rehash_with_size(size_type size) const
+		{
+			if (m_capacity < 16)
+				return true;
+			if (size + m_removed > m_capacity / 4 * 3)
+				return true;
+			return false;
+		}
+
+		hash_t get_next_hash_in_chain(hash_t prev_hash, hash_t orig_hash) const
+		{
+			// TODO: does this even provide better performance than `return prev_hash + 1`
+			//       when using "good" hash functions
+			return prev_hash * 1103515245 + (orig_hash | 1);
+		}
+
+	private:
+		Bucket* m_buckets { nullptr };
+		size_type m_capacity { 0 };
+		size_type m_size     { 0 };
+		size_type m_removed  { 0 };
+	};
+
 }

BAN/include/BAN/Math.h

@@ -65,6 +65,22 @@ namespace BAN::Math
 		return (x & (x - 1)) == 0;
 	}
 
+	template<integral T> requires(sizeof(T) <= 8)
+	inline constexpr T round_up_to_power_of_two(T x)
+	{
+		x--;
+		x |= x >> 1;
+		x |= x >> 2;
+		x |= x >> 4;
+		if constexpr(sizeof(T) >= 2)
+			x |= x >> 8;
+		if constexpr(sizeof(T) >= 4)
+			x |= x >> 16;
+		if constexpr(sizeof(T) >= 8)
+			x |= x >> 32;
+		return x + 1;
+	}
+
 	template<integral T>
 	__attribute__((always_inline))
 	inline constexpr bool will_multiplication_overflow(T a, T b)

BAN/include/BAN/RefPtr.h

@@ -2,9 +2,9 @@
 
 #include <BAN/Atomic.h>
 #include <BAN/Errors.h>
+#include <BAN/Hash.h>
 #include <BAN/Move.h>
 #include <BAN/NoCopyMove.h>
-#include <stdint.h>
 
 namespace BAN
 {
@@ -129,14 +129,9 @@ namespace BAN
 			return *this;
 		}
 
-		T* ptr() { ASSERT(!empty()); return m_pointer; }
+		T* ptr() const { return m_pointer; }
-		const T* ptr() const { ASSERT(!empty()); return m_pointer; }
+		T& operator*() const { ASSERT(!empty()); return *ptr(); }
-
+		T* operator->() const { ASSERT(!empty()); return ptr(); }
-		T& operator*() { return *ptr(); }
-		const T& operator*() const { return *ptr(); }
-
-		T* operator->() { return ptr(); }
-		const T* operator->() const { return ptr(); }
 
 		bool operator==(RefPtr other) const { return m_pointer == other.m_pointer; }
 		bool operator!=(RefPtr other) const { return m_pointer != other.m_pointer; }
@@ -158,4 +153,13 @@ namespace BAN
 		friend class RefPtr;
 	};
 
+	template<typename T>
+	struct hash<RefPtr<T>>
+	{
+		constexpr hash_t operator()(const RefPtr<T>& ptr) const
+		{
+			return hash<T*>()(ptr.ptr());
+		}
+	};
+
 }

BAN/include/BAN/Sort.h

@@ -21,26 +21,47 @@ namespace BAN::sort
 	namespace detail
 	{
 
+		template<typename It>
+		struct partition_pair
+		{
+			It lt;
+			It gt;
+		};
+
 		template<typename It, typename Comp>
-		It partition(It begin, It end, Comp comp)
+		partition_pair<It> partition(It begin, It end, Comp comp)
 		{
-			It pivot = prev(end, 1);
+			It pivot = next(begin, distance(begin, end) / 2);
 
-			It it1 = begin;
+			It lt = begin;
-			for (It it2 = begin; it2 != pivot; ++it2)
+			It eq = begin;
+			It gt = end;
+
+			while (eq != gt)
 			{
-				if (comp(*it2, *pivot))
+				if (comp(*eq, *pivot))
 				{
-					swap(*it1, *it2);
+					swap(*eq, *lt);
-					++it1;
+					if (pivot == lt)
+						pivot = eq;
+					++lt;
+					++eq;
+				}
+				else if (comp(*pivot, *eq))
+				{
+					--gt;
+					swap(*eq, *gt);
+					if (pivot == gt)
+						pivot = eq;
+				}
+				else
+				{
+					++eq;
 				}
 			}
 
-			swap(*it1, *pivot);
+			return { lt, gt };
-
-			return it1;
 		}
-
 	}
 
 	template<typename It, typename Comp = less<it_value_type_t<It>>>
@@ -48,9 +69,9 @@ namespace BAN::sort
 	{
 		if (distance(begin, end) <= 1)
 			return;
-		It mid = detail::partition(begin, end, comp);
+		const auto [lt, gt] = detail::partition(begin, end, comp);
-		quick_sort(begin, mid, comp);
+		quick_sort(begin, lt, comp);
-		quick_sort(++mid, end, comp);
+		quick_sort(gt, end, comp);
 	}
 
 	template<typename It, typename Comp = less<it_value_type_t<It>>>
@@ -85,9 +106,9 @@ namespace BAN::sort
 				return insertion_sort(begin, end, comp);
 			if (max_depth == 0)
 				return heap_sort(begin, end, comp);
-			It mid = detail::partition(begin, end, comp);
+			const auto [lt, gt] = detail::partition(begin, end, comp);
-			intro_sort_impl(begin, mid, max_depth - 1, comp);
+			intro_sort_impl(begin, lt, max_depth - 1, comp);
-			intro_sort_impl(++mid, end, max_depth - 1, comp);
+			intro_sort_impl(gt, end, max_depth - 1, comp);
 		}
 
 	}
@@ -116,27 +137,20 @@ namespace BAN::sort
 
 	template<typename It, size_t radix = 256>
 	requires is_unsigned_v<it_value_type_t<It>> && (radix > 0 && (radix & (radix - 1)) == 0)
-	BAN::ErrorOr<void> radix_sort(It begin, It end)
+	void radix_sort(It begin, It end, BAN::Span<it_value_type_t<It>> storage)
 	{
-		using value_type = it_value_type_t<It>;
-
 		const size_t len = distance(begin, end);
 		if (len <= 1)
-			return {};
+			return;
 
-		Vector<value_type> temp;
+		ASSERT(storage.size() >= len);
-		TRY(temp.resize(len));
-
-		Vector<size_t> counts;
-		TRY(counts.resize(radix));
 
 		constexpr size_t mask  = radix - 1;
 		constexpr size_t shift = detail::lsb_index(radix);
 
-		for (size_t s = 0; s < sizeof(value_type) * 8; s += shift)
+		for (size_t s = 0; s < sizeof(it_value_type_t<It>) * 8; s += shift)
 		{
-			for (auto& cnt : counts)
+			size_t counts[radix] {};
-				cnt = 0;
 			for (It it = begin; it != end; ++it)
 				counts[(*it >> s) & mask]++;
 
@@ -146,12 +160,27 @@ namespace BAN::sort
 			for (It it = end; it != begin;)
 			{
 				--it;
-				temp[--counts[(*it >> s) & mask]] = *it;
+				storage[--counts[(*it >> s) & mask]] = *it;
 			}
 
-			for (size_t j = 0; j < temp.size(); j++)
+			It it = begin;
-				*next(begin, j) = temp[j];
+			for (size_t j = 0; j < storage.size(); j++, ++it)
+				*it = storage[j];
 		}
+	}
+
+	template<typename It, size_t radix = 256>
+	requires is_unsigned_v<it_value_type_t<It>> && (radix > 0 && (radix & (radix - 1)) == 0)
+	BAN::ErrorOr<void> radix_sort(It begin, It end)
+	{
+		const size_t len = distance(begin, end);
+		if (len <= 1)
+			return {};
+
+		Vector<it_value_type_t<It>> temp;
+		TRY(temp.resize(len));
+
+		radix_sort(begin, end, temp.span());
 
 		return {};
 	}

BAN/include/BAN/UniqPtr.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <BAN/Errors.h>
+#include <BAN/Hash.h>
 #include <BAN/NoCopyMove.h>
 
 namespace BAN
@@ -53,32 +54,12 @@ namespace BAN
 			return *this;
 		}
 
-		T& operator*()
+		T* ptr() const { return m_pointer; }
-		{
+		T& operator*() const { ASSERT(!empty()); return *ptr(); }
-			ASSERT(m_pointer);
+		T* operator->() const { ASSERT(!empty()); return ptr(); }
-			return *m_pointer;
-		}
 
-		const T& operator*() const
+		bool empty() const { return m_pointer == nullptr; }
-		{
+		explicit operator bool() const { return m_pointer; }
-			ASSERT(m_pointer);
-			return *m_pointer;
-		}
-
-		T* operator->()
-		{
-			ASSERT(m_pointer);
-			return m_pointer;
-		}
-
-		const T* operator->() const
-		{
-			ASSERT(m_pointer);
-			return m_pointer;
-		}
-
-		T* ptr() { return m_pointer; }
-		const T* ptr() const { return m_pointer; }
 
 		void clear()
 		{
@@ -87,8 +68,6 @@ namespace BAN
 			m_pointer = nullptr;
 		}
 
-		operator bool() const { return m_pointer != nullptr; }
-
 	private:
 		T* m_pointer = nullptr;
 
@@ -96,4 +75,13 @@ namespace BAN
 		friend class UniqPtr;
 	};
 
+	template<typename T>
+	struct hash<UniqPtr<T>>
+	{
+		constexpr hash_t operator()(const UniqPtr<T>& ptr) const
+		{
+			return hash<T*>()(ptr.ptr());
+		}
+	};
+
 }

kernel/CMakeLists.txt

@@ -11,6 +11,7 @@ set(KERNEL_SOURCES
 	kernel/Audio/Controller.cpp
 	kernel/Audio/HDAudio/AudioFunctionGroup.cpp
 	kernel/Audio/HDAudio/Controller.cpp
+	kernel/Banos.cpp
 	kernel/BootInfo.cpp
 	kernel/CPUID.cpp
 	kernel/Credentials.cpp
@@ -121,6 +122,7 @@ set(KERNEL_SOURCES
 	kernel/USB/USBManager.cpp
 	kernel/USB/XHCI/Controller.cpp
 	kernel/USB/XHCI/Device.cpp
+	kernel/UserCopy.cpp
 	icxxabi.cpp
 )
 

kernel/arch/i686/PageTable.cpp

@@ -1,7 +1,7 @@
 #include <kernel/BootInfo.h>
 #include <kernel/CPUID.h>
 #include <kernel/Lock/SpinLock.h>
-#include <kernel/Memory/kmalloc.h>
+#include <kernel/Memory/Heap.h>
 #include <kernel/Memory/PageTable.h>
 
 extern uint8_t g_kernel_start[];
@@ -16,6 +16,8 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
@@ -24,8 +26,6 @@ namespace Kernel
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
 
-	static bool s_is_post_heap_done = false;
-
 	static PageTable* s_kernel = nullptr;
 	static bool s_has_nxe = false;
 	static bool s_has_pge = false;
@@ -33,6 +33,46 @@ namespace Kernel
 
 	static paddr_t s_global_pdpte = 0;
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
+	alignas(PAGE_SIZE) static uint64_t s_fast_page_pt_storage[512] {};
+
+	static paddr_t allocate_zeroed_page_aligned_page()
+	{
+		const paddr_t paddr = Heap::get().take_free_page();
+		ASSERT(paddr);
+
+		PageTable::with_fast_page(paddr, [] {
+			memset(PageTable::fast_page_as_ptr(), 0, PAGE_SIZE);
+		});
+
+		return paddr;
+	}
+
+	static void unallocate_page(paddr_t paddr)
+	{
+		Heap::get().release_page(paddr);
+	}
+
+	static uint64_t read_entry_from_table(paddr_t paddr, uint16_t entry)
+	{
+		uint64_t result;
+		PageTable::with_fast_page(paddr & s_page_addr_mask, [&result, entry] {
+			result = PageTable::fast_page_as_sized<uint64_t>(entry);
+		});
+		return result;
+	}
+
+	static uint64_t write_entry_to_table(paddr_t paddr, uint16_t entry, uint64_t value)
+	{
+		uint64_t old_value;
+		PageTable::with_fast_page(paddr & s_page_addr_mask, [&old_value, entry, value] {
+			old_value = PageTable::fast_page_as_sized<uint64_t>(entry);
+			PageTable::fast_page_as_sized<uint64_t>(entry) = value;
+		});
+		return old_value;
+	}
+
 	static inline PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
@@ -51,31 +91,22 @@ namespace Kernel
 		return result;
 	}
 
-	void PageTable::initialize_pre_heap()
+	void PageTable::initialize_fast_page()
+	{
+		s_fast_page_pt = g_boot_fast_page_pt;
+	}
+
+	static void detect_cpu_features()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_pat())
 			s_has_pat = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->initialize_kernel();
-		s_kernel->initial_load();
 	}
 
-	void PageTable::initialize_post_heap()
+	void PageTable::enable_cpu_features()
-	{
-		s_is_post_heap_done = true;
-	}
-
-	void PageTable::initial_load()
 	{
 		if (s_has_nxe)
 		{
@@ -116,8 +147,49 @@ namespace Kernel
 			"movl %%eax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel);
+
+		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		s_kernel->map_kernel_memory();
+
+		s_global_pdpte = read_entry_from_table(s_kernel->m_highest_paging_struct, 3);
+
+		// update fast page pt
+		{
+			constexpr vaddr_t vaddr = fast_page();
+			constexpr uint16_t pdpte = (vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					const uint64_t value = read_entry_from_table(table_paddr, entry);
+					if (value & Flags::Present)
+						return value & s_page_addr_mask;
+
+					const paddr_t paddr = allocate_zeroed_page_aligned_page();
+					write_entry_to_table(table_paddr, entry, paddr | flags);
+					return paddr;
+				};
+
+			const paddr_t pdpt = s_kernel->m_highest_paging_struct;
+			const paddr_t pd = get_or_allocate_entry(pdpt, pdpte, Flags::Present);
+
+			const paddr_t entry_paddr = reinterpret_cast<uintptr_t>(&s_fast_page_pt_storage) - KERNEL_OFFSET + g_boot_info.kernel_paddr;
+			write_entry_to_table(pd, pde, entry_paddr | PageTable::Flags::ReadWrite | PageTable::Flags::Present);
+			s_fast_page_pt = s_fast_page_pt_storage;
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -131,127 +203,89 @@ namespace Kernel
 		return true;
 	}
 
-	static uint64_t* allocate_zeroed_page_aligned_page()
+	void PageTable::map_kernel_memory()
 	{
-		void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-		ASSERT(page);
-		memset(page, 0, PAGE_SIZE);
-		return (uint64_t*)page;
-	}
-
-	template<typename T>
-	static paddr_t V2P(const T vaddr)
-	{
-		return (vaddr_t)vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-	}
-
-	template<typename T>
-	static uint64_t* P2V(const T paddr)
-	{
-		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-	}
-
-	void PageTable::initialize_kernel()
-	{
-		ASSERT(s_global_pdpte == 0);
-		s_global_pdpte = V2P(allocate_zeroed_page_aligned_page());
-
-		map_kernel_memory();
-
-		prepare_fast_page();
-
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
-		ASSERT((vaddr_t)g_kernel_start % PAGE_SIZE == 0);
+		const vaddr_t kernel_start = reinterpret_cast<vaddr_t>(g_kernel_start);
 		map_range_at(
-			V2P(g_kernel_start),
+			kernel_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
-			(vaddr_t)g_kernel_start,
+			kernel_start,
 			g_kernel_end - g_kernel_start,
 			Flags::Present
 		);
 
 		// Map executable kernel memory as executable
+		const vaddr_t kernel_execute_start = reinterpret_cast<vaddr_t>(g_kernel_execute_start);
 		map_range_at(
-			V2P(g_kernel_execute_start),
+			kernel_execute_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
-			(vaddr_t)g_kernel_execute_start,
+			kernel_execute_start,
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
 		);
 
 		// Map writable kernel memory as writable
+		const vaddr_t kernel_writable_start = reinterpret_cast<vaddr_t>(g_kernel_writable_start);
 		map_range_at(
-			V2P(g_kernel_writable_start),
+			kernel_writable_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
-			(vaddr_t)g_kernel_writable_start,
+			kernel_writable_start,
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
 		);
 
 		// Map userspace memory
+		const vaddr_t kernel_userspace_start = reinterpret_cast<vaddr_t>(g_userspace_start);
 		map_range_at(
-			V2P(g_userspace_start),
+			kernel_userspace_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
-			(vaddr_t)g_userspace_start,
+			kernel_userspace_start,
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
-
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
-		ASSERT(pdpt[pdpte] & Flags::Present);
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		ASSERT(pt[pte] == 0);
-		pt[pte] = Flags::Reserved;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
+		map_fast_page(0, paddr);
-		ASSERT(paddr);
-		ASSERT(paddr % PAGE_SIZE == 0);
-
-		ASSERT(s_fast_page_lock.current_processor_has_lock());
-
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
-
-		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
+		unmap_fast_page(0);
+	}
 
+	void* PageTable::map_fast_page(size_t index, paddr_t paddr)
+	{
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
+
+		ASSERT(index < 512);
+		ASSERT(s_fast_page_pt);
+
+		if (index < reserved_fast_pages)
 			ASSERT(s_fast_page_lock.current_processor_has_lock());
+		else
+			ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
 
-		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
+		ASSERT(!(s_fast_page_pt[index] & Flags::Present));
-		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
+		s_fast_page_pt[index] = paddr | Flags::ReadWrite | Flags::Present;
-		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
-		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
+		void* address = reinterpret_cast<void*>(fast_page() + index * PAGE_SIZE);
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
+		asm volatile("invlpg (%0)" :: "r"(address));
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
+		return address;
+	}
 
-		ASSERT(pt[pte] & Flags::Present);
+	void PageTable::unmap_fast_page(size_t index)
-		pt[pte] = Flags::Reserved;
+	{
+		ASSERT(index < 512);
+		ASSERT(s_fast_page_pt);
 
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
+		if (index < reserved_fast_pages)
+			ASSERT(s_fast_page_lock.current_processor_has_lock());
+		else
+			ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
+
+		ASSERT((s_fast_page_pt[index] & Flags::Present));
+		s_fast_page_pt[index] = 0;
+
+		asm volatile("invlpg (%0)" :: "r"(fast_page() + index * PAGE_SIZE));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -260,25 +294,19 @@ namespace Kernel
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
-		return page_table;
-	}
 
-	void PageTable::map_kernel_memory()
+		page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pdpte);
 
-		ASSERT(m_highest_paging_struct == 0);
+		PageTable::with_fast_page(page_table->m_highest_paging_struct, [] {
-		m_highest_paging_struct = V2P(kmalloc(32, 32, true));
+			uint64_t* pdpt = &PageTable::fast_page_as<uint64_t>();
-		ASSERT(m_highest_paging_struct);
-
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
 			pdpt[0] = 0;
 			pdpt[1] = 0;
 			pdpt[2] = 0;
 			pdpt[3] = s_global_pdpte | Flags::Present;
 			static_assert(KERNEL_OFFSET == 0xC0000000);
+		});
+
+		return page_table;
 	}
 
 	PageTable::~PageTable()
@@ -286,21 +314,22 @@ namespace Kernel
 		if (m_highest_paging_struct == 0)
 			return;
 
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t pdpt = m_highest_paging_struct;
 		for (uint32_t pdpte = 0; pdpte < 3; pdpte++)
 		{
-			if (!(pdpt[pdpte] & Flags::Present))
+			const uint64_t pd = read_entry_from_table(pdpt, pdpte);
+			if (!(pd & Flags::Present))
 				continue;
-			uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 			for (uint32_t pde = 0; pde < 512; pde++)
 			{
-				if (!(pd[pde] & Flags::Present))
+				const uint64_t pt = read_entry_from_table(pd, pde);
+				if (!(pt & Flags::Present))
 					continue;
-				kfree(P2V(pd[pde] & s_page_addr_mask));
+				unallocate_page(pt & s_page_addr_mask);
 			}
-			kfree(pd);
+			unallocate_page(pd & s_page_addr_mask);
 		}
-		kfree(pdpt);
+		unallocate_page(m_highest_paging_struct);
 	}
 
 	void PageTable::load()
@@ -318,32 +347,10 @@ namespace Kernel
 		const bool is_userspace = (vaddr < KERNEL_OFFSET);
 		if (is_userspace && this != &PageTable::current())
 			;
-		else if (pages <= 32 || !s_is_post_heap_done)
+		else if (pages >= full_tlb_flush_threshold)
-		{
+			invalidate_full_address_space(!is_userspace);
-			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
+		else for (size_t i = 0; i < pages; i++)
-				asm volatile("invlpg (%0)" :: "r"(vaddr));
+			asm volatile("invlpg (%0)" :: "r"(vaddr + i * PAGE_SIZE));
-		}
-		else if (is_userspace || !s_has_pge)
-		{
-			asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
-		}
-		else
-		{
-			asm volatile(
-				"movl %%cr4, %%eax;"
-
-				"andl $~0x80, %%eax;"
-				"movl %%eax, %%cr4;"
-
-				"movl %0, %%cr3;"
-
-				"orl $0x80, %%eax;"
-				"movl %%eax, %%cr4;"
-				:
-				: "r"(static_cast<uint32_t>(m_highest_paging_struct))
-				: "eax"
-			);
-		}
 
 		if (send_smp_message)
 		{
@@ -358,6 +365,34 @@ namespace Kernel
 		}
 	}
 
+	void PageTable::invalidate_full_address_space(bool global)
+	{
+		if (!global || !s_has_pge)
+		{
+			asm volatile(
+				"movl %%cr3, %%eax;"
+				"movl %%eax, %%cr3;"
+				::: "eax"
+			);
+		}
+		else
+		{
+			asm volatile(
+				"movl %%cr4, %%eax;"
+
+				"andl $~0x80, %%eax;"
+				"movl %%eax, %%cr4;"
+
+				"movl %%cr3, %%ecx;"
+				"movl %%ecx, %%cr3;"
+
+				"orl $0x80, %%eax;"
+				"movl %%eax, %%cr4;"
+				::: "eax", "ecx"
+			);
+		}
+	}
+
 	void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
 	{
 		ASSERT(vaddr);
@@ -377,15 +412,13 @@ namespace Kernel
 		if (is_page_free(vaddr))
 			Kernel::panic("trying to unmap unmapped page 0x{H}", vaddr);
 
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t pdpt = m_highest_paging_struct;
-		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
+		const uint64_t pd = read_entry_from_table(pdpt, pdpte);
-		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
+		const uint64_t pt = read_entry_from_table(pd, pde);
 
-		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
+		const uint64_t old_entry = write_entry_to_table(pt, pte, 0);
 
-		pt[pte] = 0;
+		if (invalidate && (old_entry & s_page_addr_mask))
-
-		if (invalidate && old_paddr != 0)
 			invalidate_page(vaddr, true);
 	}
 
@@ -436,28 +469,31 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t pdpt = m_highest_paging_struct;
-		if (!(pdpt[pdpte] & Flags::Present))
-			pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present;
 
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+		uint64_t pd = read_entry_from_table(pdpt, pdpte);
-		if ((pd[pde] & uwr_flags) != uwr_flags)
+		if (!(pd & Flags::Present))
 		{
-			if (!(pd[pde] & Flags::Present))
+			pd = allocate_zeroed_page_aligned_page();
-				pd[pde] = V2P(allocate_zeroed_page_aligned_page());
+			pd |= Flags::Present;
-			pd[pde] |= uwr_flags;
+			write_entry_to_table(pdpt, pdpte, pd);
+		}
+
+		uint64_t pt = read_entry_from_table(pd, pde);
+		if ((pt & uwr_flags) != uwr_flags)
+		{
+			if (!(pt & Flags::Present))
+				pt = allocate_zeroed_page_aligned_page();
+			pt |= uwr_flags;
+			write_entry_to_table(pd, pde, pt);
 		}
 
 		if (!(flags & Flags::Present))
 			uwr_flags &= ~Flags::Present;
 
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+		const uint64_t old_entry = write_entry_to_table(pt, pte, paddr | uwr_flags | extra_flags);
 
-		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
+		if (invalidate && (old_entry & s_page_addr_mask))
-
-		pt[pte] = paddr | uwr_flags | extra_flags;
-
-		if (invalidate && old_paddr != 0)
 			invalidate_page(vaddr, true);
 	}
 
@@ -488,31 +524,36 @@ namespace Kernel
 		const uint32_t e_pde   = ((vaddr + size - 1) >> 21) & 0x1FF;
 		const uint32_t e_pte   = ((vaddr + size - 1) >> 12) & 0x1FF;
 
-		SpinLockGuard _(m_lock);
+		SpinLockGuard _0(m_lock);
 
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		SpinLockGuard _1(s_fast_page_lock);
+
+		const uint64_t* pdpt = static_cast<uint64_t*>(map_fast_page(0, m_highest_paging_struct));
 		for (; pdpte <= e_pdpte; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			const uint64_t* pd = static_cast<uint64_t*>(map_fast_page(1, pdpt[pdpte] & s_page_addr_mask));
 			for (; pde < 512; pde++)
 			{
 				if (pdpte == e_pdpte && pde > e_pde)
 					break;
 				if (!(pd[pde] & Flags::ReadWrite))
 					continue;
-				uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				uint64_t* pt = static_cast<uint64_t*>(map_fast_page(2, pd[pde] & s_page_addr_mask));
 				for (; pte < 512; pte++)
 				{
 					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 						break;
 					pt[pte] &= ~static_cast<uint64_t>(Flags::ReadWrite);
 				}
+				unmap_fast_page(2);
 				pte = 0;
 			}
+			unmap_fast_page(1);
 			pde = 0;
 		}
+		unmap_fast_page(0);
 
 		invalidate_range(vaddr, size / PAGE_SIZE, true);
 	}
@@ -527,19 +568,17 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t pdpt = m_highest_paging_struct;
-		if (!(pdpt[pdpte] & Flags::Present))
+
+		const uint64_t pd = read_entry_from_table(pdpt, pdpte);
+		if (!(pd & Flags::Present))
 			return 0;
 
-		const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+		const uint64_t pt = read_entry_from_table(pd, pde);
-		if (!(pd[pde] & Flags::Present))
+		if (!(pt & Flags::Present))
 			return 0;
 
-		const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+		return read_entry_from_table(pt, pte);
-		if (!(pt[pte] & Flags::Used))
-			return 0;
-
-		return pt[pte];
 	}
 
 	PageTable::flags_t PageTable::get_page_flags(vaddr_t vaddr) const
@@ -570,30 +609,24 @@ namespace Kernel
 		return true;
 	}
 
-	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool send_smp_message)
+	void PageTable::reserve_page(vaddr_t vaddr)
 	{
-		SpinLockGuard _(m_lock);
 		ASSERT(vaddr % PAGE_SIZE == 0);
-		if (only_free && !is_page_free(vaddr))
+		SpinLockGuard _(m_lock);
-			return false;
+		ASSERT(is_page_free(vaddr));
-		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, send_smp_message);
+		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, false);
-		return true;
 	}
 
-	bool PageTable::reserve_range(vaddr_t vaddr, size_t bytes, bool only_free)
+	void PageTable::reserve_range(vaddr_t vaddr, size_t bytes)
 	{
 		if (size_t rem = bytes % PAGE_SIZE)
 			bytes += PAGE_SIZE - rem;
 		ASSERT(vaddr % PAGE_SIZE == 0);
 
 		SpinLockGuard _(m_lock);
-		if (only_free && !is_range_free(vaddr, bytes))
+		ASSERT(is_range_free(vaddr, bytes));
-			return false;
 		for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
-			reserve_page(vaddr + offset, true, false);
+			reserve_page(vaddr + offset);
-		invalidate_range(vaddr, bytes / PAGE_SIZE, true);
-
-		return true;
 	}
 
 	vaddr_t PageTable::reserve_free_page(vaddr_t first_address, vaddr_t last_address)
@@ -615,45 +648,58 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
+		auto state = s_fast_page_lock.lock();
+
 		// Try to find free page that can be mapped without
 		// allocations (page table with unused entries)
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		const uint64_t* pdpt = static_cast<uint64_t*>(map_fast_page(0, m_highest_paging_struct));
 		for (; pdpte <= e_pdpte; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			const uint64_t* pd = static_cast<uint64_t*>(map_fast_page(1, pdpt[pdpte] & s_page_addr_mask));
 			for (; pde < 512; pde++)
 			{
 				if (pdpte == e_pdpte && pde > e_pde)
 					break;
 				if (!(pd[pde] & Flags::Present))
 					continue;
-				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				const uint64_t* pt = static_cast<uint64_t*>(map_fast_page(2, pd[pde] & s_page_addr_mask));
 				for (; pte < 512; pte++)
 				{
 					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 						break;
 					if (pt[pte] & Flags::Used)
 						continue;
+
+					unmap_fast_page(2);
+					unmap_fast_page(1);
+					unmap_fast_page(0);
+					s_fast_page_lock.unlock(state);
+
 					vaddr_t vaddr = 0;
 					vaddr |= (vaddr_t)pdpte << 30;
 					vaddr |= (vaddr_t)pde   << 21;
 					vaddr |= (vaddr_t)pte   << 12;
-					ASSERT(reserve_page(vaddr));
+					reserve_page(vaddr);
 					return vaddr;
 				}
+				unmap_fast_page(2);
 				pte = 0;
 			}
+			unmap_fast_page(1);
 			pde = 0;
 		}
+		unmap_fast_page(0);
+
+		s_fast_page_lock.unlock(state);
 
 		// Find any free page
 		for (vaddr_t vaddr = first_address; vaddr < last_address; vaddr += PAGE_SIZE)
 		{
 			if (is_page_free(vaddr))
 			{
-				ASSERT(reserve_page(vaddr));
+				reserve_page(vaddr);
 				return vaddr;
 			}
 		}
@@ -686,7 +732,7 @@ namespace Kernel
 			}
 			if (valid)
 			{
-				ASSERT(reserve_range(vaddr, page_count * PAGE_SIZE));
+				reserve_range(vaddr, page_count * PAGE_SIZE);
 				return vaddr;
 			}
 		}
@@ -709,12 +755,14 @@ namespace Kernel
 
 	void PageTable::debug_dump()
 	{
-		SpinLockGuard _(m_lock);
+		SpinLockGuard _0(m_lock);
 
 		flags_t flags = 0;
 		vaddr_t start = 0;
 
-		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		SpinLockGuard _1(s_fast_page_lock);
+
+		const uint64_t* pdpt = static_cast<uint64_t*>(map_fast_page(0, m_highest_paging_struct));
 		for (uint32_t pdpte = 0; pdpte < 4; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
@@ -723,7 +771,7 @@ namespace Kernel
 				start = 0;
 				continue;
 			}
-			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			const uint64_t* pd = static_cast<uint64_t*>(map_fast_page(1, pdpt[pdpte] & s_page_addr_mask));
 			for (uint64_t pde = 0; pde < 512; pde++)
 			{
 				if (!(pd[pde] & Flags::Present))
@@ -732,7 +780,7 @@ namespace Kernel
 					start = 0;
 					continue;
 				}
-				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				const uint64_t* pt = static_cast<uint64_t*>(map_fast_page(2, pd[pde] & s_page_addr_mask));
 				for (uint64_t pte = 0; pte < 512; pte++)
 				{
 					if (parse_flags(pt[pte]) != flags)
@@ -750,8 +798,11 @@ namespace Kernel
 						start = (pdpte << 30) | (pde << 21) | (pte << 12);
 					}
 				}
+				unmap_fast_page(2);
 			}
+			unmap_fast_page(1);
 		}
+		unmap_fast_page(0);
 	}
 
 }

kernel/arch/i686/Signal.S

@@ -19,7 +19,7 @@ signal_trampoline:
 	pushl %eax
 	pushl %ebp
 
-	movl 80(%esp), %eax
+	movl 84(%esp), %eax
 	pushl %eax; addl $4, (%esp)
 	pushl (%eax)
 
@@ -59,7 +59,7 @@ signal_trampoline:
 	addl $24, %esp
 
 	// restore sigmask
-	movl $83, %eax      // SYS_SIGPROCMASK
+	movl $79, %eax      // SYS_SIGPROCMASK
 	movl $3,  %ebx      // SIG_SETMASK
 	leal 72(%esp), %ecx // set
 	xorl %edx, %edx     // oset

kernel/arch/i686/boot.S

@@ -98,8 +98,7 @@ bananboot_end:
 boot_pdpt:
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
-	.quad 0
+	.skip 2 * 8
-	.quad 0
 	.long V2P(boot_pd) + (PG_PRESENT)
 	.long 0
 .align 4096
@@ -112,13 +111,16 @@ boot_pd:
 	.endr
 boot_pts:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.rept 512
 			.long i + (PG_READ_WRITE | PG_PRESENT)
 			.long 0
 			.set i, i + 0x1000
 		.endr
 	.endr
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -274,7 +276,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -284,21 +286,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
+ap_stack_paddr:
+	.skip 4
+ap_stack_vaddr:
+	.skip 4
+ap_prepare_paging:
+	.skip 4
+ap_page_table:
+	.skip 4
+ap_ready:
 	.skip 4
-ap_stack_loaded:
-	.skip 1
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -307,8 +315,7 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
+	movl AP_REL(ap_stack_paddr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
 	leal V2P(enable_tsc),        %ecx; call *%ecx
@@ -316,24 +323,28 @@ ap_protected_mode:
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_flush_gdt)
+	ljmpl $0x08, $AP_REL(ap_flush_gdt)
 
 ap_flush_gdt:
-	# move stack pointer to higher half
+	movl $ap_higher_half, %ecx
-	movl %esp, %esp
-	addl $KERNEL_OFFSET, %esp
-
-	# jump to higher half
-	leal ap_higher_half, %ecx
 	jmp *%ecx
 
 ap_higher_half:
+	movl AP_REL(ap_prepare_paging), %eax
+	call *%eax
+
+	# load AP's initial values
+	movl AP_REL(ap_stack_vaddr), %esp
+	movl AP_REL(ap_page_table), %eax
+	movl $1, AP_REL(ap_ready)
+	movl %eax, %cr3
+
 	# clear rbp for stacktrace
 	xorl %ebp, %ebp
 
 1:	pause
 	cmpb $0, g_ap_startup_done
-	jz 1b
+	je 1b
 
 	lock incb g_ap_running_count
 

kernel/arch/i686/linker.ld

@@ -15,17 +15,17 @@ SECTIONS
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{

kernel/arch/x86_64/PageTable.cpp

@@ -2,7 +2,6 @@
 #include <kernel/CPUID.h>
 #include <kernel/Lock/SpinLock.h>
 #include <kernel/Memory/Heap.h>
-#include <kernel/Memory/kmalloc.h>
 #include <kernel/Memory/PageTable.h>
 
 extern uint8_t g_kernel_start[];
@@ -17,13 +16,14 @@ extern uint8_t g_kernel_writable_end[];
 extern uint8_t g_userspace_start[];
 extern uint8_t g_userspace_end[];
 
+extern uint64_t g_boot_fast_page_pt[];
+
 namespace Kernel
 {
 
 	SpinLock PageTable::s_fast_page_lock;
 
 	static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000;
-	static bool s_is_post_heap_done = false;
 
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
@@ -35,6 +35,8 @@ namespace Kernel
 
 	static paddr_t s_global_pml4_entries[512] { 0 };
 
+	static uint64_t* s_fast_page_pt { nullptr };
+
 	static constexpr inline bool is_canonical(uintptr_t addr)
 	{
 		constexpr uintptr_t mask = 0xFFFF800000000000;
@@ -54,34 +56,6 @@ namespace Kernel
 		return addr;
 	}
 
-	struct FuncsKmalloc
-	{
-		static paddr_t allocate_zeroed_page_aligned_page()
-		{
-			void* page = kmalloc(PAGE_SIZE, PAGE_SIZE, true);
-			ASSERT(page);
-			memset(page, 0, PAGE_SIZE);
-			return kmalloc_paddr_of(reinterpret_cast<vaddr_t>(page)).value();
-		}
-
-		static void unallocate_page(paddr_t paddr)
-		{
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			return vaddr - KERNEL_OFFSET + g_boot_info.kernel_paddr;
-		}
-
-		static uint64_t* P2V(paddr_t paddr)
-		{
-			return reinterpret_cast<uint64_t*>(paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET);
-		}
-	};
-
-	struct FuncsHHDM
-	{
 	static paddr_t allocate_zeroed_page_aligned_page()
 	{
 		const paddr_t paddr = Heap::get().take_free_page();
@@ -95,27 +69,14 @@ namespace Kernel
 		Heap::get().release_page(paddr);
 	}
 
-		static paddr_t V2P(vaddr_t vaddr)
-		{
-			ASSERT(vaddr >= s_hhdm_offset);
-			ASSERT(vaddr < KERNEL_OFFSET);
-			return vaddr - s_hhdm_offset;
-		}
-
 	static uint64_t* P2V(paddr_t paddr)
 	{
 		ASSERT(paddr != 0);
 		ASSERT(!BAN::Math::will_addition_overflow(paddr, s_hhdm_offset));
 		return reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
 	}
-	};
 
-	static paddr_t   (*allocate_zeroed_page_aligned_page)() = &FuncsKmalloc::allocate_zeroed_page_aligned_page;
+	static PageTable::flags_t parse_flags(uint64_t entry)
-	static void      (*unallocate_page)(paddr_t)            = &FuncsKmalloc::unallocate_page;
-	static paddr_t   (*V2P)(vaddr_t)                        = &FuncsKmalloc::V2P;
-	static uint64_t* (*P2V)(paddr_t)                        = &FuncsKmalloc::P2V;
-
-	static inline PageTable::flags_t parse_flags(uint64_t entry)
 	{
 		using Flags = PageTable::Flags;
 
@@ -137,7 +98,7 @@ namespace Kernel
 	//   0: 4 KiB
 	//   1: 2 MiB
 	//   2: 1 GiB
-	static void init_map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
+	static void map_hhdm_page(paddr_t pml4, paddr_t paddr, uint8_t page_size)
 	{
 		ASSERT(0 <= page_size && page_size <= 2);
 
@@ -184,7 +145,7 @@ namespace Kernel
 		const uint64_t noexec_flag = s_has_nxe ? (static_cast<uint64_t>(1) << 63) : 0;
 
 		const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, noexec_flag);
-		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags;
+		s_global_pml4_entries[pml4e] = pdpt | hhdm_flags | noexec_flag;
 
 		paddr_t lowest_paddr = pdpt;
 		uint16_t lowest_entry = pdpte;
@@ -207,23 +168,11 @@ namespace Kernel
 		});
 	}
 
-	static void init_map_hhdm(paddr_t pml4)
+	static void initialize_hhdm(paddr_t pml4)
 	{
 		for (const auto& entry : g_boot_info.memory_map_entries)
 		{
-			bool should_map = false;
+			if (entry.type != MemoryMapEntry::Type::Available)
-			switch (entry.type)
-			{
-				case MemoryMapEntry::Type::Available:
-					should_map = true;
-					break;
-				case MemoryMapEntry::Type::ACPIReclaim:
-				case MemoryMapEntry::Type::ACPINVS:
-				case MemoryMapEntry::Type::Reserved:
-					should_map = false;
-					break;
-			}
-			if (!should_map)
 				continue;
 
 			constexpr size_t one_gib = 1024 * 1024 * 1024;
@@ -235,156 +184,39 @@ namespace Kernel
 			{
 				if (s_has_gib && paddr % one_gib == 0 && paddr + one_gib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 2);
+					map_hhdm_page(pml4, paddr, 2);
 					paddr += one_gib;
 				}
 				else if (paddr % two_mib == 0 && paddr + two_mib <= entry_end)
 				{
-					init_map_hhdm_page(pml4, paddr, 1);
+					map_hhdm_page(pml4, paddr, 1);
 					paddr += two_mib;
 				}
 				else
 				{
-					init_map_hhdm_page(pml4, paddr, 0);
+					map_hhdm_page(pml4, paddr, 0);
 					paddr += PAGE_SIZE;
 				}
 			}
 		}
 	}
 
-	static paddr_t copy_page_from_kmalloc_to_heap(paddr_t kmalloc_paddr)
+	void PageTable::initialize_fast_page()
 	{
-		const paddr_t heap_paddr = Heap::get().take_free_page();
+		s_fast_page_pt = g_boot_fast_page_pt;
-		ASSERT(heap_paddr);
-
-		const vaddr_t kmalloc_vaddr = kmalloc_vaddr_of(kmalloc_paddr).value();
-
-		PageTable::with_fast_page(heap_paddr, [kmalloc_vaddr] {
-			memcpy(PageTable::fast_page_as_ptr(), reinterpret_cast<void*>(kmalloc_vaddr), PAGE_SIZE);
-		});
-
-		return heap_paddr;
 	}
 
-	static void copy_paging_structure_to_heap(uint64_t* old_table, uint64_t* new_table, int depth)
+	static void detect_cpu_features()
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t old_entry = old_table[index];
-			if (old_entry == 0)
-			{
-				new_table[index] = 0;
-				continue;
-			}
-
-			const paddr_t old_paddr = old_entry & page_addr_mask;
-			const paddr_t new_paddr = copy_page_from_kmalloc_to_heap(old_paddr);
-			new_table[index] = new_paddr | (old_entry & page_flag_mask);
-
-			uint64_t* next_old_table = reinterpret_cast<uint64_t*>(old_paddr + s_hhdm_offset);
-			uint64_t* next_new_table = reinterpret_cast<uint64_t*>(new_paddr + s_hhdm_offset);
-			copy_paging_structure_to_heap(next_old_table, next_new_table, depth - 1);
-		}
-	}
-
-	static void free_kmalloc_paging_structure(uint64_t* table, int depth)
-	{
-		if (depth == 0)
-			return;
-
-		constexpr uint64_t page_flag_mask = 0x8000000000000FFF;
-		constexpr uint64_t page_addr_mask = ~page_flag_mask;
-
-		for (uint16_t index = 0; index < 512; index++)
-		{
-			const uint64_t entry = table[index];
-			if (entry == 0)
-				continue;
-
-			const paddr_t paddr = entry & page_addr_mask;
-
-			uint64_t* next_table = reinterpret_cast<uint64_t*>(paddr + s_hhdm_offset);
-			free_kmalloc_paging_structure(next_table, depth - 1);
-
-			kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(paddr).value()));
-		}
-	}
-
-	void PageTable::initialize_pre_heap()
 	{
 		if (CPUID::has_nxe())
 			s_has_nxe = true;
-
 		if (CPUID::has_pge())
 			s_has_pge = true;
-
 		if (CPUID::has_1gib_pages())
 			s_has_gib = true;
-
-		ASSERT(s_kernel == nullptr);
-		s_kernel = new PageTable();
-		ASSERT(s_kernel);
-
-		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-		s_kernel->prepare_fast_page();
-		s_kernel->initialize_kernel();
-
-		for (auto pml4e : s_global_pml4_entries)
-			ASSERT(pml4e == 0);
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		s_global_pml4_entries[511] = pml4[511];
 	}
 
-	void PageTable::initialize_post_heap()
+	void PageTable::enable_cpu_features()
-	{
-		ASSERT(s_kernel);
-
-		init_map_hhdm(s_kernel->m_highest_paging_struct);
-
-		const paddr_t old_pml4_paddr = s_kernel->m_highest_paging_struct;
-		const paddr_t new_pml4_paddr = copy_page_from_kmalloc_to_heap(old_pml4_paddr);
-
-		uint64_t* old_pml4 = reinterpret_cast<uint64_t*>(kmalloc_vaddr_of(old_pml4_paddr).value());
-		uint64_t* new_pml4 = reinterpret_cast<uint64_t*>(new_pml4_paddr + s_hhdm_offset);
-
-		const paddr_t old_pdpt_paddr = old_pml4[511] & s_page_addr_mask;
-		const paddr_t new_pdpt_paddr = Heap::get().take_free_page();
-		ASSERT(new_pdpt_paddr);
-
-		uint64_t* old_pdpt = reinterpret_cast<uint64_t*>(old_pdpt_paddr + s_hhdm_offset);
-		uint64_t* new_pdpt = reinterpret_cast<uint64_t*>(new_pdpt_paddr + s_hhdm_offset);
-		copy_paging_structure_to_heap(old_pdpt, new_pdpt, 2);
-
-		new_pml4[511] = new_pdpt_paddr | (old_pml4[511] & s_page_flag_mask);
-		s_global_pml4_entries[511] = new_pml4[511];
-
-		s_kernel->m_highest_paging_struct = new_pml4_paddr;
-		s_kernel->load();
-
-		free_kmalloc_paging_structure(old_pdpt, 2);
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pdpt_paddr).value()));
-		kfree(reinterpret_cast<void*>(kmalloc_vaddr_of(old_pml4_paddr).value()));
-
-		allocate_zeroed_page_aligned_page = &FuncsHHDM::allocate_zeroed_page_aligned_page;
-		unallocate_page                   = &FuncsHHDM::unallocate_page;
-		V2P                               = &FuncsHHDM::V2P;
-		P2V                               = &FuncsHHDM::P2V;
-
-		s_is_post_heap_done = true;
-
-		// This is a hack to unmap fast page. fast page pt is copied
-		// while it is mapped, so we need to manually unmap it
-		SpinLockGuard _(s_fast_page_lock);
-		unmap_fast_page();
-	}
-
-	void PageTable::initial_load()
 	{
 		if (s_has_nxe)
 		{
@@ -423,8 +255,63 @@ namespace Kernel
 			"movq %%rax, %%cr0;"
 			::: "rax"
 		);
+	}
 
-		load();
+	void PageTable::initialize_and_load()
+	{
+		detect_cpu_features();
+		enable_cpu_features();
+
+		const paddr_t boot_pml4_paddr = ({
+			paddr_t paddr;
+			asm volatile("movq %%cr3, %0" : "=r"(paddr));
+			paddr;
+		});
+
+		initialize_hhdm(boot_pml4_paddr);
+
+		ASSERT(s_kernel == nullptr);
+		s_kernel = new PageTable();
+		ASSERT(s_kernel != nullptr);
+
+		s_kernel->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		ASSERT(s_kernel->m_highest_paging_struct);
+
+		uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+		s_kernel->map_kernel_memory();
+		s_global_pml4_entries[511] = pml4[511];
+
+		// update fast page pt
+		{
+			constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+			constexpr uint16_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+			constexpr uint16_t pdpte = (uc_vaddr >> 30) & 0x1FF;
+			constexpr uint16_t pde   = (uc_vaddr >> 21) & 0x1FF;
+
+			const auto get_or_allocate_entry =
+				[](paddr_t table_paddr, uint16_t entry, uint64_t flags)
+				{
+					uint64_t* table = P2V(table_paddr);
+
+					if (!(table[entry] & Flags::Present))
+					{
+						table[entry] = allocate_zeroed_page_aligned_page();
+						ASSERT(table[entry]);
+					}
+
+					table[entry] |= flags;
+
+					return table[entry] & s_page_addr_mask;
+				};
+
+			const paddr_t pml4 = s_kernel->m_highest_paging_struct;
+			const paddr_t pdpt = get_or_allocate_entry(pml4, pml4e, Flags::ReadWrite | Flags::Present);
+			const paddr_t pd   = get_or_allocate_entry(pdpt, pdpte, Flags::ReadWrite | Flags::Present);
+			s_fast_page_pt = P2V(get_or_allocate_entry(pd,   pde,   Flags::ReadWrite | Flags::Present));
+		}
+
+		s_kernel->load();
 	}
 
 	PageTable& PageTable::kernel()
@@ -440,12 +327,12 @@ namespace Kernel
 		return true;
 	}
 
-	void PageTable::initialize_kernel()
+	void PageTable::map_kernel_memory()
 	{
 		// Map (phys_kernel_start -> phys_kernel_end) to (virt_kernel_start -> virt_kernel_end)
 		const vaddr_t kernel_start = reinterpret_cast<vaddr_t>(g_kernel_start);
 		map_range_at(
-			V2P(kernel_start),
+			kernel_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_start,
 			g_kernel_end - g_kernel_start,
 			Flags::Present
@@ -454,7 +341,7 @@ namespace Kernel
 		// Map executable kernel memory as executable
 		const vaddr_t kernel_execute_start = reinterpret_cast<vaddr_t>(g_kernel_execute_start);
 		map_range_at(
-			V2P(kernel_execute_start),
+			kernel_execute_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_execute_start,
 			g_kernel_execute_end - g_kernel_execute_start,
 			Flags::Execute | Flags::Present
@@ -463,7 +350,7 @@ namespace Kernel
 		// Map writable kernel memory as writable
 		const vaddr_t kernel_writable_start = reinterpret_cast<vaddr_t>(g_kernel_writable_start);
 		map_range_at(
-			V2P(kernel_writable_start),
+			kernel_writable_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			kernel_writable_start,
 			g_kernel_writable_end - g_kernel_writable_start,
 			Flags::ReadWrite | Flags::Present
@@ -472,114 +359,80 @@ namespace Kernel
 		// Map userspace memory
 		const vaddr_t userspace_start = reinterpret_cast<vaddr_t>(g_userspace_start);
 		map_range_at(
-			V2P(userspace_start),
+			userspace_start - KERNEL_OFFSET + g_boot_info.kernel_paddr,
 			userspace_start,
 			g_userspace_end - g_userspace_start,
 			Flags::Execute | Flags::UserSupervisor | Flags::Present
 		);
 	}
 
-	void PageTable::prepare_fast_page()
-	{
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
-
-		uint64_t* pml4 = P2V(m_highest_paging_struct);
-		ASSERT(!(pml4[pml4e] & Flags::Present));
-		pml4[pml4e] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		ASSERT(!(pdpt[pdpte] & Flags::Present));
-		pdpt[pdpte] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-		ASSERT(!(pd[pde] & Flags::Present));
-		pd[pde] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
-
-		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-		ASSERT(pt[pte] == 0);
-		pt[pte] = Flags::Reserved;
-	}
-
 	void PageTable::map_fast_page(paddr_t paddr)
 	{
-		ASSERT(s_kernel);
+		map_fast_page(0, paddr);
-		ASSERT(paddr);
-		ASSERT(paddr % PAGE_SIZE == 0);
-
-		ASSERT(s_fast_page_lock.current_processor_has_lock());
-
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
-
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
-
-		ASSERT(!(pt[pte] & Flags::Present));
-		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
-
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	void PageTable::unmap_fast_page()
 	{
-		ASSERT(s_kernel);
+		unmap_fast_page(0);
+	}
 
+	void* PageTable::map_fast_page(size_t index, paddr_t paddr)
+	{
+		ASSERT(paddr && paddr % PAGE_SIZE == 0);
+
+		ASSERT(index < 512);
+		ASSERT(s_fast_page_pt);
+
+		if (index < reserved_fast_pages)
 			ASSERT(s_fast_page_lock.current_processor_has_lock());
+		else
+			ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
 
-		constexpr vaddr_t uc_vaddr = uncanonicalize(fast_page());
+		ASSERT(!(s_fast_page_pt[index] & Flags::Present));
-		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
+		s_fast_page_pt[index] = paddr | Flags::ReadWrite | Flags::Present;
-		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
-		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
-		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
 
-		const uint64_t* pml4 = P2V(s_kernel->m_highest_paging_struct);
+		void* address = reinterpret_cast<void*>(fast_page() + index * PAGE_SIZE);
-		const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
+		asm volatile("invlpg (%0)" :: "r"(address));
-		const uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
+		return address;
-		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
+	}
 
-		ASSERT(pt[pte] & Flags::Present);
+	void PageTable::unmap_fast_page(size_t index)
-		pt[pte] = Flags::Reserved;
+	{
+		ASSERT(index < 512);
+		ASSERT(s_fast_page_pt);
 
-		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
+		if (index < reserved_fast_pages)
+			ASSERT(s_fast_page_lock.current_processor_has_lock());
+		else
+			ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
+
+		ASSERT((s_fast_page_pt[index] & Flags::Present));
+		s_fast_page_pt[index] = 0;
+
+		asm volatile("invlpg (%0)" :: "r"(fast_page() + index * PAGE_SIZE));
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
 	{
 		SpinLockGuard _(s_kernel->m_lock);
+
 		PageTable* page_table = new PageTable;
 		if (page_table == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
-		page_table->map_kernel_memory();
+
+		page_table->m_highest_paging_struct = allocate_zeroed_page_aligned_page();
+		if (page_table->m_highest_paging_struct == 0)
+		{
+			delete page_table;
+			return BAN::Error::from_errno(ENOMEM);
+		}
+
+		uint64_t* pml4 = P2V(page_table->m_highest_paging_struct);
+		memcpy(pml4, s_global_pml4_entries, sizeof(s_global_pml4_entries));
+
 		return page_table;
 	}
 
-	void PageTable::map_kernel_memory()
-	{
-		ASSERT(s_kernel);
-		ASSERT(s_global_pml4_entries[511]);
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = allocate_zeroed_page_aligned_page();
-
-		PageTable::with_fast_page(m_highest_paging_struct, [] {
-			for (size_t i = 0; i < 512; i++)
-			{
-				if (s_global_pml4_entries[i] == 0)
-					continue;
-				ASSERT(i >= 256);
-				PageTable::fast_page_as_sized<uint64_t>(i) = s_global_pml4_entries[i];
-			}
-		});
-	}
-
 	PageTable::~PageTable()
 	{
 		if (m_highest_paging_struct == 0)
@@ -624,32 +477,10 @@ namespace Kernel
 		const bool is_userspace = (vaddr < KERNEL_OFFSET);
 		if (is_userspace && this != &PageTable::current())
 			;
-		else if (pages <= 32 || !s_is_post_heap_done)
+		else if (pages >= full_tlb_flush_threshold)
-		{
+			invalidate_full_address_space(!is_userspace);
-			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
+		else for (size_t i = 0; i < pages; i++)
-				asm volatile("invlpg (%0)" :: "r"(vaddr));
+			asm volatile("invlpg (%0)" :: "r"(vaddr + i * PAGE_SIZE));
-		}
-		else if (is_userspace || !s_has_pge)
-		{
-			asm volatile("movq %0, %%cr3" :: "r"(m_highest_paging_struct));
-		}
-		else
-		{
-			asm volatile(
-				"movq %%cr4, %%rax;"
-
-				"andq $~0x80, %%rax;"
-				"movq %%rax, %%cr4;"
-
-				"movq %0, %%cr3;"
-
-				"orq $0x80, %%rax;"
-				"movq %%rax, %%cr4;"
-				:
-				: "r"(m_highest_paging_struct)
-				: "rax"
-			);
-		}
 
 		if (send_smp_message)
 		{
@@ -664,6 +495,34 @@ namespace Kernel
 		}
 	}
 
+	void PageTable::invalidate_full_address_space(bool global)
+	{
+		if (!global || !s_has_pge)
+		{
+			asm volatile(
+				"movq %%cr3, %%rax;"
+				"movq %%rax, %%cr3;"
+				::: "rax"
+			);
+		}
+		else
+		{
+			asm volatile(
+				"movq %%cr4, %%rax;"
+
+				"andq $~0x80, %%rax;"
+				"movq %%rax, %%cr4;"
+
+				"movq %%cr3, %%rcx;"
+				"movq %%rcx, %%cr3;"
+
+				"orq $0x80, %%rax;"
+				"movq %%rax, %%cr4;"
+				::: "rax", "rcx"
+			);
+		}
+	}
+
 	void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
 	{
 		ASSERT(vaddr);
@@ -705,12 +564,70 @@ namespace Kernel
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
 
-		const size_t page_count = range_page_count(vaddr, size);
+		ASSERT(is_canonical(vaddr));
+		ASSERT(is_canonical(vaddr + size - 1));
+
+		const vaddr_t uc_vaddr_start = uncanonicalize(vaddr);
+		const vaddr_t uc_vaddr_end   = uncanonicalize(vaddr + size - 1);
+
+		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
+		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
+		uint16_t pde   = (uc_vaddr_start >> 21) & 0x1FF;
+		uint16_t pte   = (uc_vaddr_start >> 12) & 0x1FF;
+
+		const uint16_t e_pml4e = (uc_vaddr_end >> 39) & 0x1FF;
+		const uint16_t e_pdpte = (uc_vaddr_end >> 30) & 0x1FF;
+		const uint16_t e_pde   = (uc_vaddr_end >> 21) & 0x1FF;
+		const uint16_t e_pte   = (uc_vaddr_end >> 12) & 0x1FF;
 
 		SpinLockGuard _(m_lock);
-		for (vaddr_t page = 0; page < page_count; page++)
+
-			unmap_page(vaddr + page * PAGE_SIZE, false);
+		uint64_t* pml4 = P2V(m_highest_paging_struct);
-		invalidate_range(vaddr, page_count, true);
+		for (; pml4e <= e_pml4e; pml4e++)
+		{
+#define UNALLOCATE_TABLE_IF_EMPTY(outer, inner) \
+			if (old_##inner##e == 0 && inner##e == 512) { \
+				unallocate_page(outer[outer##e] & s_page_addr_mask); \
+				outer[outer##e] = 0; \
+			}
+			if (!(pml4[pml4e] & Flags::Present))
+				continue;
+			const uint16_t old_pdpte = pdpte;
+			uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
+			for (; pdpte < 512; pdpte++)
+			{
+				if (pml4e == e_pml4e && pdpte > e_pdpte)
+					break;
+				if (!(pdpt[pdpte] & Flags::Present))
+					continue;
+				const uint16_t old_pde = pde;
+				uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+				for (; pde < 512; pde++)
+				{
+					if (pml4e == e_pml4e && pdpte == e_pdpte && pde > e_pde)
+						break;
+					if (!(pd[pde] & Flags::Present))
+						continue;
+					const uint16_t old_pte = pte;
+					uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+					for (; pte < 512; pte++)
+					{
+						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
+							break;
+						pt[pte] = 0;
+					}
+					UNALLOCATE_TABLE_IF_EMPTY(pd, pt);
+					pte = 0;
+				}
+				UNALLOCATE_TABLE_IF_EMPTY(pdpt, pd);
+				pde = 0;
+			}
+			UNALLOCATE_TABLE_IF_EMPTY(pml4, pdpt);
+			pdpte = 0;
+#undef UNALLOCATE_TABLE_IF_EMPTY
+		}
+
+		invalidate_range(vaddr, range_page_count(vaddr, size), true);
 	}
 
 	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
@@ -897,33 +814,74 @@ namespace Kernel
 
 	paddr_t PageTable::physical_address_of(vaddr_t addr) const
 	{
-		uint64_t page_data = get_page_data(addr);
+		return get_page_data(addr) & s_page_addr_mask;
-		return page_data & s_page_addr_mask;
 	}
 
-	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool invalidate)
+	void PageTable::reserve_page(vaddr_t vaddr)
 	{
-		SpinLockGuard _(m_lock);
 		ASSERT(vaddr % PAGE_SIZE == 0);
-		if (only_free && !is_page_free(vaddr))
+		SpinLockGuard _(m_lock);
-			return false;
+		ASSERT(is_page_free(vaddr));
-		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, invalidate);
+		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, false);
-		return true;
 	}
 
-	bool PageTable::reserve_range(vaddr_t vaddr, size_t bytes, bool only_free)
+	void PageTable::reserve_range(vaddr_t vaddr, size_t bytes)
 	{
 		if (size_t rem = bytes % PAGE_SIZE)
 			bytes += PAGE_SIZE - rem;
 		ASSERT(vaddr % PAGE_SIZE == 0);
 
+		ASSERT(is_canonical(vaddr));
+		ASSERT(is_canonical(vaddr + bytes - 1));
+
+		const vaddr_t uc_vaddr_start = uncanonicalize(vaddr);
+		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
+		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
+		uint16_t pde   = (uc_vaddr_start >> 21) & 0x1FF;
+		uint16_t pte   = (uc_vaddr_start >> 12) & 0x1FF;
+
+		size_t pages_to_reserve = bytes / PAGE_SIZE;
+		ASSERT(pages_to_reserve);
+
 		SpinLockGuard _(m_lock);
-		if (only_free && !is_range_free(vaddr, bytes))
+
-			return false;
+		uint64_t* pml4 = P2V(m_highest_paging_struct);
-		for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
+		for (;; pml4e++)
-			reserve_page(vaddr + offset, true, false);
+		{
-		invalidate_range(vaddr, bytes / PAGE_SIZE, true);
+#define CHECK_IF_PRESENT(expr) \
-		return true;
+			if (!((expr) & Flags::Present)) { \
+				const paddr_t paddr = allocate_zeroed_page_aligned_page(); \
+				ASSERT(paddr); \
+				(expr) = paddr | Flags::Present; \
+			}
+			CHECK_IF_PRESENT(pml4[pml4e]);
+			uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
+			for (; pdpte < 512; pdpte++)
+			{
+				CHECK_IF_PRESENT(pdpt[pdpte]);
+				uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+				for (; pde < 512; pde++)
+				{
+					CHECK_IF_PRESENT(pd[pde]);
+					uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+					for (; pte < 512; pte++)
+					{
+						ASSERT(!(pt[pte] & Flags::Used));
+						pt[pte] = Flags::Reserved;
+
+						pages_to_reserve--;
+						if (pages_to_reserve == 0)
+							return;
+					}
+					pte = 0;
+				}
+				pde = 0;
+			}
+			pdpte = 0;
+#undef CHECK_IF_PRESENT
+		}
+
+		ASSERT_NOT_REACHED();
 	}
 
 	vaddr_t PageTable::reserve_free_page(vaddr_t first_address, vaddr_t last_address)
@@ -937,6 +895,7 @@ namespace Kernel
 
 		ASSERT(is_canonical(first_address));
 		ASSERT(is_canonical(last_address - 1));
+
 		const vaddr_t uc_vaddr_start = uncanonicalize(first_address);
 		const vaddr_t uc_vaddr_end   = uncanonicalize(last_address - 1);
 
@@ -986,7 +945,7 @@ namespace Kernel
 						vaddr |= static_cast<uint64_t>(pde)   << 21;
 						vaddr |= static_cast<uint64_t>(pte)   << 12;
 						vaddr = canonicalize(vaddr);
-						ASSERT(reserve_page(vaddr));
+						reserve_page(vaddr);
 						return vaddr;
 					}
 					pte = 0;
@@ -1000,7 +959,7 @@ namespace Kernel
 		{
 			if (vaddr_t vaddr = canonicalize(uc_vaddr); is_page_free(vaddr))
 			{
-				ASSERT(reserve_page(vaddr));
+				reserve_page(vaddr);
 				return vaddr;
 			}
 		}
@@ -1010,44 +969,90 @@ namespace Kernel
 
 	vaddr_t PageTable::reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address)
 	{
-		if (first_address >= KERNEL_OFFSET && first_address < (vaddr_t)g_kernel_start)
+		if (first_address >= KERNEL_OFFSET && first_address < reinterpret_cast<vaddr_t>(g_kernel_start))
-			first_address = (vaddr_t)g_kernel_start;
+			first_address = reinterpret_cast<vaddr_t>(g_kernel_start);
-		if (size_t rem = first_address % PAGE_SIZE)
+		if (const auto rem = first_address % PAGE_SIZE)
 			first_address += PAGE_SIZE - rem;
-		if (size_t rem = last_address % PAGE_SIZE)
+		if (const auto rem = last_address % PAGE_SIZE)
 			last_address -= rem;
 
 		ASSERT(is_canonical(first_address));
 		ASSERT(is_canonical(last_address - 1));
 
+		const vaddr_t uc_vaddr_start = uncanonicalize(first_address);
+		const vaddr_t uc_vaddr_end   = uncanonicalize(last_address - 1);
+
+		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
+		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
+		uint16_t pde   = (uc_vaddr_start >> 21) & 0x1FF;
+		uint16_t pte   = (uc_vaddr_start >> 12) & 0x1FF;
+
+		const uint16_t e_pml4e = (uc_vaddr_end >> 39) & 0x1FF;
+		const uint16_t e_pdpte = (uc_vaddr_end >> 30) & 0x1FF;
+		const uint16_t e_pde   = (uc_vaddr_end >> 21) & 0x1FF;
+		const uint16_t e_pte   = (uc_vaddr_end >> 12) & 0x1FF;
+
+		vaddr_t vaddr = first_address;
+		size_t free_count = 0;
+
 		SpinLockGuard _(m_lock);
 
-		for (vaddr_t vaddr = first_address; vaddr < last_address;)
+		const uint64_t* pml4 = P2V(m_highest_paging_struct);
+		for (; pml4e <= e_pml4e; pml4e++)
 		{
-			bool valid { true };
+#define CHECK_IF_PRESENT(expr, advance) \
-			for (size_t page = 0; page < page_count; page++)
+			if (!((expr) & Flags::Present)) { \
+				if ((free_count += advance) >= page_count) \
+					goto found_free_region; \
+				continue; \
+			}
+			CHECK_IF_PRESENT(pml4[pml4e], 512 * 512 * 512);
+			const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
+			for (; pdpte < 512; pdpte++)
 			{
-				if (!is_canonical(vaddr + page * PAGE_SIZE))
+				if (pml4e == e_pml4e && pdpte > e_pdpte)
-				{
-					vaddr = canonicalize(uncanonicalize(vaddr) + page * PAGE_SIZE);
-					valid = false;
 					break;
-				}
+				CHECK_IF_PRESENT(pdpt[pdpte], 512 * 512);
-				if (!is_page_free(vaddr + page * PAGE_SIZE))
+				const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+				for (; pde < 512; pde++)
 				{
-					vaddr += (page + 1) * PAGE_SIZE;
+					if (pml4e == e_pml4e && pdpte == e_pdpte && pde > e_pde)
-					valid = false;
 						break;
-				}
+					CHECK_IF_PRESENT(pd[pde], 512);
-			}
+					uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-			if (valid)
+					for (; pte < 512; pte++)
 					{
-				ASSERT(reserve_range(vaddr, page_count * PAGE_SIZE));
+						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
-				return vaddr;
+							break;
+						if (!(pt[pte] & Flags::Used))
+						{
+							if (++free_count >= page_count)
+								goto found_free_region;
 						}
+						else
+						{
+							vaddr = 0;
+							vaddr |= static_cast<uint64_t>(pml4e) << 39;
+							vaddr |= static_cast<uint64_t>(pdpte) << 30;
+							vaddr |= static_cast<uint64_t>(pde)   << 21;
+							vaddr |= static_cast<uint64_t>(pte)   << 12;
+							vaddr = canonicalize(vaddr + PAGE_SIZE);
+							free_count = 0;
+						}
+					}
+					pte = 0;
+				}
+				pde = 0;
+			}
+			pdpte = 0;
+#undef CHECK_IF_PRESENT
 		}
 
 		return 0;
+
+	found_free_region:
+		reserve_range(vaddr, page_count * PAGE_SIZE);
+		return vaddr;
 	}
 
 	bool PageTable::is_page_free(vaddr_t page) const

kernel/arch/x86_64/Signal.S

@@ -61,7 +61,7 @@ signal_trampoline:
 	addq $40, %rsp
 
 	// restore sigmask
-	movq $83, %rdi       // SYS_SIGPROCMASK
+	movq $79, %rdi       // SYS_SIGPROCMASK
 	movq $3,  %rsi       // SIG_SETMASK
 	leaq 192(%rsp), %rdx // set
 	xorq %r10, %r10      // oset

kernel/arch/x86_64/boot.S

@@ -97,27 +97,25 @@ bananboot_end:
 .align 4096
 boot_pml4:
 	.quad V2P(boot_pdpt_lo) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 510
+	.skip 510 * 8
-		.quad 0
-	.endr
 	.quad V2P(boot_pdpt_hi) + (PG_READ_WRITE | PG_PRESENT)
 boot_pdpt_lo:
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.rept 511
+	.skip 511 * 8
-		.quad 0
-	.endr
 boot_pdpt_hi:
-	.rept 510
+	.skip 510 * 8
-		.quad 0
-	.endr
 	.quad V2P(boot_pd) + (PG_READ_WRITE | PG_PRESENT)
-	.quad 0
+	.skip 8
 boot_pd:
 	.set i, 0
-	.rept 512
+	.rept 511
 		.quad i + (PG_PAGE_SIZE | PG_READ_WRITE | PG_PRESENT)
 		.set i, i + 0x200000
 	.endr
+	.quad V2P(g_boot_fast_page_pt) + (PG_READ_WRITE | PG_PRESENT)
+.global g_boot_fast_page_pt
+g_boot_fast_page_pt:
+	.skip 512 * 8
 
 boot_gdt:
 	.quad 0x0000000000000000 # null descriptor
@@ -273,7 +271,7 @@ system_halt:
 	jmp 1b
 
 
-#define AP_V2P(vaddr) ((vaddr) - ap_trampoline + 0xF000)
+#define AP_REL(vaddr) ((vaddr) - ap_trampoline + 0xF000)
 
 .section .ap_init, "ax"
 
@@ -283,21 +281,27 @@ ap_trampoline:
 	jmp 1f
 
 .align 8
-ap_stack_ptr:
+ap_stack_paddr:
-	.skip 4
+	.skip 8
-ap_stack_loaded:
+ap_stack_vaddr:
-	.skip 1
+	.skip 8
+ap_prepare_paging:
+	.skip 8
+ap_page_table:
+	.skip 8
+ap_ready:
+	.skip 8
 
 1:	cli; cld
-	ljmpl $0x00, $AP_V2P(ap_cs_clear)
+	ljmpl $0x00, $AP_REL(ap_cs_clear)
 
 ap_cs_clear:
 	# load ap gdt and enter protected mode
-	lgdt AP_V2P(ap_gdtr)
+	lgdt AP_REL(ap_gdtr)
 	movl %cr0, %eax
 	orb $1, %al
 	movl %eax, %cr0
-	ljmpl $0x08, $AP_V2P(ap_protected_mode)
+	ljmpl $0x08, $AP_REL(ap_protected_mode)
 
 .code32
 ap_protected_mode:
@@ -306,8 +310,7 @@ ap_protected_mode:
 	movw %ax, %ss
 	movw %ax, %es
 
-	movl AP_V2P(ap_stack_ptr), %esp
+	movl AP_REL(ap_stack_paddr), %esp
-	movb $1, AP_V2P(ap_stack_loaded)
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
 	leal V2P(enable_tsc),        %ecx; call *%ecx
@@ -315,28 +318,34 @@ ap_protected_mode:
 
 	# load boot gdt and enter long mode
 	lgdt V2P(boot_gdtr)
-	ljmpl $0x08, $AP_V2P(ap_long_mode)
+	ljmpl $0x08, $AP_REL(ap_long_mode)
 
 .code64
 ap_long_mode:
-	# move stack pointer to higher half
+	movq $ap_higher_half, %rax
-	movl %esp, %esp
+	jmp *%rax
-	addq $KERNEL_OFFSET, %rsp
+
+ap_higher_half:
+	movq AP_REL(ap_prepare_paging), %rax
+	call *%rax
+
+	# load AP's initial values
+	movq AP_REL(ap_stack_vaddr), %rsp
+	movq AP_REL(ap_page_table), %rax
+	movq $1, AP_REL(ap_ready)
+	movq %rax, %cr3
 
 	# clear rbp for stacktrace
 	xorq %rbp, %rbp
 
-	xorb %al, %al
 1:	pause
-	cmpb %al, g_ap_startup_done
+	cmpb $0, g_ap_startup_done
-	jz 1b
+	je 1b
 
 	lock incb g_ap_running_count
 
-	# jump to ap_main in higher half
+	call ap_main
-	movabsq $ap_main, %rcx
+	jmp system_halt
-	call *%rcx
-	jmp V2P(system_halt)
 
 ap_gdt:
 	.quad 0x0000000000000000 # null descriptor

kernel/arch/x86_64/linker.ld

@@ -15,17 +15,17 @@ SECTIONS
 		*(.bananboot)
 		*(.text.*)
 	}
+	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
+	{
+		g_ap_init_addr = .;
+		*(.ap_init)
+		g_kernel_execute_end = .;
+	}
 	.userspace ALIGN(4K) : AT(ADDR(.userspace) - KERNEL_OFFSET)
 	{
 		g_userspace_start = .;
 		*(.userspace)
 		g_userspace_end = .;
-		g_kernel_execute_end = .;
-	}
-	.ap_init ALIGN(4K) : AT(ADDR(.ap_init) - KERNEL_OFFSET)
-	{
-		g_ap_init_addr = .;
-		*(.ap_init)
 	}
 	.rodata ALIGN(4K) : AT(ADDR(.rodata) - KERNEL_OFFSET)
 	{
@@ -41,7 +41,18 @@ SECTIONS
 	{
 		g_kernel_writable_start = .;
 		*(.data)
+
+		. = ALIGN(8);
+		g_drv_builtin_begin = .;
+		KEEP(*(.banos-driver))
+		g_drv_builtin_end = .;
+		. = ALIGN(8);
+		g_banos_export = .;
+		KEEP(*(.banos-export))
+		g_banos_export_end = .;
 	}
+
+
 	.bss ALIGN(4K) : AT(ADDR(.bss) - KERNEL_OFFSET)
 	{
 		g_kernel_bss_start = .;

kernel/include/banos/driver.h

@@ -0,0 +1,27 @@
+#pragma once
+// Copyright (c) 2026 Dcraftbg
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#include "version.h"
+#include "revision.h"
+
+#define BANOS_DRIVER_REVISION_CURRENT 0
+
+typedef struct Banos_Driver Banos_Driver;
+struct Banos_Driver {
+    unsigned long driver_size;
+    banos_version_t minimal_banos_version;
+    const char* name;
+    const char* license;
+    banos_version_t version;
+
+    // NOTE: checkout BANOS_DRIVER_INSTANCE_SIZE.
+    // You may use this instance data for anything you wish to store.
+    // If you need more than that just allocate it on the heap or
+    // globally if you add the proper verification of having your driver run only
+    // within a single instance
+    int (*init)(Banos_Driver* drv);
+    int (*uninit)(Banos_Driver* drv);
+};
+#define BANOS_DRIVER_API static __attribute__((section(".banos-driver"), used, aligned(8)))

kernel/include/banos/export.h

@@ -0,0 +1,15 @@
+#pragma once
+// Copyright (c) 2026 Dcraftbg
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+typedef struct Banos_Symbol {
+    const char* name;
+    void* arg;
+} Banos_Symbol;
+#define BANOS_EXPORT_SYMBOL(symname, str, ptr) \
+    static __attribute__((section(".banos-export"), used, aligned(8))) Banos_Symbol __symbol_##symname = {\
+        .name = str, \
+        .arg = ptr \
+    };
+#define BANOS_EXPORT(name) BANOS_EXPORT_SYMBOL(name, #name, (void*)&name)

kernel/include/banos/print.h

@@ -0,0 +1,13 @@
+#pragma once
+// Copyright (c) 2026 Dcraftbg
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void banos_dprintln(const char* str);
+#ifdef __cplusplus
+}
+#endif

kernel/include/banos/revision.h

@@ -0,0 +1,2 @@
+#pragma once
+typedef unsigned long banos_revision_t;

kernel/include/banos/version.h

@@ -0,0 +1,28 @@
+#pragma once
+// Copyright (c) 2026 Dcraftbg
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+// THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+// [ 8 bit major ] [ 8 minor ] [ 16 patch]
+typedef unsigned int banos_version_t;
+#define BANOS_VERSION_MAJOR_SHIFT 24
+#define BANOS_VERSION_MINOR_SHIFT 16
+#define BANOS_VERSION_PATCH_SHIFT 0
+
+#define BANOS_VERSION_MAJOR_MASK 0xFF
+#define BANOS_VERSION_MINOR_MASK 0xFF
+#define BANOS_VERSION_PATCH_MASK 0xFFFF
+
+#define BANOS_VERSION_MAKE(major, minor, patch) \
+    (banos_version_t)( \
+        (((major) & BANOS_VERSION_MAJOR_MASK) << BANOS_VERSION_MAJOR_SHIFT) | \
+        (((minor) & BANOS_VERSION_MINOR_MASK) << BANOS_VERSION_MINOR_SHIFT) | \
+        (((patch) & BANOS_VERSION_PATCH_MASK) << BANOS_VERSION_PATCH_SHIFT) \
+    )
+
+#define BANOS_VERSION_CURRENT BANOS_VERSION_MAKE(0, 0, 1)
+
+#define BANOS_VERSION_GET_MAJOR(v) (((v) >> BANOS_VERSION_MAJOR_SHIFT) & BANOS_VERSION_MAJOR_MASK)
+#define BANOS_VERSION_GET_MINOR(v) (((v) >> BANOS_VERSION_MINOR_SHIFT) & BANOS_VERSION_MINOR_MASK)
+#define BANOS_VERSION_GET_PATCH(v) (((v) >> BANOS_VERSION_PATCH_SHIFT) & BANOS_VERSION_PATCH_MASK)

kernel/include/kernel/API/SharedPage.h

@@ -12,7 +12,7 @@ namespace Kernel::API
 
 	struct SharedPage
 	{
-		uint8_t __sequence[0x100];
+		uint16_t gdt_cpu_offset;
 
 		uint32_t features;
 

kernel/include/kernel/API/Syscall.h

@@ -14,7 +14,7 @@
 	#define _kas_globbers
 #endif
 
-#define _kas_argument_var(index, value) register long _kas_a##index asm(_ban_stringify(_ban_get(index, _kas_arguments))) = (long)value;
+#define _kas_argument_var(index, value) register long _kas_a##index asm(_ban_stringify(_ban_get(index, _kas_arguments))) = (long)(value);
 #define _kas_dummy_var(index, value) register long _kas_d##index asm(#value);
 #define _kas_input(index, _) "r"(_kas_a##index)
 #define _kas_output(index, _) , "=r"(_kas_d##index)

kernel/include/kernel/Audio/Controller.h

@@ -14,7 +14,6 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
 
-		dev_t rdev() const override { return m_rdev; }
 		BAN::StringView name() const override { return m_name; }
 
 	protected:
@@ -51,7 +50,6 @@ namespace Kernel
 		snd_volume_info m_volume_info {};
 
 	private:
-		const dev_t m_rdev;
 		char m_name[10] {};
 	};
 

kernel/include/kernel/Audio/HDAudio/AudioFunctionGroup.h

@@ -43,7 +43,7 @@ namespace Kernel
 		BAN::ErrorOr<void> enable_output_path(uint8_t index);
 		BAN::ErrorOr<void> disable_output_path(uint8_t index);
 
-		void reset_stream();
+		BAN::ErrorOr<void> reset_stream();
 
 		BAN::ErrorOr<void> recurse_output_paths(const HDAudio::AFGWidget& widget, BAN::Vector<const HDAudio::AFGWidget*>& path);
 

kernel/include/kernel/Banos.h

@@ -0,0 +1,10 @@
+#pragma once
+#include <BAN/Vector.h>
+#include <BAN/StringView.h>
+typedef struct Banos_Symbol Banos_Symbol;
+namespace Banos {
+	void* resolve_symbol(const char* name);
+	void import_symbols(Banos_Symbol* symbols, size_t count);
+	void initialize_initial_drivers(void);
+	BAN::ErrorOr<size_t> load_driver_from_image(const char* u_image);
+}

kernel/include/kernel/Debug.h

@@ -49,6 +49,8 @@
 
 #define DEBUG_VTTY 1
 
+#define DEBUG_DEVFS 0
+
 #define DEBUG_PCI 0
 #define DEBUG_SCHEDULER 0
 #define DEBUG_PS2 1

kernel/include/kernel/Device/DebugDevice.h

@@ -8,15 +8,14 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<DebugDevice>> create(mode_t, uid_t, gid_t);
 
-		virtual dev_t rdev() const override { return m_rdev; }
-
 		virtual BAN::StringView name() const override { return "debug"_sv; }
 
 	protected:
 		DebugDevice(mode_t mode, uid_t uid, gid_t gid, dev_t rdev)
 			: CharacterDevice(mode, uid, gid)
-			, m_rdev(rdev)
+		{
-		{ }
+			m_rdev = rdev;
+		}
 
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override { return 0; }
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan buffer) override;
@@ -25,9 +24,6 @@ namespace Kernel
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }
 		virtual bool has_hungup_impl() const override { return false; }
-
-	private:
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Device/Device.h

@@ -12,24 +12,20 @@ namespace Kernel
 		virtual ~Device() = default;
 		virtual void update() {}
 
-		virtual bool is_device() const override { return true; }
-		virtual bool is_partition() const { return false; }
-		virtual bool is_storage_device() const { return false; }
-
 		virtual BAN::ErrorOr<BAN::UniqPtr<MemoryRegion>> mmap_region(PageTable&, off_t offset, size_t len, AddressRange, MemoryRegion::Type, PageTable::flags_t, int status_flags)
 		{
 			(void)offset; (void)len; (void)status_flags;
 			return BAN::Error::from_errno(ENOTSUP);
 		}
 
-		virtual dev_t rdev() const override = 0;
-
 		virtual BAN::StringView name() const = 0;
 
 	protected:
 		Device(mode_t, uid_t, gid_t);
 
-		virtual BAN::ErrorOr<void> fsync_impl() final override { return BAN::Error::from_errno(EINVAL); }
+	private:
+		BAN::ErrorOr<void> sync_inode(SyncType) final override { return {}; }
+		BAN::ErrorOr<void> sync_data()          final override { return {}; }
 	};
 
 	class BlockDevice : public Device, public BAN::Weakable<BlockDevice>
@@ -45,7 +41,7 @@ namespace Kernel
 		BlockDevice(mode_t mode, uid_t uid, gid_t gid)
 			: Device(mode, uid, gid)
 		{
-			m_inode_info.mode |= Inode::Mode::IFBLK;
+			m_mode |= Inode::Mode::IFBLK;
 		}
 	};
 
@@ -55,7 +51,7 @@ namespace Kernel
 		CharacterDevice(mode_t mode, uid_t uid, gid_t gid)
 			: Device(mode, uid, gid)
 		{
-			m_inode_info.mode |= Inode::Mode::IFCHR;
+			m_mode |= Inode::Mode::IFCHR;
 		}
 	};
 

kernel/include/kernel/Device/FramebufferDevice.h

@@ -30,7 +30,6 @@ namespace Kernel
 
 		virtual BAN::ErrorOr<BAN::UniqPtr<MemoryRegion>> mmap_region(PageTable&, off_t offset, size_t len, AddressRange, MemoryRegion::Type, PageTable::flags_t, int status_flags) override;
 
-		virtual dev_t rdev() const override { return m_rdev; }
 		virtual BAN::StringView name() const override { return m_name.sv(); }
 
 	protected:
@@ -50,7 +49,6 @@ namespace Kernel
 
 	private:
 		const BAN::String m_name;
-		const dev_t m_rdev;
 
 		vaddr_t			m_video_memory_vaddr { 0 };
 		const paddr_t	m_video_memory_paddr;

kernel/include/kernel/Device/NullDevice.h

@@ -10,15 +10,14 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<NullDevice>> create(mode_t, uid_t, gid_t);
 
-		virtual dev_t rdev() const override { return m_rdev; }
-
 		virtual BAN::StringView name() const override { return "null"_sv; }
 
 	protected:
 		NullDevice(mode_t mode, uid_t uid, gid_t gid, dev_t rdev)
 			: CharacterDevice(mode, uid, gid)
-			, m_rdev(rdev)
+		{
-		{ }
+			m_rdev = rdev;
+		}
 
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override { return 0; }
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan buffer) override { return buffer.size(); };
@@ -28,8 +27,6 @@ namespace Kernel
 		virtual bool has_error_impl() const override { return false; }
 		virtual bool has_hungup_impl() const override { return false; }
 
-	private:
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Device/RandomDevice.h

@@ -8,15 +8,14 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<RandomDevice>> create(mode_t, uid_t, gid_t);
 
-		virtual dev_t rdev() const override { return m_rdev; }
-
 		virtual BAN::StringView name() const override { return "random"_sv; }
 
 	protected:
 		RandomDevice(mode_t mode, uid_t uid, gid_t gid, dev_t rdev)
 			: CharacterDevice(mode, uid, gid)
-			, m_rdev(rdev)
+		{
-		{ }
+			m_rdev = rdev;
+		}
 
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan buffer) override { return buffer.size(); };
@@ -25,9 +24,6 @@ namespace Kernel
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
 		virtual bool has_hungup_impl() const override { return false; }
-
-	private:
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Device/ZeroDevice.h

@@ -8,15 +8,14 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ZeroDevice>> create(mode_t, uid_t, gid_t);
 
-		virtual dev_t rdev() const override { return m_rdev; }
-
 		virtual BAN::StringView name() const override { return "zero"_sv; }
 
 	protected:
 		ZeroDevice(mode_t mode, uid_t uid, gid_t gid, dev_t rdev)
 			: CharacterDevice(mode, uid, gid)
-			, m_rdev(rdev)
+		{
-		{ }
+			m_rdev = rdev;
+		}
 
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan buffer) override { return buffer.size(); };
@@ -25,9 +24,6 @@ namespace Kernel
 		virtual bool can_write_impl() const override { return false; }
 		virtual bool has_error_impl() const override { return false; }
 		virtual bool has_hungup_impl() const override { return false; }
-
-	private:
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Epoll.h

@@ -1,12 +1,9 @@
 #pragma once
 
-#include <BAN/Array.h>
-#include <BAN/CircularQueue.h>
 #include <BAN/HashMap.h>
-#include <BAN/HashSet.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/Mutex.h>
 
-#include <limits.h>
 #include <sys/epoll.h>
 
 namespace Kernel
@@ -24,24 +21,7 @@ namespace Kernel
 		void notify(BAN::RefPtr<Inode> inode, uint32_t event);
 
 	private:
-		Epoll() = default;
+		Epoll();
-
-	public:
-		ino_t ino() const override         { return 0; }
-		Mode mode() const override         { return { Mode::IRUSR | Mode::IWUSR }; }
-		nlink_t nlink() const override     { return 0; }
-		uid_t uid() const override         { return 0; }
-		gid_t gid() const override         { return 0; }
-		off_t size() const override        { return 0; }
-		timespec atime() const override    { return {}; }
-		timespec mtime() const override    { return {}; }
-		timespec ctime() const override    { return {}; }
-		blksize_t blksize() const override { return PAGE_SIZE; }
-		blkcnt_t blocks() const override   { return 0; }
-		dev_t dev() const override         { return 0; }
-		dev_t rdev() const override        { return 0; }
-
-		bool is_epoll() const override { return true; }
 
 		const FileSystem* filesystem() const override { return nullptr; }
 
@@ -50,60 +30,57 @@ namespace Kernel
 		bool has_error_impl() const override { return false; }
 		bool has_hungup_impl() const override { return false; }
 
-		BAN::ErrorOr<void> fsync_impl() override { return {}; }
+		BAN::ErrorOr<void> sync_inode(SyncType) override { return {}; }
+		BAN::ErrorOr<void> sync_data() override { return {}; }
 
 	private:
-		struct InodeRefPtrHash
-		{
-			BAN::hash_t operator()(const BAN::RefPtr<Inode>& inode)
-			{
-				return BAN::hash<const Inode*>()(inode.ptr());
-			}
-		};
-
 		struct ListenEventList
 		{
-			BAN::Array<epoll_event, OPEN_MAX> events;
+			ListenEventList() = default;
-			uint32_t bitmap[(OPEN_MAX + 31) / 32] {};
+
+			ListenEventList(const ListenEventList&) = delete;
+			ListenEventList& operator=(const ListenEventList&) = delete;
+
+			ListenEventList(ListenEventList&& other)
+				: events(BAN::move(other.events))
+			{}
+			ListenEventList& operator=(ListenEventList&& other)
+			{
+				events = BAN::move(other.events);
+				return *this;
+			}
+
+			BAN::HashMap<int, epoll_event> events;
 
 			bool has_fd(int fd) const
 			{
-				// For some reason having (fd < 0 || ...) makes GCC 15.1.0
+				return events.contains(fd);
-				// think bitmap access can be out of bounds...
-				if (static_cast<size_t>(fd) >= events.size())
-					return false;
-				return bitmap[fd / 32] & (1u << (fd % 32));
 			}
 
 			bool empty() const
 			{
-				for (auto val : bitmap)
+				return events.empty();
-					if (val != 0)
-						return false;
-				return true;
 			}
 
-			void add_fd(int fd, epoll_event event)
+			BAN::ErrorOr<void> add_fd(int fd, epoll_event event)
 			{
-				ASSERT(!has_fd(fd));
+				TRY(events.insert(fd, event));
-				bitmap[fd / 32] |= (1u << (fd % 32));
+				return {};
-				events[fd] = event;
 			}
 
 			void remove_fd(int fd)
 			{
-				ASSERT(has_fd(fd));
+				events.remove(fd);
-				bitmap[fd / 32] &= ~(1u << (fd % 32));
-				events[fd] = {};
 			}
 		};
 
 	private:
+		Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 		SpinLock m_ready_lock;
-		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t,        InodeRefPtrHash> m_ready_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_ready_events;
-		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t,        InodeRefPtrHash> m_processing_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, uint32_t> m_processing_events;
-		BAN::HashMap<BAN::RefPtr<Inode>, ListenEventList, InodeRefPtrHash> m_listening_events;
+		BAN::HashMap<BAN::RefPtr<Inode>, ListenEventList> m_listening_events;
 	};
 
 }

kernel/include/kernel/FS/EventFD.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/Mutex.h>
 
 namespace Kernel
 {
@@ -10,42 +11,27 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<Inode>> create(uint64_t initval, bool semaphore);
 
-		ino_t ino() const override { return 0; }
+	private:
-		Mode mode() const override { return { Mode::IFCHR | Mode::IRUSR | Mode::IWUSR }; }
+		EventFD(uint64_t initval, bool is_semaphore);
-		nlink_t nlink() const override { return ref_count(); }
-		uid_t uid() const override { return 0; }
-		gid_t gid() const override { return 0; }
-		off_t size() const override { return 0; }
-		timespec atime() const override { return {}; }
-		timespec mtime() const override { return {}; }
-		timespec ctime() const override { return {}; }
-		blksize_t blksize() const override { return 8; }
-		blkcnt_t blocks() const override { return 0; }
-		dev_t dev() const override { return 0; }
-		dev_t rdev() const override { return 0; }
 
 		const FileSystem* filesystem() const override { return nullptr; }
 
-	protected:
+		BAN::ErrorOr<void> sync_inode(SyncType) override { return {}; }
+		BAN::ErrorOr<void> sync_data() override { return {}; }
+
 		BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
-		BAN::ErrorOr<void> fsync_impl() final override { return {}; }
 
 		bool can_read_impl() const override { return m_value > 0; }
 		bool can_write_impl() const override { return m_value < UINT64_MAX - 1; }
 		bool has_error_impl() const override { return false; }
 		bool has_hungup_impl() const override { return false; }
 
-	private:
-		EventFD(uint64_t initval, bool is_semaphore)
-			: m_is_semaphore(is_semaphore)
-			, m_value(initval)
-		{ }
-
 	private:
 		const bool m_is_semaphore;
-		uint64_t m_value;
+		BAN::Atomic<uint64_t> m_value;
 
+		Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 	};
 

kernel/include/kernel/FS/Ext2/Definitions.h

@@ -79,6 +79,12 @@ namespace Kernel::Ext2
 		uint8_t __reserved[12];
 	};
 
+	struct InodeBlocks {
+		uint32_t block[15];
+	};
+	struct Osd2 {
+		uint32_t osd2[3];
+	};
 	struct Inode
 	{
 		uint16_t mode;
@@ -93,12 +99,12 @@ namespace Kernel::Ext2
 		uint32_t blocks;
 		uint32_t flags;
 		uint32_t osd1;
-		uint32_t block[15];
+		InodeBlocks block;
 		uint32_t generation;
 		uint32_t file_acl;
 		uint32_t dir_acl;
 		uint32_t faddr;
-		uint32_t osd2[3];
+		Osd2 osd2;
 	};
 
 	struct LinkedDirectoryEntry

kernel/include/kernel/FS/Ext2/FileSystem.h

@@ -99,7 +99,8 @@ namespace Kernel
 		BAN::ErrorOr<uint32_t> reserve_free_block(uint32_t primary_bgd);
 		BAN::ErrorOr<void> release_block(uint32_t block);
 
-		BAN::HashMap<ino_t, BAN::RefPtr<Ext2Inode>>& inode_cache() { return m_inode_cache; }
+		BAN::ErrorOr<BAN::RefPtr<Ext2Inode>> open_inode(ino_t);
+		void remove_from_cache(ino_t);
 
 		const Ext2::Superblock& superblock() const { return m_superblock; }
 
@@ -155,6 +156,7 @@ namespace Kernel
 		BAN::RefPtr<Inode> m_root_inode;
 		BAN::Vector<uint32_t> m_superblock_backups;
 
+		Mutex m_inode_cache_lock;
 		BAN::HashMap<ino_t, BAN::RefPtr<Ext2Inode>> m_inode_cache;
 
 		BlockBufferManager m_buffer_manager;

kernel/include/kernel/FS/Ext2/Inode.h

@@ -4,6 +4,7 @@
 #include <BAN/StringView.h>
 #include <kernel/FS/Ext2/Definitions.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/RWLock.h>
 
 namespace Kernel
 {
@@ -15,23 +16,12 @@ namespace Kernel
 	public:
 		~Ext2Inode();
 
-		virtual ino_t ino() const override { return m_ino; };
-		virtual Mode mode() const override { return { m_inode.mode }; }
-		virtual nlink_t nlink() const override { return m_inode.links_count; }
-		virtual uid_t uid() const override { return m_inode.uid; }
-		virtual gid_t gid() const override { return m_inode.gid; }
-		virtual off_t size() const override { return m_inode.size; }
-		virtual timespec atime() const override { return timespec { .tv_sec = m_inode.atime, .tv_nsec = 0 }; }
-		virtual timespec mtime() const override { return timespec { .tv_sec = m_inode.mtime, .tv_nsec = 0 }; }
-		virtual timespec ctime() const override { return timespec { .tv_sec = m_inode.ctime, .tv_nsec = 0 }; }
-		virtual blksize_t blksize() const override;
-		virtual blkcnt_t blocks() const override;
-		virtual dev_t dev() const override { return 0; }
-		virtual dev_t rdev() const override { return 0; }
-
 		virtual const FileSystem* filesystem() const override;
 
-	protected:
+	private:
+		virtual BAN::ErrorOr<void> sync_inode(SyncType) override;
+		virtual BAN::ErrorOr<void> sync_data() override;
+
 		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView) override;
 		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t) override;
 		virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t) override;
@@ -46,10 +36,6 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
-		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
-		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t) override;
-		virtual BAN::ErrorOr<void> utimens_impl(const timespec[2]) override;
-		virtual BAN::ErrorOr<void> fsync_impl() override;
 
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
@@ -57,58 +43,96 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
+		uint32_t block_group() const;
+
 		// Returns maximum number of data blocks in use
 		// NOTE: the inode might have more blocks than what this suggests if it has been shrinked
 		uint32_t max_used_data_block_count() const { return size() / blksize(); }
 
-		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block(uint32_t& block, uint32_t index, uint32_t depth, bool allocate);
+		BAN::ErrorOr<void> sync_inode_no_lock();
-		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index(uint32_t data_block_index, bool allocate);
 
-		BAN::ErrorOr<void> link_inode_to_directory(Ext2Inode&, BAN::StringView name);
+		BAN::ErrorOr<bool> is_directory_empty_no_lock();
-		BAN::ErrorOr<void> remove_inode_from_directory(BAN::StringView name, bool cleanup_directory);
+		BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_no_lock(BAN::StringView);
-		BAN::ErrorOr<bool> is_directory_empty();
 
-		BAN::ErrorOr<void> cleanup_indirect_block(uint32_t block, uint32_t depth);
+		/* needs write end of the lock when allocate is true*/
-		BAN::ErrorOr<void> cleanup_default_links();
+		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block_no_lock(uint32_t block, uint32_t index, uint32_t depth, bool allocate);
-		BAN::ErrorOr<void> cleanup_data_blocks();
+		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index_no_lock(uint32_t data_block_index, bool allocate);
-		BAN::ErrorOr<void> cleanup_from_fs();
 
-		BAN::ErrorOr<void> sync();
+		/* needs write end of the lock */
+		BAN::ErrorOr<void> link_inode_to_directory_no_lock(Ext2Inode&, BAN::StringView name);
+		BAN::ErrorOr<void> remove_inode_from_directory_no_lock(BAN::StringView name, bool cleanup_directory);
 
-		uint32_t block_group() const;
+		/* needs write end of the lock */
+		BAN::ErrorOr<void> cleanup_indirect_block_no_lock(uint32_t block, uint32_t depth);
+		BAN::ErrorOr<void> cleanup_default_links_no_lock();
+		BAN::ErrorOr<void> cleanup_data_blocks_no_lock();
+		BAN::ErrorOr<void> cleanup_from_fs_no_lock();
 
 	private:
-		Ext2Inode(Ext2FS& fs, Ext2::Inode inode, uint32_t ino)
+		Ext2Inode(Ext2FS& fs, Ext2::Inode inode, uint32_t ino);
-			: m_fs(fs)
+
-			, m_inode(inode)
+		BAN::Optional<uint32_t> block_cache_find(uint32_t block, uint32_t index) const;
-			, m_ino(ino)
+		void block_cache_remove(uint32_t block, uint32_t index);
-		{}
+		void block_cache_add(uint32_t block, uint32_t index, uint32_t target);
-		static BAN::ErrorOr<BAN::RefPtr<Ext2Inode>> create(Ext2FS&, uint32_t);
+
+		BAN::RefPtr<Inode> dir_cache_find(BAN::StringView) const;
+		void dir_cache_remove(BAN::StringView);
+		void dir_cache_add(BAN::StringView, BAN::RefPtr<Inode>);
 
 	private:
 		struct ScopedSync
 		{
 			ScopedSync(Ext2Inode& inode)
 				: inode(inode)
-				, inode_info(inode.m_inode)
 			{ }
 
 			~ScopedSync()
 			{
-				if (memcmp(&inode.m_inode, &inode_info, sizeof(Ext2::Inode)) == 0)
+				// TODO: there was some memcmp smarty pants stuff here.
-					return;
+				// How do we wanna approach it?
-				if (auto ret = inode.sync(); ret.is_error())
+				if (auto ret = inode.sync_inode_no_lock(); ret.is_error())
 					dwarnln("failed to sync inode: {}", ret.error());
 			}
 
 			Ext2Inode& inode;
-			Ext2::Inode inode_info;
 		};
 
 	private:
 		Ext2FS& m_fs;
-		Ext2::Inode m_inode;
+		RWLock m_lock;
-		const uint32_t m_ino;
+
+		Ext2::InodeBlocks m_ext2_blocks;
+		// NOTE: some fields from the original disk inode
+		// that we do not use, but we keep for serialise.
+		const uint32_t m_og_dtime;
+		const uint32_t m_og_flags;
+		const uint32_t m_og_osd1;
+		const uint32_t m_og_generation;
+		const uint32_t m_og_file_acl;
+		const uint32_t m_og_dir_acl;
+		const uint32_t m_og_faddr;
+		const Ext2::Osd2 m_og_osd2;
+
+		struct BlockCacheEntry
+		{
+			mutable uint32_t freq;
+			uint32_t block;
+			uint32_t index;
+			uint32_t target;
+		};
+		mutable SpinLock m_block_cache_lock;
+		BAN::Array<BlockCacheEntry, 8> m_block_cache;
+
+		struct DirCacheEntry
+		{
+			mutable size_t freq { 0 };
+			BAN::RefPtr<Inode> inode;
+			size_t name_len { 0 };
+			char   name[256];
+		};
+		static constexpr size_t dir_cache_size = 32;
+		mutable RWLock m_dir_cache_lock;
+		BAN::Vector<DirCacheEntry> m_dir_cache;
 
 		friend class Ext2FS;
 		friend class BAN::RefPtr<Ext2Inode>;

kernel/include/kernel/FS/FAT/Inode.h

@@ -15,25 +15,14 @@ namespace Kernel
 	class FATInode final : public Inode, public BAN::Weakable<FATInode>
 	{
 	public:
-		virtual ino_t ino() const override { return m_ino; };
-		virtual Mode mode() const override { return Mode { ((m_entry.attr & FAT::FileAttr::DIRECTORY) ? Mode::IFDIR : Mode::IFREG) | 0777 }; }
-		virtual nlink_t nlink() const override { return 1; }
-		virtual uid_t uid() const override { return 0; }
-		virtual gid_t gid() const override { return 0; }
-		virtual off_t size() const override { return m_entry.file_size; }
-		virtual timespec atime() const override;
-		virtual timespec mtime() const override;
-		virtual timespec ctime() const override;
-		virtual blksize_t blksize() const override;
-		virtual blkcnt_t blocks() const override { return m_block_count; }
-		virtual dev_t dev() const override { return 0; }
-		virtual dev_t rdev() const override { return 0; }
-
 		virtual const FileSystem* filesystem() const override;
 
 		const FAT::DirectoryEntry& entry() const { return m_entry; }
 
-	protected:
+	private:
+		virtual BAN::ErrorOr<void> sync_inode(SyncType) override { return {}; }
+		virtual BAN::ErrorOr<void> sync_data() override { return {}; }
+
 		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView) override;
 		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t) override;
 		//virtual BAN::ErrorOr<void> create_file_impl(BAN::StringView, mode_t, uid_t, gid_t) override;
@@ -43,9 +32,6 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		//virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		//virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
-		//virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
-		//virtual BAN::ErrorOr<void> utimens_impl(const timespec[2]) override;
-		virtual BAN::ErrorOr<void> fsync_impl() override { return {}; }
 
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
@@ -53,12 +39,8 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
-		FATInode(FATFS& fs, const FAT::DirectoryEntry& entry, ino_t ino, uint32_t block_count)
+		FATInode(FATFS& fs, const FAT::DirectoryEntry& entry, ino_t ino, uint32_t block_count);
-			: m_fs(fs)
+
-			, m_entry(entry)
-			, m_ino(ino)
-			, m_block_count(block_count)
-		{ }
 		~FATInode()	{}
 
 		BAN::ErrorOr<void> for_each_directory_entry(BAN::ConstByteSpan, BAN::Function<BAN::Iteration(const FAT::DirectoryEntry&)>);
@@ -67,7 +49,6 @@ namespace Kernel
 	private:
 		FATFS& m_fs;
 		FAT::DirectoryEntry m_entry;
-		const ino_t m_ino;
 		uint32_t m_block_count;
 
 		friend class Ext2FS;

kernel/include/kernel/FS/Inode.h

@@ -10,7 +10,6 @@
 
 #include <kernel/Credentials.h>
 #include <kernel/Debug.h>
-#include <kernel/Lock/Mutex.h>
 
 #include <dirent.h>
 #include <sys/socket.h>
@@ -63,6 +62,24 @@ namespace Kernel
 			mode_t mode;
 		};
 
+		enum InodeKind : uint8_t
+		{
+			DEVICE    = 0x01,
+			EPOLL     = 0x02,
+			PIPE      = 0x04,
+			TTY       = 0x08,
+			PARTITION = 0x10,
+			STORAGE   = 0x20,
+		};
+
+		enum class SyncType
+		{
+			General,
+			Mode,
+			UidGid,
+			Times,
+		};
+
 	public:
 		virtual ~Inode() {}
 
@@ -70,24 +87,26 @@ namespace Kernel
 
 		bool operator==(const Inode& other) const { return dev() == other.dev() && ino() == other.ino(); }
 
-		virtual ino_t ino() const = 0;
+		ino_t ino() const { return m_ino; }
-		virtual Mode mode() const = 0;
+		Mode mode() const { return Mode(m_mode); }
-		virtual nlink_t nlink() const = 0;
+		nlink_t nlink() const { return m_nlink; }
-		virtual uid_t uid() const = 0;
+		uid_t uid() const { return m_uid; }
-		virtual gid_t gid() const = 0;
+		gid_t gid() const { return m_gid; }
-		virtual off_t size() const = 0;
+		off_t size() const { return m_size; }
-		virtual timespec atime() const = 0;
+		timespec atime() const { return m_atime; }
-		virtual timespec mtime() const = 0;
+		timespec mtime() const { return m_mtime; }
-		virtual timespec ctime() const = 0;
+		timespec ctime() const { return m_ctime; }
-		virtual blksize_t blksize() const = 0;
+		blksize_t blksize() const { return m_blksize; }
-		virtual blkcnt_t blocks() const = 0;
+		blkcnt_t blocks() const { return m_blocks; }
-		virtual dev_t dev() const = 0;
+		dev_t dev() const { return m_dev; }
-		virtual dev_t rdev() const = 0;
+		dev_t rdev() const { return m_rdev; }
 
-		virtual bool is_device() const { return false; }
+		bool is_device()         const { return m_kind & InodeKind::DEVICE;    }
-		virtual bool is_epoll() const { return false; }
+		bool is_epoll()          const { return m_kind & InodeKind::EPOLL;     }
-		virtual bool is_pipe() const { return false; }
+		bool is_pipe()           const { return m_kind & InodeKind::PIPE;      }
-		virtual bool is_tty() const { return false; }
+		bool is_tty()            const { return m_kind & InodeKind::TTY;       }
+		bool is_partition()      const { return m_kind & InodeKind::PARTITION; }
+		bool is_storage_device() const { return m_kind & InodeKind::STORAGE;   }
 
 		virtual const FileSystem* filesystem() const = 0;
 
@@ -126,10 +145,10 @@ namespace Kernel
 		BAN::ErrorOr<void> fsync();
 
 		// Select/Non blocking API
-		bool can_read() const;
+		bool can_read() const   { return can_read_impl(); }
-		bool can_write() const;
+		bool can_write() const  { return can_write_impl(); }
-		bool has_error() const;
+		bool has_error() const  { return has_error_impl(); }
-		bool has_hungup() const;
+		bool has_hungup() const { return has_hungup_impl(); }
 
 		BAN::ErrorOr<long> ioctl(int request, void* arg);
 
@@ -140,6 +159,9 @@ namespace Kernel
 		virtual void on_close(int status_flags) { (void)status_flags; }
 		virtual void on_clone(int status_flags) { (void)status_flags; }
 
+		virtual BAN::ErrorOr<void> sync_inode(SyncType) = 0;
+		virtual BAN::ErrorOr<void> sync_data() = 0;
+
 	protected:
 		// Directory API
 		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView)							{ return BAN::Error::from_errno(ENOTSUP); }
@@ -170,10 +192,6 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan)		{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan)	{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> truncate_impl(size_t)					{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> chmod_impl(mode_t)						{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t)					{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> utimens_impl(const timespec[2])			{ return BAN::Error::from_errno(ENOTSUP); }
-		virtual BAN::ErrorOr<void> fsync_impl() = 0;
 
 		// Select/Non blocking API
 		virtual bool can_read_impl() const = 0;
@@ -184,12 +202,33 @@ namespace Kernel
 		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) { return BAN::Error::from_errno(ENOTSUP); }
 
 	protected:
-		mutable PriorityMutex m_mutex;
+		// TODO: this is supposed to be const I guess?
+		// But the thing is I would have to refactor a big chunk of the codebase
+		// to add it as a parameter to Inode() soooooo yeah no, not doing that rn.
+		uint8_t m_kind = 0;
+
+		BAN::Atomic<ino_t>     m_ino;
+		BAN::Atomic<mode_t>    m_mode;
+		BAN::Atomic<nlink_t>   m_nlink;
+		BAN::Atomic<uid_t>     m_uid;
+		BAN::Atomic<gid_t>     m_gid;
+		BAN::Atomic<off_t>     m_size;
+		// TODO: make these guys atomic :)
+		timespec m_atime;
+		timespec m_mtime;
+		timespec m_ctime;
+		BAN::Atomic<blksize_t> m_blksize;
+		BAN::Atomic<blkcnt_t>  m_blocks;
+		BAN::Atomic<dev_t>     m_dev;
+		BAN::Atomic<dev_t>     m_rdev;
 
 	private:
+		SpinLock m_shared_region_lock;
 		BAN::WeakPtr<SharedFileData> m_shared_region;
+
 		SpinLock m_epoll_lock;
 		BAN::LinkedList<class Epoll*> m_epolls;
+
 		friend class Epoll;
 		friend class FileBackedRegion;
 		friend class OpenFileDescriptorSet;

kernel/include/kernel/FS/Pipe.h

@@ -1,43 +1,34 @@
 #pragma once
 
-#include <BAN/Array.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/Mutex.h>
 #include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/ThreadBlocker.h>
 
+#include <sys/stat.h>
+
 namespace Kernel
 {
 
-	class Pipe : public Inode
+	class Pipe final : public Inode, public BAN::Weakable<Pipe>
 	{
 	public:
-		static BAN::ErrorOr<BAN::RefPtr<Inode>> create(const Credentials&);
+		static BAN::ErrorOr<BAN::RefPtr<Inode>> open(BAN::RefPtr<Inode>, int status_flags);
-
+		static BAN::ErrorOr<BAN::RefPtr<Inode>> create(uid_t, gid_t);
-		virtual bool is_pipe() const override { return true; }
+		~Pipe();
 
 		void on_close(int status_flags) override;
 		void on_clone(int status_flags) override;
 
-		virtual ino_t ino() const override { return 0; } // FIXME
-		virtual Mode mode() const override { return { Mode::IFIFO | Mode::IRUSR | Mode::IWUSR }; }
-		virtual nlink_t nlink() const override { return 1; }
-		virtual uid_t uid() const override { return m_uid; }
-		virtual gid_t gid() const override { return m_gid; }
-		virtual off_t size() const override { return 0; }
-		virtual timespec atime() const override { return m_atime; }
-		virtual timespec mtime() const override { return m_mtime; }
-		virtual timespec ctime() const override { return m_ctime; }
-		virtual blksize_t blksize() const override { return 4096; }
-		virtual blkcnt_t blocks() const override { return 0; }
-		virtual dev_t dev() const override { return 0; } // FIXME
-		virtual dev_t rdev() const override { return 0; } // FIXME
-
 		virtual const FileSystem* filesystem() const override { return nullptr; }
 
-	protected:
+	private:
+		virtual BAN::ErrorOr<void> sync_inode(SyncType) override;
+		virtual BAN::ErrorOr<void> sync_data() override;
+
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
-		virtual BAN::ErrorOr<void> fsync_impl() final override { return {}; }
+		virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
 
 		virtual bool can_read_impl() const override { return !m_buffer->empty(); }
 		virtual bool can_write_impl() const override { return true; }
@@ -45,20 +36,18 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return m_writing_count == 0; }
 
 	private:
-		Pipe(const Credentials&);
+		Pipe(const struct stat&);
 
 	private:
-		const uid_t m_uid;
+		Mutex m_mutex;
-		const gid_t m_gid;
-		timespec m_atime {};
-		timespec m_mtime {};
-		timespec m_ctime {};
 		ThreadBlocker m_thread_blocker;
 
 		BAN::UniqPtr<ByteRingBuffer> m_buffer;
 
-		BAN::Atomic<uint32_t> m_writing_count { 1 };
+		BAN::Atomic<uint32_t> m_writing_count { 0 };
-		BAN::Atomic<uint32_t> m_reading_count { 1 };
+		BAN::Atomic<uint32_t> m_reading_count { 0 };
+
+		BAN::RefPtr<Inode> m_named_inode;
 	};
 
 }

kernel/include/kernel/FS/ProcFS/FileSystem.h

@@ -13,6 +13,8 @@ namespace Kernel
 		static void initialize();
 		static ProcFileSystem& get();
 
+		void post_scheduler_initialize();
+
 		BAN::ErrorOr<void> on_process_create(Process&);
 		void on_process_delete(Process&);
 

kernel/include/kernel/FS/ProcFS/Inode.h

@@ -9,34 +9,27 @@ namespace Kernel
 
 	class ProcPidInode final : public TmpDirectoryInode
 	{
+		// FIXME: dynamically update ruid/rgid.
+		// Possibly just have a magic uid/gid of -1 or something
+		// which means use current process ID
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ProcPidInode>> create_new(Process&, TmpFileSystem&, mode_t);
 		~ProcPidInode() = default;
 
-		virtual uid_t uid() const override { return m_process.credentials().ruid(); }
-		virtual gid_t gid() const override { return m_process.credentials().rgid(); }
-
 		void cleanup();
-
 	protected:
 		virtual BAN::ErrorOr<void> unlink_impl(BAN::StringView) override { return BAN::Error::from_errno(EPERM); }
 
 	private:
 		ProcPidInode(Process&, TmpFileSystem&, const TmpInodeInfo&);
-
-	private:
-		Process& m_process;
 	};
 
 	class ProcROProcessInode final : public TmpInode
 	{
+		//FIXME: dynamically update ruid/rgid
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ProcROProcessInode>> create_new(Process&, size_t (Process::*callback)(off_t, BAN::ByteSpan) const, TmpFileSystem&, mode_t);
 		~ProcROProcessInode() = default;
-
-		virtual uid_t uid() const override { return m_process.credentials().ruid(); }
-		virtual gid_t gid() const override { return m_process.credentials().rgid(); }
-
 	protected:
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 
@@ -59,13 +52,11 @@ namespace Kernel
 
 	class ProcSymlinkProcessInode final : public TmpInode
 	{
+		//FIXME: dynamically update ruid/rgid
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ProcSymlinkProcessInode>> create_new(Process& process, BAN::ErrorOr<BAN::String> (Process::*callback)() const, TmpFileSystem&, mode_t);
 		~ProcSymlinkProcessInode() = default;
 
-		virtual uid_t uid() const override { return m_process.credentials().ruid(); }
-		virtual gid_t gid() const override { return m_process.credentials().rgid(); }
-
 	protected:
 		virtual BAN::ErrorOr<BAN::String> link_target_impl() override;
 
@@ -85,7 +76,7 @@ namespace Kernel
 	class ProcROInode final : public TmpInode
 	{
 	public:
-		static BAN::ErrorOr<BAN::RefPtr<ProcROInode>> create_new(size_t (*callback)(off_t, BAN::ByteSpan), TmpFileSystem&, mode_t, uid_t, gid_t);
+		static BAN::ErrorOr<BAN::RefPtr<ProcROInode>> create_new(BAN::ErrorOr<size_t> (*callback)(off_t, BAN::ByteSpan, void*), TmpFileSystem&, void*, mode_t, uid_t, gid_t);
 		~ProcROInode() = default;
 
 	protected:
@@ -101,10 +92,11 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
-		ProcROInode(size_t (*callback)(off_t, BAN::ByteSpan), TmpFileSystem&, const TmpInodeInfo&);
+		ProcROInode(BAN::ErrorOr<size_t> (*callback)(off_t, BAN::ByteSpan, void*), TmpFileSystem&, void*, const TmpInodeInfo&);
 
 	private:
-		size_t (*m_callback)(off_t, BAN::ByteSpan);
+		BAN::ErrorOr<size_t> (*m_callback)(off_t, BAN::ByteSpan, void*);
+		void* m_argument;
 	};
 
 	class ProcSymlinkInode final : public TmpInode
@@ -132,13 +124,10 @@ namespace Kernel
 
 	class ProcFDDirectoryInode final : public TmpInode
 	{
+		//FIXME: dynamically update ruid/rgid
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<ProcFDDirectoryInode>> create_new(Process&, TmpFileSystem&, mode_t);
 		~ProcFDDirectoryInode() = default;
-
-		virtual uid_t uid() const override { return m_process.credentials().ruid(); }
-		virtual gid_t gid() const override { return m_process.credentials().rgid(); }
-
 	protected:
 		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView) override;
 		virtual BAN::ErrorOr<size_t> list_next_inodes_impl(off_t, struct dirent*, size_t) override;

kernel/include/kernel/FS/Socket.h

@@ -30,31 +30,19 @@ namespace Kernel
 		};
 
 	public:
-		ino_t		ino()		const final override { ASSERT_NOT_REACHED(); }
-		Mode		mode()		const final override { return Mode(m_info.mode); }
-		nlink_t		nlink()		const final override { ASSERT_NOT_REACHED(); }
-		uid_t		uid()		const final override { return m_info.uid; }
-		gid_t		gid()		const final override { return m_info.gid; }
-		off_t		size()		const final override { ASSERT_NOT_REACHED(); }
-		timespec	atime()		const final override { ASSERT_NOT_REACHED(); }
-		timespec	mtime()		const final override { ASSERT_NOT_REACHED(); }
-		timespec	ctime()		const final override { ASSERT_NOT_REACHED(); }
-		blksize_t	blksize()	const final override { ASSERT_NOT_REACHED(); }
-		blkcnt_t	blocks()	const final override { ASSERT_NOT_REACHED(); }
-		dev_t		dev()		const final override { ASSERT_NOT_REACHED(); }
-		dev_t		rdev()		const final override { ASSERT_NOT_REACHED(); }
-
 		const FileSystem* filesystem() const final override { return nullptr; }
 
 	protected:
 		Socket(const Info& info)
-			: m_info(info)
+		{
-		{}
+			m_mode = info.mode;
-
+			m_uid = info.uid;
-		BAN::ErrorOr<void> fsync_impl() final override { return {}; }
+			m_gid = info.gid;
+		}
 
 	private:
-		const Info m_info;
+		BAN::ErrorOr<void> sync_inode(SyncType) final override { return {}; }
+		BAN::ErrorOr<void> sync_data() final override { return {}; }
 	};
 
 }

kernel/include/kernel/FS/TmpFS/Definitions.h

@@ -11,18 +11,8 @@
 namespace Kernel
 {
 
-	struct TmpInodeInfo
+	struct TmpBlocks
 	{
-		mode_t		mode	{ 0 };
-		uid_t		uid		{ 0 };
-		gid_t		gid		{ 0 };
-		timespec	atime	{ 0, 0 };
-		timespec	ctime	{ 0, 0 };
-		timespec	mtime	{ 0, 0 };
-		nlink_t		nlink	{ 0 };
-		size_t		size	{ 0 };
-		blkcnt_t	blocks	{ 0 };
-
 #if ARCH(x86_64)
 		// 2x direct blocks
 		// 1x singly indirect
@@ -41,8 +31,23 @@ namespace Kernel
 #else
 	#error
 #endif
+	};
+
+	struct TmpInodeInfo
+	{
+		mode_t		mode	{ 0 };
+		uid_t		uid		{ 0 };
+		gid_t		gid		{ 0 };
+		timespec	atime	{ 0, 0 };
+		timespec	ctime	{ 0, 0 };
+		timespec	mtime	{ 0, 0 };
+		nlink_t		nlink	{ 0 };
+		size_t		size	{ 0 };
+		blkcnt_t	blocks	{ 0 };
+		TmpBlocks   tmp_blocks;
+
 		static constexpr size_t max_size =
-			direct_block_count * PAGE_SIZE +
+			TmpBlocks::direct_block_count * PAGE_SIZE +
 			(PAGE_SIZE / sizeof(paddr_t)) * PAGE_SIZE +
 			(PAGE_SIZE / sizeof(paddr_t)) * (PAGE_SIZE / sizeof(paddr_t)) * PAGE_SIZE +
 			(PAGE_SIZE / sizeof(paddr_t)) * (PAGE_SIZE / sizeof(paddr_t)) * (PAGE_SIZE / sizeof(paddr_t)) * PAGE_SIZE;

kernel/include/kernel/FS/TmpFS/Inode.h

@@ -4,6 +4,7 @@
 #include <BAN/Optional.h>
 #include <kernel/FS/Inode.h>
 #include <kernel/FS/TmpFS/Definitions.h>
+#include <kernel/Lock/Mutex.h>
 
 namespace Kernel
 {
@@ -23,51 +24,37 @@ namespace Kernel
 
 	class TmpInode : public Inode
 	{
-	public:
-		virtual ino_t		ino()		const override { return m_ino; }
-		virtual Mode		mode()		const override { return Mode(m_inode_info.mode); }
-		virtual nlink_t		nlink()		const override { return m_inode_info.nlink; }
-		virtual uid_t		uid()		const override { return m_inode_info.uid; }
-		virtual gid_t		gid()		const override { return m_inode_info.gid; }
-		virtual off_t		size()		const override { return m_inode_info.size; }
-		virtual timespec	atime()		const override { return m_inode_info.atime; }
-		virtual timespec	mtime()		const override { return m_inode_info.mtime; }
-		virtual timespec	ctime()		const override { return m_inode_info.ctime; }
-		virtual blksize_t	blksize()	const override { return PAGE_SIZE; }
-		virtual blkcnt_t	blocks()	const override { return m_inode_info.blocks; }
-		virtual dev_t		dev()		const override;
-		virtual dev_t		rdev()		const override { return 0; }
-
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<TmpInode>> create_from_existing(TmpFileSystem&, ino_t, const TmpInodeInfo&);
-		~TmpInode();
+		virtual ~TmpInode();
 
 		virtual const FileSystem* filesystem() const override;
 
 	protected:
 		TmpInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
 
-		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
+		void write_inode_to_fs();
-		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t) override;
+		virtual BAN::ErrorOr<void> prepare_unlink_no_lock() { return {}; };
-		virtual BAN::ErrorOr<void> utimens_impl(const timespec[2]) override;
-		virtual BAN::ErrorOr<void> fsync_impl() override { return {}; }
-
-		void sync();
-		virtual BAN::ErrorOr<void> prepare_unlink() { return {}; };
 
 		void free_all_blocks();
-		void free_indirect_blocks(size_t block, uint32_t depth);
+		void free_indirect_blocks_no_lock(size_t block, uint32_t depth);
 
 		BAN::Optional<size_t> block_index(size_t data_block_index);
-		BAN::Optional<size_t> block_index_from_indirect(size_t block, size_t index, uint32_t depth);
+		BAN::Optional<size_t> block_index_from_indirect_no_lock(size_t block, size_t index, uint32_t depth);
 
 		BAN::ErrorOr<size_t> block_index_with_allocation(size_t data_block_index);
-		BAN::ErrorOr<size_t> block_index_from_indirect_with_allocation(size_t& block, size_t index, uint32_t depth);
+		BAN::ErrorOr<size_t> block_index_from_indirect_with_allocation_no_lock(size_t& block, size_t index, uint32_t depth);
+
+	private:
+		BAN::ErrorOr<void> sync_inode(SyncType) override;
+		BAN::ErrorOr<void> sync_data() override;
 
 	protected:
 		TmpFileSystem& m_fs;
-		TmpInodeInfo m_inode_info;
+		TmpBlocks m_tmp_blocks;
-		const ino_t m_ino;
+
+		// TODO: try to reduce locking or replace this with rwlock(?)
+		Mutex m_lock;
 
 		// has to be able to increase link count
 		friend class TmpDirectoryInode;
@@ -79,7 +66,7 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<TmpFileInode>> create_new(TmpFileSystem&, mode_t, uid_t, gid_t);
 		~TmpFileInode();
 
-	protected:
+	private:
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		virtual BAN::ErrorOr<void> truncate_impl(size_t) override;
@@ -95,13 +82,36 @@ namespace Kernel
 		friend class TmpInode;
 	};
 
+	// NOTE: this is just a dummy, when opening a fifo a pipe is created
+	class TmpFIFOInode : public TmpInode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<TmpFIFOInode>> create_new(TmpFileSystem&, mode_t, uid_t, gid_t);
+		~TmpFIFOInode();
+
+	private:
+		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override { return BAN::Error::from_errno(ENODEV); }
+		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override { return BAN::Error::from_errno(ENODEV); }
+		virtual BAN::ErrorOr<void> truncate_impl(size_t) override { return BAN::Error::from_errno(ENODEV); }
+
+		virtual bool can_read_impl() const override { return false; }
+		virtual bool can_write_impl() const override { return false; }
+		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
+
+	private:
+		TmpFIFOInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
+
+		friend class TmpInode;
+	};
+
 	class TmpSocketInode : public TmpInode
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<TmpSocketInode>> create_new(TmpFileSystem&, mode_t, uid_t, gid_t);
 		~TmpSocketInode();
 
-	protected:
+	private:
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override { return BAN::Error::from_errno(ENODEV); }
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override { return BAN::Error::from_errno(ENODEV); }
 		virtual BAN::ErrorOr<void> truncate_impl(size_t) override { return BAN::Error::from_errno(ENODEV); }
@@ -123,7 +133,7 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<TmpSymlinkInode>> create_new(TmpFileSystem&, mode_t, uid_t, gid_t, BAN::StringView target);
 		~TmpSymlinkInode();
 
-	protected:
+	private:
 		BAN::ErrorOr<BAN::String> link_target_impl() override;
 		BAN::ErrorOr<void> set_link_target_impl(BAN::StringView) override;
 
@@ -149,7 +159,7 @@ namespace Kernel
 	protected:
 		TmpDirectoryInode(TmpFileSystem&, ino_t, const TmpInodeInfo&);
 
-		virtual BAN::ErrorOr<void> prepare_unlink() override;
+		virtual BAN::ErrorOr<void> prepare_unlink_no_lock() override;
 
 	protected:
 		virtual BAN::ErrorOr<BAN::RefPtr<Inode>> find_inode_impl(BAN::StringView) override final;

kernel/include/kernel/FS/VirtualFileSystem.h

@@ -92,8 +92,9 @@ namespace Kernel
 		MountPoint* mount_from_root_inode(BAN::RefPtr<Inode>);
 
 	private:
-		Mutex					m_mutex;
 		BAN::RefPtr<FileSystem>	m_root_fs;
+
+		Mutex					m_mount_point_lock;
 		BAN::Vector<MountPoint>	m_mount_points;
 
 		friend class BAN::RefPtr<VirtualFileSystem>;

kernel/include/kernel/GDT.h

@@ -133,6 +133,12 @@ namespace Kernel
 		void set_gsbase(uintptr_t addr);
 #endif
 
+		static uint16_t cpu_index_offset() { return m_cpu_index_offset; }
+		void set_cpu_index(uint8_t index)
+		{
+			write_entry(m_cpu_index_offset, 0, index, 0xF2, 0x4);
+		}
+
 	private:
 		GDT() = default;
 
@@ -151,11 +157,13 @@ namespace Kernel
 
 	private:
 #if ARCH(x86_64)
-		BAN::Array<SegmentDescriptor, 8> m_gdt; // null, kernel code, kernel data, user code (32 bit), user data, user code (64 bit), tss low, tss high
+		BAN::Array<SegmentDescriptor, 9> m_gdt; // null, kernel code, kernel data, user code (32 bit), user data, user code (64 bit), cpu-index, tss low, tss high
-		static constexpr uint16_t m_tss_offset = 0x30;
+		static constexpr uint16_t m_cpu_index_offset = 0x30;
+		static constexpr uint16_t m_tss_offset = 0x38;
 #elif ARCH(i686)
-		BAN::Array<SegmentDescriptor, 9> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, tss
+		BAN::Array<SegmentDescriptor, 10> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, cpu-index, tss
-		static constexpr uint16_t m_tss_offset = 0x40;
+		static constexpr uint16_t m_cpu_index_offset = 0x40;
+		static constexpr uint16_t m_tss_offset = 0x48;
 #endif
 		TaskStateSegment m_tss;
 		const GDTR m_gdtr {

kernel/include/kernel/Input/InputDevice.h

@@ -22,8 +22,6 @@ namespace Kernel
 		InputDevice(Type type);
 
 		BAN::StringView name() const final override { return m_name; }
-		dev_t rdev() const final override { return m_rdev; }
-
 	protected:
 		void add_event(BAN::ConstByteSpan);
 
@@ -38,8 +36,7 @@ namespace Kernel
 		BAN::ErrorOr<size_t> read_non_block(BAN::ByteSpan);
 
 	private:
-		const dev_t m_rdev;
+		BAN::String m_name;
-		const BAN::String m_name;
 
 		const Type m_type;
 
@@ -64,6 +61,7 @@ namespace Kernel
 	{
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<KeyboardDevice>> create(mode_t mode, uid_t uid, gid_t gid);
+		static BAN::ErrorOr<void> initialize_tty_thread();
 
 		void notify();
 
@@ -77,10 +75,8 @@ namespace Kernel
 		bool has_hungup_impl() const override { return false; }
 
 		BAN::StringView name() const final override { return m_name; }
-		dev_t rdev() const final override { return m_rdev; }
 
 	private:
-		const dev_t m_rdev;
 		const BAN::StringView m_name;
 		ThreadBlocker m_thread_blocker;
 
@@ -104,10 +100,7 @@ namespace Kernel
 		bool has_hungup_impl() const override { return false; }
 
 		BAN::StringView name() const final override { return m_name; }
-		dev_t rdev() const final override { return m_rdev; }
-
 	private:
-		const dev_t m_rdev;
 		const BAN::StringView m_name;
 		ThreadBlocker m_thread_blocker;
 

kernel/include/kernel/Lock/LockGuard.h

@@ -7,7 +7,7 @@
 namespace Kernel
 {
 
-	template<typename Lock>
+	template<typename Lock> requires requires(Lock& lock) { lock.lock(); lock.unlock(); }
 	class LockGuard
 	{
 		BAN_NON_COPYABLE(LockGuard);

kernel/include/kernel/Lock/Mutex.h

@@ -21,7 +21,7 @@ namespace Kernel
 		virtual uint32_t lock_depth() const = 0;
 	};
 
-	class Mutex : public BaseMutex
+	class Mutex final : public BaseMutex
 	{
 		BAN_NON_COPYABLE(Mutex);
 		BAN_NON_MOVABLE(Mutex);
@@ -40,6 +40,7 @@ namespace Kernel
 				pid_t expected = -1;
 				while (!m_locker.compare_exchange(expected, tid))
 				{
+					ASSERT(Processor::get_interrupt_state() == InterruptState::Enabled);
 					Processor::yield();
 					expected = -1;
 				}
@@ -84,13 +85,14 @@ namespace Kernel
 		pid_t locker() const override { return m_locker; }
 		bool is_locked() const override { return m_locker != -1; }
 		uint32_t lock_depth() const override { return m_lock_depth; }
+		bool is_locked_by_current_thread() const { return m_locker == Thread::current_tid(); }
 
 	private:
 		BAN::Atomic<pid_t>	m_locker		{ -1 };
 		uint32_t			m_lock_depth	{  0 };
 	};
 
-	class PriorityMutex : public BaseMutex
+	class PriorityMutex final : public BaseMutex
 	{
 		BAN_NON_COPYABLE(PriorityMutex);
 		BAN_NON_MOVABLE(PriorityMutex);
@@ -113,6 +115,7 @@ namespace Kernel
 				pid_t expected = -1;
 				while (!(has_priority || m_queue_length == 0) || !m_locker.compare_exchange(expected, tid))
 				{
+					ASSERT(Processor::get_interrupt_state() == InterruptState::Enabled);
 					Processor::yield();
 					expected = -1;
 				}
@@ -164,6 +167,7 @@ namespace Kernel
 		pid_t locker() const override { return m_locker; }
 		bool is_locked() const override { return m_locker != -1; }
 		uint32_t lock_depth() const override { return m_lock_depth; }
+		bool is_locked_by_current_thread() const { return m_locker == Thread::current_tid(); }
 
 	private:
 		BAN::Atomic<pid_t>		m_locker		{ -1 };

kernel/include/kernel/Lock/RWLock.h

@@ -1,7 +1,7 @@
 #pragma once
 
-#include <kernel/Lock/Mutex.h>
+#include <kernel/Lock/SpinLock.h>
-#include <kernel/Lock/LockGuard.h>
+#include <kernel/Lock/SpinLockAsMutex.h>
 
 namespace Kernel
 {
@@ -15,43 +15,60 @@ namespace Kernel
 
 		void rd_lock()
 		{
-			LockGuard _(m_mutex);
+			SpinLockGuard _(m_lock);
-			while (m_writers_waiting > 0 || m_writer_active)
+			while (m_writers_waiting > 0 || m_writer != -1)
-				m_thread_blocker.block_indefinite(&m_mutex);
+			{
+				SpinLockGuardAsMutex smutex(_);
+				m_thread_blocker.block_indefinite(&smutex);
+			}
 			m_readers_active++;
 		}
 
 		void rd_unlock()
 		{
-			LockGuard _(m_mutex);
+			SpinLockGuard _(m_lock);
 			if (--m_readers_active == 0)
 				m_thread_blocker.unblock();
 		}
 
 		void wr_lock()
 		{
-			LockGuard _(m_mutex);
+			if (m_writer == Thread::current_tid())
-			m_writers_waiting++;
+			{
-			while (m_readers_active > 0 || m_writer_active)
+				m_writer_depth++;
-				m_thread_blocker.block_indefinite(&m_mutex);
+				return;
-			m_writers_waiting--;
-			m_writer_active = true;
 			}
 
+			SpinLockGuard _(m_lock);
+
+			m_writers_waiting++;
+			while (m_readers_active > 0 || m_writer != -1)
+			{
+				SpinLockGuardAsMutex smutex(_);
+				m_thread_blocker.block_indefinite(&smutex);
+			}
+			m_writers_waiting--;
+
+			m_writer = Thread::current_tid();
+			m_writer_depth = 1;
+		}
 
 		void wr_unlock()
 		{
-			LockGuard _(m_mutex);
+			if (--m_writer_depth != 0)
-			m_writer_active = false;
+				return;
+			SpinLockGuard _(m_lock);
+			m_writer = -1;
 			m_thread_blocker.unblock();
 		}
 
 	private:
-		Mutex m_mutex;
+		SpinLock m_lock;
 		ThreadBlocker m_thread_blocker;
 		uint32_t m_readers_active  { 0 };
 		uint32_t m_writers_waiting { 0 };
-		bool m_writer_active { false };
+		pid_t m_writer { -1 };
+		uint32_t m_writer_depth { 0 };
 	};
 
 	class RWLockRDGuard

kernel/include/kernel/Lock/SpinLockAsMutex.h

@@ -46,7 +46,7 @@ namespace Kernel
 		uint32_t lock_depth() const override { return m_lock_depth; }
 
 	private:
-		SpinLock& m_lock;
+		Lock& m_lock;
 		uint32_t m_lock_depth { 0 };
 		InterruptState m_state;
 		const pid_t m_locker;

kernel/include/kernel/Memory/ByteRingBuffer.h

@@ -51,7 +51,7 @@ namespace Kernel
 		uint8_t back() const
 		{
 			ASSERT(!empty());
-			return reinterpret_cast<const uint8_t*>(m_vaddr)[m_tail + m_size];
+			return reinterpret_cast<const uint8_t*>(m_vaddr)[m_tail + m_size - 1];
 		}
 
 		bool empty() const { return m_size == 0; }

kernel/include/kernel/Memory/DMARegion.h

@@ -8,7 +8,7 @@ namespace Kernel
 	class DMARegion
 	{
 	public:
-		static BAN::ErrorOr<BAN::UniqPtr<DMARegion>> create(size_t size);
+		static BAN::ErrorOr<BAN::UniqPtr<DMARegion>> create(size_t size, PageTable::MemoryType type = PageTable::MemoryType::Uncached);
 		~DMARegion();
 
 		size_t size() const { return m_size; }

kernel/include/kernel/Memory/FileBackedRegion.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <kernel/FS/Inode.h>
+#include <kernel/Lock/RWLock.h>
 #include <kernel/Memory/MemoryRegion.h>
 
 namespace Kernel
@@ -10,15 +11,15 @@ namespace Kernel
 	{
 		~SharedFileData();
 
-		void sync(size_t page_index);
+		void sync_no_lock(size_t page_index);
 
-		Mutex mutex;
+		RWLock rw_lock;
 
 		// FIXME: this should probably be ordered tree like map
 		//        for fast lookup and less memory usage
 		BAN::Vector<paddr_t> pages;
+		BAN::Vector<uint32_t> writers;
 		BAN::RefPtr<Inode> inode;
-		uint8_t page_buffer[PAGE_SIZE];
 	};
 
 	class FileBackedRegion final : public MemoryRegion

kernel/include/kernel/Memory/MemoryRegion.h

@@ -9,12 +9,6 @@
 namespace Kernel
 {
 
-	struct AddressRange
-	{
-		vaddr_t start;
-		vaddr_t end;
-	};
-
 	class MemoryRegion
 	{
 		BAN_NON_COPYABLE(MemoryRegion);

kernel/include/kernel/Memory/PageTable.h

@@ -14,6 +14,12 @@ namespace Kernel
 		requires BAN::is_same_v<decltype(func()), void>;
 	};
 
+	template<typename F>
+	concept with_per_cpu_fast_page_callback = requires(F func, void* addr)
+	{
+		requires BAN::is_same_v<decltype(func(addr)), void>;
+	};
+
 	template<typename F>
 	concept with_fast_page_callback_error = requires(F func)
 	{
@@ -45,14 +51,27 @@ namespace Kernel
 			WriteThrough,
 		};
 
+		static constexpr bool full_tlb_flush_threshold = 32;
+
+		static constexpr size_t reserved_fast_pages = 0x10;
+
 	public:
-		static void initialize_pre_heap();
+		static void initialize_fast_page();
-		static void initialize_post_heap();
+		static void initialize_and_load();
+
+		static void enable_cpu_features();
 
 		static PageTable& kernel();
 		static PageTable& current() { return *reinterpret_cast<PageTable*>(Processor::get_current_page_table()); }
 
-		static constexpr vaddr_t fast_page() { return KERNEL_OFFSET; }
+		static constexpr vaddr_t fast_page()
+		{
+#if ARCH(x86_64)
+			return 0xffffffffbfe00000;
+#elif ARCH(i686)
+			return 0xffe00000;
+#endif
+		}
 
 		template<with_fast_page_callback F>
 		static void with_fast_page(paddr_t paddr, F callback)
@@ -63,6 +82,18 @@ namespace Kernel
 			unmap_fast_page();
 		}
 
+		template<with_per_cpu_fast_page_callback F>
+		static void with_per_cpu_fast_page(paddr_t paddr, F callback)
+		{
+			const auto state = Processor::get_interrupt_state();
+			Processor::set_interrupt_state(InterruptState::Disabled);
+			const size_t index = Processor::current_index() + reserved_fast_pages;
+			void* addr = map_fast_page(index, paddr);
+			callback(addr);
+			unmap_fast_page(index);
+			Processor::set_interrupt_state(state);
+		}
+
 		template<with_fast_page_callback_error F>
 		static BAN::ErrorOr<void> with_fast_page(paddr_t paddr, F callback)
 		{
@@ -114,33 +145,36 @@ namespace Kernel
 		bool is_page_free(vaddr_t) const;
 		bool is_range_free(vaddr_t, size_t bytes) const;
 
-		bool reserve_page(vaddr_t, bool only_free = true, bool invalidate = true);
+		void reserve_page(vaddr_t);
-		bool reserve_range(vaddr_t, size_t bytes, bool only_free = true);
+		void reserve_range(vaddr_t, size_t bytes);
 
 		vaddr_t reserve_free_page(vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX);
 		vaddr_t reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX);
 
 		void load();
-		void initial_load();
 
 		void invalidate_page(vaddr_t addr, bool send_smp_message) { invalidate_range(addr, 1, send_smp_message); }
 		void invalidate_range(vaddr_t addr, size_t pages, bool send_smp_message);
+		void invalidate_full_address_space(bool global);
 
 		InterruptState lock() const { return m_lock.lock(); }
 		void unlock(InterruptState state) const { m_lock.unlock(state); }
 
+		paddr_t paddr() const { return m_highest_paging_struct; }
+
 		void debug_dump();
 
 	private:
 		PageTable() = default;
 		uint64_t get_page_data(vaddr_t) const;
-		void initialize_kernel();
 		void map_kernel_memory();
-		void prepare_fast_page();
 
 		static void map_fast_page(paddr_t);
 		static void unmap_fast_page();
 
+		static void* map_fast_page(size_t index, paddr_t);
+		static void unmap_fast_page(size_t index);
+
 	private:
 		paddr_t						m_highest_paging_struct { 0 };
 		mutable RecursiveSpinLock	m_lock;

kernel/include/kernel/Memory/Types.h

@@ -23,4 +23,10 @@ namespace Kernel
 	using vaddr_t = uintptr_t;
 	using paddr_t = uint64_t;
 
+	struct AddressRange
+	{
+		vaddr_t start;
+		vaddr_t end;
+	};
+
 }

kernel/include/kernel/Memory/VirtualRange.h

@@ -14,42 +14,27 @@ namespace Kernel
 		BAN_NON_MOVABLE(VirtualRange);
 
 	public:
-		// Create virtual range to fixed virtual address
+		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr_range(PageTable&, AddressRange address_range, size_t, PageTable::flags_t flags, bool add_guard_pages);
-		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr(PageTable&, vaddr_t, size_t, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages);
-		// Create virtual range to virtual address range
-		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr_range(PageTable&, vaddr_t vaddr_start, vaddr_t vaddr_end, size_t, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages);
 		~VirtualRange();
 
 		vaddr_t vaddr() const { return m_vaddr + (m_has_guard_pages ? PAGE_SIZE : 0); }
 		size_t size() const { return m_size - (m_has_guard_pages ? 2 * PAGE_SIZE : 0); }
 		PageTable::flags_t flags() const { return m_flags; }
 
-		paddr_t paddr_of(vaddr_t vaddr) const
+		paddr_t paddr_of(vaddr_t vaddr) const { return m_page_table.physical_address_of(vaddr & PAGE_ADDR_MASK); }
-		{
-			ASSERT(vaddr % PAGE_SIZE == 0);
-			const size_t index = (vaddr - this->vaddr()) / PAGE_SIZE;
-			ASSERT(index < m_paddrs.size());
-			const paddr_t paddr = m_paddrs[index];
-			ASSERT(paddr);
-			return paddr;
-		}
 
 		bool contains(vaddr_t address) const { return vaddr() <= address && address < vaddr() + size(); }
 
-		BAN::ErrorOr<bool> allocate_page_for_demand_paging(vaddr_t address);
-
 	private:
-		VirtualRange(PageTable&, bool preallocated, bool has_guard_pages, vaddr_t, size_t, PageTable::flags_t);
+		VirtualRange(PageTable&, bool has_guard_pages, vaddr_t, size_t, PageTable::flags_t);
 		BAN::ErrorOr<void> initialize();
 
 	private:
 		PageTable&               m_page_table;
-		const bool               m_preallocated;
 		const bool               m_has_guard_pages;
 		const vaddr_t            m_vaddr;
 		const size_t             m_size;
 		const PageTable::flags_t m_flags;
-		BAN::Vector<paddr_t>     m_paddrs;
 		SpinLock                 m_lock;
 
 		friend class BAN::UniqPtr<VirtualRange>;

kernel/include/kernel/Memory/kmalloc.h

@@ -1,16 +1,8 @@
 #pragma once
 
-#include <BAN/Optional.h>
-#include <kernel/Memory/Types.h>
-
 #include <stddef.h>
 
 void kmalloc_initialize();
-void kmalloc_dump_info();
 
-void* kmalloc(size_t size);
+void* kmalloc(size_t);
-void* kmalloc(size_t size, size_t align, bool force_identity_map = false);
 void kfree(void*);
-
-BAN::Optional<Kernel::paddr_t> kmalloc_paddr_of(Kernel::vaddr_t);
-BAN::Optional<Kernel::vaddr_t> kmalloc_vaddr_of(Kernel::paddr_t);

kernel/include/kernel/Networking/E1000/E1000.h

@@ -73,11 +73,15 @@ namespace Kernel
 		BAN::UniqPtr<DMARegion>	m_tx_buffer_region;
 		BAN::UniqPtr<DMARegion>	m_rx_descriptor_region;
 		BAN::UniqPtr<DMARegion>	m_tx_descriptor_region;
-		SpinLock m_lock;
+
+		BAN::Atomic<uint32_t> m_tx_head1 { 0 };
+		BAN::Atomic<uint32_t> m_tx_head2 { 0 };
+
+		SpinLock m_rx_lock;
+		ThreadBlocker m_rx_blocker;
 
 		bool m_thread_should_die { false };
 		BAN::Atomic<bool> m_thread_is_dead { true };
-		ThreadBlocker m_thread_blocker;
 
 		BAN::MACAddress	m_mac_address {};
 		bool			m_link_up { false };

kernel/include/kernel/Networking/NetworkInterface.h

@@ -57,7 +57,6 @@ namespace Kernel
 
 		virtual size_t payload_mtu() const = 0;
 
-		virtual dev_t rdev() const override { return m_rdev; }
 		virtual BAN::StringView name() const override { return m_name; }
 
 		BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::ConstByteSpan payload)
@@ -67,9 +66,10 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::Span<const BAN::ConstByteSpan> payload) = 0;
 
 	private:
-		const Type m_type;
+		BAN::ErrorOr<long> ioctl_impl(int, void*) override;
 
-		const dev_t m_rdev;
+	private:
+		const Type m_type;
 		char m_name[10];
 
 		BAN::IPv4Address m_ipv4_address { 0 };

kernel/include/kernel/Networking/TCPSocket.h

@@ -181,6 +181,7 @@ namespace Kernel
 
 		uint64_t m_time_wait_start_ms { 0 };
 
+		mutable Mutex m_mutex;
 		ThreadBlocker m_thread_blocker;
 
 		RecvWindowInfo m_recv_window;

kernel/include/kernel/Networking/UDPSocket.h

@@ -66,9 +66,12 @@ namespace Kernel
 		SpinLock							m_packet_lock;
 		ThreadBlocker						m_packet_thread_blocker;
 
+		SpinLock							m_peer_address_lock;
 		sockaddr_storage					m_peer_address {};
 		socklen_t							m_peer_address_len { 0 };
 
+		Mutex								m_bind_lock;
+
 		friend class BAN::RefPtr<UDPSocket>;
 	};
 

kernel/include/kernel/Networking/UNIX/Socket.h

@@ -43,15 +43,16 @@ namespace Kernel
 		UnixDomainSocket(Socket::Type, const Socket::Info&);
 		~UnixDomainSocket();
 
-		bool is_bound() const { return !m_bound_file.canonical_path.empty(); }
+		bool is_bound() const;
-		bool is_bound_to_unused() const { return !m_bound_file.inode; }
+		bool is_bound_to_unused() const;
+		BAN::ErrorOr<void> bind_to_unused_if_not_bound();
 
 		bool is_streaming() const;
 
 	private:
 		struct ConnectionInfo
 		{
-			bool										listening { false };
+			BAN::Atomic<bool>							listening { false };
 			BAN::Atomic<bool>							connection_done { false };
 			mutable BAN::Atomic<bool>					target_closed { false };
 			BAN::WeakPtr<UnixDomainSocket>				connection;
@@ -62,6 +63,7 @@ namespace Kernel
 
 		struct ConnectionlessInfo
 		{
+			SpinLock lock;
 			BAN::String peer_address;
 		};
 
@@ -77,6 +79,8 @@ namespace Kernel
 
 	private:
 		const Socket::Type m_socket_type;
+
+		mutable Mutex m_bind_mutex;
 		VirtualFileSystem::File	m_bound_file;
 
 		BAN::Variant<ConnectionInfo, ConnectionlessInfo> m_info;

kernel/include/kernel/OpenFileDescriptorSet.h

@@ -58,6 +58,8 @@ namespace Kernel
 		BAN::ErrorOr<size_t> recvmsg(int socket, msghdr& message, int flags);
 		BAN::ErrorOr<size_t> sendmsg(int socket, const msghdr& message, int flags);
 
+		int get_max_open_fd() const;
+
 		BAN::ErrorOr<VirtualFileSystem::File> file_of(int) const;
 		BAN::ErrorOr<BAN::String> path_of(int) const;
 		BAN::ErrorOr<BAN::RefPtr<Inode>> inode_of(int);

kernel/include/kernel/PCI.h

@@ -66,11 +66,13 @@ namespace Kernel::PCI
 		};
 
 	public:
-		Device() = default;
+		Device(uint8_t bus, uint8_t dev, uint8_t func)
+			: m_bus(bus)
+			, m_dev(dev)
+			, m_func(func)
+		{ }
 
-		void set_location(uint8_t bus, uint8_t dev, uint8_t func);
 		void initialize(paddr_t pcie_paddr);
-		bool is_valid() const { return m_is_valid; }
 
 		uint32_t read_dword(uint8_t) const;
 		uint16_t read_word(uint8_t) const;
@@ -124,10 +126,9 @@ namespace Kernel::PCI
 		BAN::ErrorOr<uint8_t> find_intx_interrupt();
 
 	private:
-		bool m_is_valid	{ false };
+		const uint8_t m_bus	{ 0 };
-		uint8_t m_bus	{ 0 };
+		const uint8_t m_dev	{ 0 };
-		uint8_t m_dev	{ 0 };
+		const uint8_t m_func	{ 0 };
-		uint8_t m_func	{ 0 };
 
 		vaddr_t m_mmio_config { 0 };
 
@@ -161,11 +162,8 @@ namespace Kernel::PCI
 		template<typename F>
 		void for_each_device(F callback)
 		{
-			for (auto& bus : m_buses)
+			for (auto& dev : m_devices)
-				for (auto& dev : bus)
+				callback(dev);
-					for (auto& func : dev)
-						if (func.is_valid())
-							callback(func);
 		};
 
 		uint32_t read_config_dword(uint8_t bus, uint8_t dev, uint8_t func, uint8_t offset);
@@ -179,19 +177,22 @@ namespace Kernel::PCI
 		BAN::Optional<uint8_t> reserve_msi();
 
 	private:
-		PCIManager() : m_bus_pcie_paddr(0) {}
+		struct PCIeInfo
-		void check_function(uint8_t bus, uint8_t dev, uint8_t func);
+		{
-		void check_device(uint8_t bus, uint8_t dev);
+			paddr_t bus_paddr[256];
-		void check_bus(uint8_t bus);
+		};
-		void check_all_buses();
+
+		PCIManager() = default;
+		void check_function(const PCIeInfo&, uint8_t bus, uint8_t dev, uint8_t func);
+		void check_device(const PCIeInfo&, uint8_t bus, uint8_t dev);
+		void check_bus(const PCIeInfo&, uint8_t bus);
+		void check_all_buses(const PCIeInfo&);
 		void initialize_impl();
 
 	private:
 		static constexpr uint8_t m_msi_count = IRQ_MSI_END - IRQ_MSI_BASE;
-		using PCIBus = BAN::Array<BAN::Array<Device, 8>, 32>;
+
-		BAN::Array<PCIBus, 256>  m_buses;
+		BAN::Vector<Device> m_devices;
-		BAN::Array<paddr_t, 256> m_bus_pcie_paddr;
-		bool                     m_is_pcie { false };
 
 		SpinLock                             m_reserved_msi_lock;
 		BAN::Array<uint8_t, m_msi_count / 8> m_reserved_msi_bitmap;

kernel/include/kernel/Process.h

@@ -56,6 +56,7 @@ namespace Kernel
 		pid_t pid() const { return m_pid; }
 
 		bool is_session_leader() const { return pid() == sid(); }
+		bool is_pgrpg_in_this_session(pid_t) const;
 
 		const char* name() const { return m_cmdline.empty() ? "<unknown>" : m_cmdline.front().data(); }
 
@@ -63,9 +64,6 @@ namespace Kernel
 
 		BAN::ErrorOr<long> sys_exit(int status);
 
-		BAN::ErrorOr<long> sys_tcgetattr(int fildes, termios*);
-		BAN::ErrorOr<long> sys_tcsetattr(int fildes, int optional_actions, const termios*);
-
 		BAN::ErrorOr<long> sys_fork(uintptr_t rsp, uintptr_t rip);
 		BAN::ErrorOr<long> sys_exec(const char* path, const char* const* argv, const char* const* envp);
 
@@ -102,13 +100,14 @@ namespace Kernel
 
 		BAN::ErrorOr<long> open_inode(VirtualFileSystem::File&&, int flags);
 
-		BAN::ErrorOr<void> create_file_or_dir(int fd, const char* path, mode_t mode) const;
+		BAN::ErrorOr<void> create_file(int fd, const char* path, mode_t) const;
-		BAN::ErrorOr<long> sys_openat(int, const char* path, int, mode_t);
+		BAN::ErrorOr<long> sys_openat(int fd, const char* path, int flags, mode_t);
 		BAN::ErrorOr<long> sys_close(int fd);
 		BAN::ErrorOr<long> sys_read(int fd, void* buffer, size_t count);
 		BAN::ErrorOr<long> sys_write(int fd, const void* buffer, size_t count);
 		BAN::ErrorOr<long> sys_access(const char* path, int amode);
-		BAN::ErrorOr<long> sys_create_dir(const char*, mode_t);
+		BAN::ErrorOr<long> sys_mkdirat(int fd, const char* path, mode_t);
+		BAN::ErrorOr<long> sys_mkfifoat(int fd, const char* path, mode_t);
 		BAN::ErrorOr<long> sys_hardlinkat(int fd1, const char* path1, int fd2, const char* path2, int flag);
 		BAN::ErrorOr<long> sys_renameat(int oldfd, const char* old, int newfd, const char* _new);
 		BAN::ErrorOr<long> sys_unlinkat(int fd, const char* path, int flag);
@@ -186,7 +185,6 @@ namespace Kernel
 		BAN::ErrorOr<long> sys_smo_map(SharedMemoryObjectManager::Key);
 
 		BAN::ErrorOr<long> sys_ttyname(int fildes, char* name, size_t namesize);
-		BAN::ErrorOr<long> sys_isatty(int fildes);
 		BAN::ErrorOr<long> sys_posix_openpt(int flags);
 		BAN::ErrorOr<long> sys_ptsname(int fildes, char* buffer, size_t buffer_len);
 
@@ -217,14 +215,14 @@ namespace Kernel
 		BAN::ErrorOr<long> sys_pthread_join(pthread_t thread, void** value);
 		BAN::ErrorOr<long> sys_pthread_self();
 		BAN::ErrorOr<long> sys_pthread_kill(pthread_t thread, int signal);
-
+		BAN::ErrorOr<long> sys_pthread_detach(pthread_t thread);
-		BAN::ErrorOr<long> sys_tcgetpgrp(int fd);
-		BAN::ErrorOr<long> sys_tcsetpgrp(int fd, pid_t pgid);
 
 		BAN::ErrorOr<long> sys_clock_gettime(clockid_t, timespec*);
 
 		BAN::ErrorOr<long> sys_load_keymap(const char* path);
 
+		BAN::ErrorOr<long> sys_banos_install(const char* object);
+
 		BAN::RefPtr<TTY> controlling_terminal() { return m_controlling_terminal; }
 
 		static Process& current() { return Thread::current().process(); }
@@ -282,13 +280,12 @@ namespace Kernel
 		// You must hold reader end of m_mapped_region_lock when calling this.
 		size_t find_mapped_region(vaddr_t) const;
 
+		BAN::ErrorOr<AddressRange> find_free_address_range(size_t size);
+
 		BAN::ErrorOr<VirtualFileSystem::File> find_file(int fd, const char* path, int flags) const;
 		BAN::ErrorOr<FileParent> find_parent_file(int fd, const char* path, int flags) const;
 		BAN::ErrorOr<VirtualFileSystem::File> find_relative_parent(int fd, const char* path) const;
 
-		BAN::ErrorOr<void> read_from_user(const void* user_addr, void* out, size_t size);
-		BAN::ErrorOr<void> read_string_from_user(const char* user_addr, char* out, size_t max_size);
-		BAN::ErrorOr<void> write_to_user(void* user_addr, const void* in, size_t size);
 		BAN::ErrorOr<MemoryRegion*> validate_and_pin_pointer_access(const void*, size_t, bool needs_write);
 
 		uint64_t signal_pending_mask() const

kernel/include/kernel/Processor.h

@@ -2,6 +2,7 @@
 
 #include <BAN/Atomic.h>
 #include <BAN/ForwardList.h>
+#include <BAN/Math.h>
 
 #include <kernel/API/SharedPage.h>
 #include <kernel/Arch.h>
@@ -28,6 +29,13 @@ namespace Kernel
 		BAN_NON_MOVABLE(Processor);
 
 	public:
+		struct TLBEntry
+		{
+			vaddr_t vaddr;
+			size_t page_count;
+			class PageTable* page_table;
+		};
+
 		struct SMPMessage
 		{
 			enum class Type
@@ -42,22 +50,25 @@ namespace Kernel
 			Type type;
 			union
 			{
-				struct
+				TLBEntry flush_tlb;
-				{
-					uintptr_t vaddr;
-					size_t page_count;
-					void* page_table;
-				} flush_tlb;
 				SchedulerQueue::Node* new_thread;
 				SchedulerQueue::Node* unblock_thread;
 				bool dummy;
 			};
 		};
 
+		struct LoadStats
+		{
+			uint64_t ns_idle;
+			uint64_t ns_total;
+		};
+
 	public:
 		static Processor& create(ProcessorID id);
 		static Processor& initialize();
 
+		void allocate_stack();
+
 		static ProcessorID current_id() { return read_gs_sized<ProcessorID>(offsetof(Processor, m_id)); }
 		static uint8_t current_index()  { return read_gs_sized<uint8_t>(offsetof(Processor, m_index)); }
 		static ProcessorID id_from_index(size_t index);
@@ -67,9 +78,6 @@ namespace Kernel
 		static void set_smp_enabled() { s_is_smp_enabled = true; }
 		static void wait_until_processors_ready();
 
-		static void toggle_should_print_cpu_load() { s_should_print_cpu_load = !s_should_print_cpu_load; }
-		static bool get_should_print_cpu_load() { return s_should_print_cpu_load; }
-
 		static ProcessorID bsp_id() { return s_bsp_id; }
 		static bool current_is_bsp() { return current_id() == bsp_id(); }
 
@@ -100,11 +108,8 @@ namespace Kernel
 				handle_smp_messages();
 		}
 
-		static uintptr_t current_stack_bottom() { return read_gs_sized<uintptr_t>(offsetof(Processor, m_stack)); }
+		vaddr_t stack_top_vaddr() const { return m_stack_vaddr + s_stack_size; }
-		static uintptr_t current_stack_top()	{ return current_stack_bottom() + s_stack_size; }
+		paddr_t stack_top_paddr() const { return m_stack_paddr + s_stack_size; }
-
-		uintptr_t stack_bottom() const	{ return reinterpret_cast<uintptr_t>(m_stack); }
-		uintptr_t stack_top() const		{ return stack_bottom() + s_stack_size; }
 
 		static void set_thread_syscall_stack(vaddr_t vaddr) { write_gs_sized<vaddr_t>(offsetof(Processor, m_thread_syscall_stack), vaddr); }
 
@@ -114,6 +119,8 @@ namespace Kernel
 		static void* get_current_page_table()					{ return read_gs_sized<void*>(offsetof(Processor, m_current_page_table)); }
 		static void set_current_page_table(void* page_table)	{ write_gs_sized<void*>(offsetof(Processor, m_current_page_table), page_table); }
 
+		static LoadStats get_load_stats(size_t index);
+
 		static void yield();
 		static Scheduler& scheduler() { return *read_gs_sized<Scheduler*>(offsetof(Processor, m_scheduler)); }
 
@@ -130,7 +137,7 @@ namespace Kernel
 		static void handle_ipi();
 
 		static void handle_smp_messages();
-		static void send_smp_message(ProcessorID, const SMPMessage&, bool send_ipi = true);
+		static bool send_smp_message(ProcessorID, const SMPMessage&, bool send_ipi = true);
 		static void broadcast_smp_message(const SMPMessage&);
 
 		static void load_segments();
@@ -140,11 +147,7 @@ namespace Kernel
 		static void disable_sse()
 		{
 			uintptr_t dummy;
-#if ARCH(x86_64)
+			asm volatile("mov %%cr0, %0; or $0x08, %0; mov %0, %%cr0" : "=r"(dummy));
-			asm volatile("movq %%cr0, %0; orq $0x08, %0; movq %0, %%cr0" : "=r"(dummy));
-#elif ARCH(i686)
-			asm volatile("movl %%cr0, %0; orl $0x08, %0; movl %0, %%cr0" : "=r"(dummy));
-#endif
 		}
 
 		static void enable_sse()
@@ -169,42 +172,26 @@ namespace Kernel
 		}
 
 		template<typename T>
-		static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8)
+		static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8 && BAN::Math::is_power_of_two(sizeof(T)))
 		{
-#define __ASM_INPUT(operation) asm volatile(operation " %%gs:%a[offset], %[result]" : [result]"=r"(result) : [offset]"ir"(offset))
+			T value;
-			T result;
+			asm volatile("mov %%gs:%a[offset], %[value]" : [value]"=r"(value) : [offset]"ir"(offset));
-			if constexpr(sizeof(T) == 8)
+			return value;
-				__ASM_INPUT("movq");
-			if constexpr(sizeof(T) == 4)
-				__ASM_INPUT("movl");
-			if constexpr(sizeof(T) == 2)
-				__ASM_INPUT("movw");
-			if constexpr(sizeof(T) == 1)
-				__ASM_INPUT("movb");
-			return result;
-#undef __ASM_INPUT
 		}
 
 		template<typename T>
-		static void write_gs_sized(uintptr_t offset, T value) requires(sizeof(T) <= 8)
+		static void write_gs_sized(uintptr_t offset, T value) requires(sizeof(T) <= 8 && BAN::Math::is_power_of_two(sizeof(T)))
 		{
-#define __ASM_INPUT(operation) asm volatile(operation " %[value], %%gs:%a[offset]" :: [value]"r"(value), [offset]"ir"(offset) : "memory")
+			asm volatile("mov %[value], %%gs:%a[offset]" :: [value]"r"(value), [offset]"ir"(offset) : "memory");
-			if constexpr(sizeof(T) == 8)
-				__ASM_INPUT("movq");
-			if constexpr(sizeof(T) == 4)
-				__ASM_INPUT("movl");
-			if constexpr(sizeof(T) == 2)
-				__ASM_INPUT("movw");
-			if constexpr(sizeof(T) == 1)
-				__ASM_INPUT("movb");
-#undef __ASM_INPUT
 		}
 
+		void lock_tlb_lock();
+		void unlock_tlb_lock();
+
 	private:
 		static ProcessorID s_bsp_id;
 		static BAN::Atomic<uint8_t> s_processor_count;
 		static BAN::Atomic<bool>    s_is_smp_enabled;
-		static BAN::Atomic<bool>    s_should_print_cpu_load;
 		static paddr_t              s_shared_page_paddr;
 		static vaddr_t              s_shared_page_vaddr;
 
@@ -215,23 +202,28 @@ namespace Kernel
 
 		Thread* m_sse_thread { nullptr };
 
-		static constexpr size_t s_stack_size { 4096 };
+		static constexpr size_t s_stack_size { PAGE_SIZE };
-		void* m_stack { nullptr };
+		vaddr_t m_stack_vaddr { 0 };
+		paddr_t m_stack_paddr { 0 };
 
 		GDT* m_gdt { nullptr };
 		IDT* m_idt { nullptr };
 
 		Scheduler* m_scheduler { nullptr };
 
-		uint64_t m_start_ns { 0 };
+		BAN::Atomic<bool> m_load_stat_lock;
-		uint64_t m_idle_ns { 0 };
+		uint64_t m_load_start_ns { 0 };
-		uint64_t m_last_update_ns { 0 };
+		LoadStats m_load_stats {};
-		uint64_t m_next_update_ns { 0 };
 
 		BAN::Atomic<SMPMessage*> m_smp_pending { nullptr };
 		BAN::Atomic<SMPMessage*> m_smp_free    { nullptr };
 		SMPMessage* m_smp_message_storage { nullptr };
 
+		BAN::Atomic<bool> m_tlb_lock { false };
+		size_t m_tlb_entry_count { 0 };
+		BAN::Array<TLBEntry, 32> m_tlb_entries;
+		bool m_tlb_global { false };
+
 		void* m_current_page_table { nullptr };
 
 		friend class BAN::Array<Processor, 0xFF>;

kernel/include/kernel/SchedulerQueueNode.h

@@ -22,8 +22,9 @@ namespace Kernel
 
 		uint64_t wake_time_ns { static_cast<uint64_t>(-1) };
 
-		SpinLock blocker_lock;
+		BAN::Atomic<ThreadBlocker*> blocker { nullptr };
-		ThreadBlocker* blocker { nullptr };
+		SchedulerQueueNode* block_chain_prev { nullptr };
+		SchedulerQueueNode* block_chain_next { nullptr };
 
 		ProcessorID processor_id { PROCESSOR_NONE };
 		bool blocked { false };

kernel/include/kernel/Storage/ATA/ATADevice.h

@@ -31,8 +31,6 @@ namespace Kernel
 			BAN::StringView model() const { return m_model; }
 			BAN::StringView name() const override { return m_name; }
 
-			dev_t rdev() const override { return m_rdev; }
-
 		protected:
 			ATABaseDevice();
 			BAN::ErrorOr<void> initialize(BAN::Span<const uint16_t> identify_data);
@@ -46,8 +44,6 @@ namespace Kernel
 			bool m_has_lba;
 			char m_model[41];
 			char m_name[4] {};
-
-			const dev_t m_rdev;
 		};
 
 	}

kernel/include/kernel/Storage/NVMe/Controller.h

@@ -20,7 +20,6 @@ namespace Kernel
 
 		NVMeQueue& io_queue() { return *m_io_queue; }
 
-		virtual dev_t rdev() const override { return m_rdev; }
 		virtual BAN::StringView name() const override { return m_name; }
 
 	protected:
@@ -51,7 +50,6 @@ namespace Kernel
 		BAN::Vector<BAN::RefPtr<NVMeNamespace>> m_namespaces;
 
 		char m_name[20];
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Storage/NVMe/Namespace.h

@@ -16,7 +16,6 @@ namespace Kernel
 		virtual uint32_t sector_size() const override { return m_block_size; }
 		virtual uint64_t total_size() const override { return m_block_size * m_block_count; }
 
-		virtual dev_t rdev() const override { return m_rdev; }
 		virtual BAN::StringView name() const override { return m_name; }
 
 	private:
@@ -35,7 +34,6 @@ namespace Kernel
 		const uint64_t m_block_count;
 
 		char m_name[10] {};
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Storage/Partition.h

@@ -42,11 +42,6 @@ namespace Kernel
 		char m_label[36 * 4 + 1];
 		const BAN::String m_name;
 
-	public:
-		virtual bool is_partition() const override { return true; }
-
-		virtual dev_t rdev() const override { return m_rdev; }
-
 	protected:
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 
@@ -56,7 +51,6 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
-		const dev_t m_rdev;
 	};
 
 }

kernel/include/kernel/Storage/StorageDevice.h

@@ -14,7 +14,10 @@ namespace Kernel
 	public:
 		StorageDevice()
 			: BlockDevice(0660, 0, 0)
-		{ }
+		{
+			m_kind |= InodeKind::STORAGE;
+		}
+
 		virtual ~StorageDevice();
 
 		BAN::ErrorOr<void> initialize_partitions(BAN::StringView name_prefix);
@@ -35,7 +38,6 @@ namespace Kernel
 
 		size_t drop_disk_cache();
 		BAN::ErrorOr<void> sync_disk_cache();
-		virtual bool is_storage_device() const override { return true; }
 
 	protected:
 		virtual BAN::ErrorOr<void> read_sectors_impl(uint64_t lba, uint64_t sector_count, BAN::ByteSpan) = 0;

kernel/include/kernel/Terminal/PseudoTerminal.h

@@ -13,7 +13,6 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<PseudoTerminalMaster>> create(mode_t, uid_t, gid_t);
 
-		dev_t rdev() const override { return m_rdev; }
 		BAN::StringView name() const override { return "<ptmx>"_sv; }
 
 		BAN::ErrorOr<BAN::RefPtr<PseudoTerminalSlave>> slave();
@@ -48,8 +47,6 @@ namespace Kernel
 		size_t m_buffer_tail { 0 };
 		size_t m_buffer_size { 0 };
 
-		const dev_t m_rdev;
-
 		friend class PseudoTerminalSlave;
 		friend class BAN::RefPtr<PseudoTerminalMaster>;
 	};
@@ -67,7 +64,7 @@ namespace Kernel
 		bool putchar_impl(uint8_t ch) override;
 
 		bool can_write_impl() const override;
-		bool has_hungup_impl() const override { return !m_master.valid(); }
+		bool has_hungup_impl() const override { return master_has_closed(); }
 
 	private:
 		PseudoTerminalSlave(BAN::String&& name, uint32_t number, mode_t, uid_t, gid_t);

kernel/include/kernel/Terminal/TTY.h

@@ -33,9 +33,6 @@ namespace Kernel
 	public:
 		virtual BAN::ErrorOr<void> set_font(LibFont::Font&&) { return BAN::Error::from_errno(EINVAL); }
 
-		void set_foreground_pgrp(pid_t pgrp) { m_foreground_pgrp = pgrp; }
-		pid_t foreground_pgrp() const { return m_foreground_pgrp; }
-
 		BAN::ErrorOr<void> tty_ctrl(int command, int flags);
 
 		// for kprint
@@ -46,28 +43,14 @@ namespace Kernel
 
 		static void keyboard_task(void*);
 		static void initialize_devices();
+
+		bool should_receive_input() const { return m_tty_ctrl.receive_input; }
+		void on_key_event(LibInput::RawKeyEvent);
 		void on_key_event(LibInput::KeyEvent);
 		void handle_input_byte(uint8_t);
 
-		void get_termios(termios* termios) { *termios = m_termios; }
-		// FIXME: validate termios
-		BAN::ErrorOr<void> set_termios(const termios* termios) { m_termios = *termios; return {}; }
-
-		virtual bool is_tty() const override { return true; }
-
-		virtual dev_t rdev() const final override { return m_rdev; }
-
 		virtual void clear() = 0;
 
-		virtual BAN::ErrorOr<void> chmod_impl(mode_t) override;
-		virtual BAN::ErrorOr<void> chown_impl(uid_t, gid_t) override;
-
-		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) override;
-
-		virtual bool can_read_impl() const override { return m_output.flush; }
-		virtual bool has_error_impl() const override { return false; }
-		virtual bool has_hungup_impl() const override { return false; }
-
 		virtual bool master_has_closed() const { return false; }
 
 	protected:
@@ -76,43 +59,50 @@ namespace Kernel
 		virtual bool putchar_impl(uint8_t ch) = 0;
 		virtual void after_write() {}
 
+		void update_winsize(unsigned short cols, unsigned short rows);
+
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) final override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) final override;
 
-		void update_winsize(unsigned short cols, unsigned short rows);
+		virtual bool can_read_impl() const override { return m_output.flush; }
+		virtual bool has_error_impl() const override { return false; }
+		virtual bool has_hungup_impl() const override { return false; }
+
+		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) override;
 
 	private:
 		bool putchar(uint8_t ch);
 		void do_backspace();
 
-	protected:
+		termios get_termios();
-		termios m_termios;
 
 	private:
-		const dev_t m_rdev;
+		BAN::Atomic<pid_t> m_foreground_pgrp { 0 };
-
-		pid_t m_foreground_pgrp { 0 };
 
 		struct tty_ctrl_t
 		{
-			bool draw_graphics { true };
+			BAN::Atomic<bool> draw_graphics { true };
-			bool receive_input { true };
+			BAN::Atomic<bool> receive_input { true };
-			ThreadBlocker thread_blocker;
 		};
 		tty_ctrl_t m_tty_ctrl;
 
 		struct Buffer
 		{
 			BAN::UniqPtr<ByteRingBuffer> buffer;
-			bool flush { false };
+			BAN::Atomic<bool> flush { false };
 			ThreadBlocker thread_blocker;
 		};
 		Buffer m_output;
 
 		winsize m_winsize {};
 
+		SpinLock m_termios_lock;
+		termios m_termios;
+
 	protected:
-		RecursiveSpinLock m_write_lock;
+		Mutex m_mutex;
+
+		Mutex m_write_lock;
 		ThreadBlocker m_write_blocker;
 	};
 

kernel/include/kernel/Thread.h

@@ -117,6 +117,9 @@ namespace Kernel
 		const Process& process() const;
 		bool has_process() const { return m_process; }
 
+		void detach() { m_is_detached = true; }
+		bool is_detached() const { return m_is_detached; }
+
 		bool is_userspace() const { return m_is_userspace; }
 
 		uint64_t cpu_time_ns() const;
@@ -176,11 +179,10 @@ namespace Kernel
 		State                      m_state                { State::NotStarted };
 		Process*                   m_process              { nullptr };
 		bool                       m_is_userspace         { false };
+		BAN::Atomic<bool>          m_is_detached          { false };
 		bool                       m_delete_process       { false };
 
-		bool                       m_has_custom_fsbase    { false };
 		vaddr_t                    m_fsbase               { 0 };
-		bool                       m_has_custom_gsbase    { false };
 		vaddr_t                    m_gsbase               { 0 };
 
 		SchedulerQueue::Node*      m_scheduler_node       { nullptr };

kernel/include/kernel/ThreadBlocker.h

@@ -15,7 +15,6 @@ namespace Kernel
 		void block_with_wake_time_ns(uint64_t wake_time_ns, BaseMutex*);
 		void unblock();
 
-
 		void block_with_timeout_ms(uint64_t timeout_ms, BaseMutex* mutex)
 		{
 			ASSERT(!BAN::Math::will_multiplication_overflow<uint64_t>(timeout_ms, 1'000'000));
@@ -29,14 +28,12 @@ namespace Kernel
 
 	private:
 		void add_thread_to_block_queue(SchedulerQueue::Node*);
-		void remove_blocked_thread(SchedulerQueue::Node*);
+		void remove_thread_from_block_queue(SchedulerQueue::Node*);
 
 	private:
+		SchedulerQueue::Node* m_block_chain { nullptr };
 		SpinLock m_lock;
 
-		SchedulerQueue::Node* m_block_chain[32] {};
-		size_t m_block_chain_length { 0 };
-
 		friend class Scheduler;
 	};
 

kernel/include/kernel/USB/HID/Joystick.h

@@ -62,8 +62,8 @@ namespace Kernel
 		Mutex m_command_mutex;
 
 		BAN::Atomic<bool> m_has_initialized_leds { false };
-		uint8_t m_led_state       { 0b0001 };
+		BAN::Atomic<uint8_t> m_led_state         { 0b0001 };
-		uint8_t m_rumble_strength { 0x00 };
+		BAN::Atomic<uint8_t> m_rumble_strength   { 0x00 };
 
 		friend class BAN::RefPtr<USBJoystick>;
 	};

kernel/include/kernel/USB/HID/Keyboard.h

@@ -11,6 +11,8 @@ namespace Kernel
 		BAN_NON_MOVABLE(USBKeyboard);
 
 	public:
+		BAN::ErrorOr<void> initialize() override;
+
 		void start_report() override;
 		void stop_report() override;
 
@@ -38,6 +40,8 @@ namespace Kernel
 		uint16_t m_toggle_mask { 0 };
 
 		uint16_t m_led_mask { 0 };
+		BAN::UniqPtr<DMARegion> m_led_region;
+
 		BAN::Vector<USBHID::Report> m_outputs;
 
 		BAN::Optional<uint8_t> m_repeat_scancode;

kernel/include/kernel/USB/MassStorage/SCSIDevice.h

@@ -15,7 +15,6 @@ namespace Kernel
 		uint32_t sector_size() const override { return m_block_size; }
 		uint64_t total_size() const override { return m_block_size * m_block_count; }
 
-		dev_t rdev() const override { return m_rdev; }
 		BAN::StringView name() const override { return m_name; }
 
 	private:
@@ -34,7 +33,6 @@ namespace Kernel
 		const uint64_t m_block_count;
 		const uint32_t m_block_size;
 
-		const dev_t m_rdev;
 		const char m_name[4];
 
 		friend class BAN::RefPtr<USBSCSIDevice>;

kernel/include/kernel/UserCopy.h

@@ -0,0 +1,12 @@
+#pragma once
+
+#include <BAN/Errors.h>
+
+namespace Kernel
+{
+
+	BAN::ErrorOr<void> read_from_user(const void* user_addr, void* out, size_t size);
+	BAN::ErrorOr<void> read_string_from_user(const char* user_addr, char* out, size_t max_size);
+	BAN::ErrorOr<void> write_to_user(void* user_addr, const void* in, size_t size);
+
+};

kernel/kernel/ACPI/BatterySystem.cpp

@@ -57,7 +57,7 @@ namespace Kernel::ACPI
 				m_last_value = target_conv.value().as.integer.value;
 			}
 
-			auto target_str = TRY(BAN::String::formatted("{}", m_last_value));
+			auto target_str = TRY(BAN::String::formatted("{}", m_last_value.load()));
 
 			if (static_cast<size_t>(offset) >= target_str.size())
 				return 0;
@@ -90,8 +90,8 @@ namespace Kernel::ACPI
 		AML::NameString m_method_name;
 		size_t m_result_index;
 
-		uint64_t m_last_read_ms = 0;
+		BAN::Atomic<uint64_t> m_last_read_ms = 0;
-		uint64_t m_last_value = 0;
+		BAN::Atomic<uint64_t> m_last_value = 0;
 	};
 
 	BAN::ErrorOr<void> BatterySystem::initialize(AML::Namespace& acpi_namespace)

kernel/kernel/APIC.cpp

@@ -293,9 +293,25 @@ namespace Kernel
 			dprintln("Trying to enable processor (lapic id {})", processor.apic_id);
 
 			auto& proc = Kernel::Processor::create(ProcessorID(processor.apic_id));
+			proc.allocate_stack();
+
+			struct ap_init_info_t
+			{
+				uintptr_t stack_paddr;
+				uintptr_t stack_vaddr;
+				uintptr_t prepare_paging;
+				uintptr_t page_table;
+				uintptr_t ready;
+			};
+
 			PageTable::with_fast_page(ap_init_paddr, [&] {
-				PageTable::fast_page_as_sized<uint32_t>(2) = kmalloc_paddr_of(proc.stack_top()).value();
+				PageTable::fast_page_as<ap_init_info_t>(8) = {
-				PageTable::fast_page_as_sized<uint8_t>(13) = 0;
+					.stack_paddr    = static_cast<uintptr_t>(proc.stack_top_paddr()),
+					.stack_vaddr    = proc.stack_top_vaddr(),
+					.prepare_paging = reinterpret_cast<uintptr_t>(&PageTable::enable_cpu_features),
+					.page_table     = static_cast<uintptr_t>(PageTable::kernel().paddr()),
+					.ready          = 0,
+				};
 			});
 
 			write_to_local_apic(LAPIC_ERROR_REG, 0x00);
@@ -334,12 +350,9 @@ namespace Kernel
 
 			// give processor upto 100 * 100 us + 200 us to boot
 			PageTable::with_fast_page(ap_init_paddr, [&] {
-				for (int i = 0; i < 100; i++)
+				for (int i = 0; i < 100; i++, udelay(100))
-				{
+					if (__atomic_load_n(&PageTable::fast_page_as<ap_init_info_t>(8).ready, __ATOMIC_SEQ_CST))
-					if (__atomic_load_n(&PageTable::fast_page_as_sized<uint8_t>(13), __ATOMIC_SEQ_CST))
 						break;
-					udelay(100);
-				}
 			});
 
 			initialized_aps++;

kernel/kernel/Audio/Controller.cpp

@@ -15,8 +15,8 @@ namespace Kernel
 
 	AudioController::AudioController()
 		: CharacterDevice(0644, 0, 0)
-		, m_rdev(makedev(DeviceNumber::AudioController, s_next_audio_minor++))
 	{
+		m_rdev = makedev(DeviceNumber::AudioController, s_next_audio_minor++);
 		char* ptr = m_name;
 		BAN::Formatter::print([&ptr](char c) { *ptr++ = c; }, "audio{}", minor(m_rdev));
 	}

kernel/kernel/Audio/HDAudio/AudioFunctionGroup.cpp

@@ -1,6 +1,7 @@
 #include <kernel/Audio/HDAudio/AudioFunctionGroup.h>
 #include <kernel/Audio/HDAudio/Registers.h>
 #include <kernel/FS/DevFS/FileSystem.h>
+#include <kernel/Timer/Timer.h>
 
 #include <BAN/Sort.h>
 
@@ -202,12 +203,12 @@ namespace Kernel
 		ASSERT(m_stream_index == 0xFF);
 		m_stream_index = TRY(m_controller->allocate_stream(HDAudio::StreamType::Output, this));
 
-		reset_stream();
+		TRY(reset_stream());
 
 		return {};
 	}
 
-	void HDAudioFunctionGroup::reset_stream()
+	BAN::ErrorOr<void> HDAudioFunctionGroup::reset_stream()
 	{
 		using Regs = HDAudio::Regs;
 
@@ -219,13 +220,23 @@ namespace Kernel
 		// stop stream
 		bar.write8(base + Regs::SDCTL, bar.read8(base + Regs::SDCTL) & 0xFD);
 
+		const auto timeout_ms = SystemTimer::get().ms_since_boot() + 100;
+
 		// reset stream
 		bar.write8(base + Regs::SDCTL, (bar.read8(base + Regs::SDCTL) & 0xFE) | 1);
 		while (!(bar.read8(base + Regs::SDCTL) & 1))
+		{
+			if (SystemTimer::get().ms_since_boot() > timeout_ms)
+				return BAN::Error::from_errno(ETIMEDOUT);
 			Processor::pause();
+		}
 		bar.write8(base + Regs::SDCTL, (bar.read8(base + Regs::SDCTL) & 0xFE));
 		while ((bar.read8(base + Regs::SDCTL) & 1))
+		{
+			if (SystemTimer::get().ms_since_boot() > timeout_ms)
+				return BAN::Error::from_errno(ETIMEDOUT);
 			Processor::pause();
+		}
 
 		// set bdl address, total size and lvi
 		const paddr_t bdl_paddr = m_bdl_region->paddr() + bdl_offset();
@@ -244,6 +255,8 @@ namespace Kernel
 		m_bdl_head = 0;
 		m_bdl_tail = 0;
 		m_stream_running = false;
+
+		return {};
 	}
 
 	BAN::ErrorOr<void> HDAudioFunctionGroup::initialize_output()
@@ -621,7 +634,13 @@ namespace Kernel
 
 			m_bdl_tail = (m_bdl_tail + 1) % m_bdl_entry_count;
 			if (m_bdl_tail == m_bdl_head)
-				reset_stream();
+			{
+				if (auto ret = reset_stream(); ret.is_error())
+				{
+					dwarnln("failed to reset HDA stream: {}", ret.error());
+					return;
+				}
+			}
 
 			queue_bdl_data();
 		}

kernel/kernel/Banos.cpp

@@ -0,0 +1,215 @@
+#include <kernel/Debug.h>
+#include <kernel/Banos.h>
+#include <BAN/Assert.h>
+#include <banos/driver.h>
+#include <banos/print.h>
+#include <banos/export.h>
+#include <kernel/FS/VirtualFileSystem.h>
+#include <kernel/Memory/PageTable.h>
+#include <kernel/ELF.h>
+#include <LibELF/Types.h>
+#include <LibELF/Values.h>
+#include <kernel/Process.h>
+#include <BAN/HashMap.h>
+#include <kernel/Lock/SpinLock.h>
+#include <kernel/UserCopy.h>
+
+using namespace LibELF;
+using namespace Kernel;
+
+extern "C" {
+	void banos_dprintln(const char* str) {
+		dprintln("{}", str);
+	}
+	void* banos_lookup_symbol(const char* str) {
+		return Banos::resolve_symbol(str);
+	}
+}
+BANOS_EXPORT(banos_dprintln);
+BANOS_EXPORT(banos_lookup_symbol);
+
+BAN::HashMap<BAN::StringView, void*> g_banos_symbols;
+void* Banos::resolve_symbol(const char* name) {
+	auto it = g_banos_symbols.find(name);
+	return it == g_banos_symbols.end() ? NULL : it->value;
+}
+void Banos::import_symbols(Banos_Symbol* symbols, size_t count) {
+	for(size_t i = 0; i < count; ++i) {
+		auto sym = symbols + i;
+		MUST(g_banos_symbols.insert(sym->name, sym->arg));
+	}
+}
+// TODO: driver unloading with a reference counter
+struct Driver_Instance {
+	Banos_Driver* drv;
+};
+static BAN::Vector<Driver_Instance> s_driver_instaces;
+static SpinLock s_driver_instaces_lock;
+
+extern Banos_Symbol g_banos_export[],
+					g_banos_export_end[];
+static void load_drv(Banos_Driver* drv) {
+	ASSERT(drv->driver_size >= sizeof(Banos_Driver));
+	dprintln("Loading driver:");
+	dprintln("  name: {}", drv->name);
+	if(drv->license) dprintln("  license: {}", drv->license);
+	dprintln("  version: {}.{}.{}", BANOS_VERSION_GET_MAJOR(drv->version), BANOS_VERSION_GET_MINOR(drv->version), BANOS_VERSION_GET_PATCH(drv->version));
+	int e = drv->init(drv);
+	if(e < 0) dprintln(" Failed to init {} => {}", drv->name, -e);
+}
+BAN::ErrorOr<size_t> Banos::load_driver_from_image(const char* u_image) {
+	if(!Process::current().credentials().is_superuser()) return BAN::Error::from_errno(EPERM);
+	// TODO: permission verification. Only root should be allowed to do this
+	LibELF::ElfNativeFileHeader header;
+
+	const unsigned char elf_class =
+		#if ARCH(i686)
+			ELFCLASS32;
+		#elif ARCH(x86_64)
+			ELFCLASS64;
+		#else
+		#   error update elf class
+		#endif
+
+	// TODO: is banan-os really ever gonna be running on MSB machines?
+	const unsigned char elf_data = ELFDATA2LSB;
+	// TODO: do we need to verify e_machine? I mean we do not really care.
+	// But I'm leaving this todo:
+	//   Look up EM_X86_64 and EM_360|EM_860|EM_960
+	TRY(read_from_user(u_image, &header, sizeof header));
+	if( header.e_ident[EI_MAG0]	   != ELFMAG0        ||
+		header.e_ident[EI_MAG1]	   != ELFMAG1        ||
+		header.e_ident[EI_MAG2]	   != ELFMAG2        ||
+		header.e_ident[EI_MAG3]	   != ELFMAG3        ||
+		header.e_ident[EI_CLASS]   != elf_class	     ||
+		header.e_ident[EI_DATA]	   != elf_data       ||
+		header.e_ident[EI_VERSION] != EV_CURRENT     ||
+		header.e_type			   != ET_REL         ||
+		header.e_version		   != EV_CURRENT     ||
+		header.e_ehsize			   != sizeof(header) ||
+		header.e_shentsize		   != sizeof(ElfNativeSectionHeader))
+		return BAN::Error::from_errno(EINVAL);
+
+
+	BAN::Vector<LibELF::ElfNativeSectionHeader> secs(header.e_shnum);
+	TRY(read_from_user(u_image + header.e_shoff, secs.data(), secs.size() * sizeof(*secs.data())));
+	auto shstr = secs[header.e_shstrndx];
+
+	size_t total_size = 0;
+
+	LibELF::ElfNativeSectionHeader  *strtab = nullptr,
+									*symtab = nullptr,
+									*driver_section = nullptr;
+
+	for(auto& sec : secs) {
+		if(sec.sh_flags & LibELF::SHF_ALLOC) {
+			sec.sh_addr = total_size;
+			total_size += sec.sh_size;
+		}
+		if(sec.sh_name == 0) continue;
+		char name[256];
+		TRY(read_string_from_user(u_image + shstr.sh_offset + sec.sh_name, name, sizeof name));
+		BAN::StringView name_sv(name);
+
+		if(sec.sh_type == LibELF::SHT_SYMTAB) {
+			symtab = &sec;
+		}
+		// TODO: verify sh_type for both of these?
+		if(name_sv == ".strtab") {
+			strtab = &sec;
+		} else if(name_sv == ".banos-driver") {
+			driver_section = &sec;
+		}
+	}
+	if(!symtab || !strtab || !driver_section)
+		return BAN::Error::from_errno(EINVAL);
+	total_size += PAGE_SIZE;
+	total_size &= ~(PAGE_SIZE-1);
+	auto driver = TRY(VirtualRange::create_to_vaddr_range(PageTable::kernel(), { KERNEL_OFFSET, UINTPTR_MAX }, total_size, PageTable::Execute | PageTable::ReadWrite | PageTable::Present, true));
+
+	for(auto& sec : secs) {
+		if(sec.sh_flags & LibELF::SHF_ALLOC) {
+			sec.sh_addr += driver->vaddr();
+		}
+	}
+	Banos_Driver* banos_driver = reinterpret_cast<Banos_Driver*>(driver_section->sh_addr);
+	for(auto& sec : secs) {
+		if(sec.sh_name == 0) continue;
+		if(sec.sh_flags & LibELF::SHF_ALLOC) {
+			TRY(read_from_user(u_image + sec.sh_offset, reinterpret_cast<char*>(sec.sh_addr), sec.sh_size));
+		}
+		if(sec.sh_type == LibELF::SHT_RELA) {
+			auto& link_sec = secs[sec.sh_info];
+			size_t rela_count = sec.sh_size/sizeof(LibELF::ElfNativeRelocationA);
+			BAN::Vector<LibELF::ElfNativeRelocationA> rela_data(rela_count);
+			TRY(read_from_user(u_image + sec.sh_offset, rela_data.data(), rela_count * sizeof *rela_data.data()));
+
+			for(auto rela : rela_data) {
+				auto type   = ELF64_R_TYPE(rela.r_info);
+				auto symbol = ELF64_R_SYM(rela.r_info);
+
+				vaddr_t value = 0;
+
+				LibELF::ElfNativeSymbol sym;
+				TRY(read_from_user(u_image + symtab->sh_offset + sizeof(sym) * symbol, &sym, sizeof sym));
+
+				if(sym.st_shndx) {
+					value = secs[sym.st_shndx].sh_addr;
+				} else {
+					char name[256];
+					TRY(read_string_from_user(u_image + strtab->sh_offset + sym.st_name, name, sizeof name));
+
+					value = reinterpret_cast<vaddr_t>(Banos::resolve_symbol(name));
+					if(!value) {
+						derrorln("Failed to find symbol {}", name);
+						return BAN::Error::from_errno(ENOENT);
+					}
+				}
+				vaddr_t at = link_sec.sh_addr + rela.r_offset;
+				size_t size = 0;
+				switch(type) {
+				case LibELF::R_X86_64_PLT32:
+				case LibELF::R_X86_64_PC32:
+					value -= at;
+					// fallthrough
+				case LibELF::R_X86_64_32:
+				case LibELF::R_X86_64_32S:
+					value += rela.r_addend;
+					size = sizeof(uint32_t);
+					break;
+				case LibELF::R_X86_64_64:
+					value += rela.r_addend;
+					size = sizeof(uint64_t);
+					break;
+				default:
+					derrorln("TODO: Unsupported relocation type {}", type);
+					return BAN::Error::from_errno(ENOSYS);
+				}
+
+				switch(size) {
+				case 4: *reinterpret_cast<uint32_t*>(at) = value; break;
+				case 8: *reinterpret_cast<uint64_t*>(at) = value; break;
+				}
+			}
+		}
+	}
+	Driver_Instance instance;
+	instance.drv = banos_driver;
+	load_drv(instance.drv);
+	SpinLockGuard _(s_driver_instaces_lock);
+	TRY(s_driver_instaces.push_back(instance));
+	// TODO: import symbols and resolve redefintions :)
+	return s_driver_instaces.size() - 1;
+}
+// NOTE: should be more than plenty ;)
+extern char g_drv_builtin_begin[];
+extern char g_drv_builtin_end[];
+void Banos::initialize_initial_drivers(void) {
+	import_symbols(g_banos_export, g_banos_export_end - g_banos_export);
+	char* head = g_drv_builtin_begin;
+	while(head < g_drv_builtin_end) {
+		Banos_Driver* drv = (Banos_Driver*)head;
+		load_drv(drv);
+		head += drv->driver_size;
+	}
+}

kernel/kernel/Debug.cpp

@@ -7,6 +7,8 @@
 #include <kernel/Terminal/TTY.h>
 #include <kernel/Timer/Timer.h>
 
+#include <BAN/ScopeGuard.h>
+
 #include <LibDEFLATE/Compressor.h>
 #include <LibQR/QRCode.h>
 
@@ -14,6 +16,13 @@
 
 bool g_disable_debug = false;
 
+extern "C" bool safe_user_memcpy(void*, const void*, size_t);
+
+namespace Kernel
+{
+	extern bool g_safe_user_alloc_nonexisting;
+}
+
 namespace Debug
 {
 
@@ -45,48 +54,25 @@ namespace Debug
 
 		SpinLockGuard _(s_debug_lock);
 
-		const stackframe* frame = reinterpret_cast<const stackframe*>(bp);
+		const bool temp = g_safe_user_alloc_nonexisting;
-
+		g_safe_user_alloc_nonexisting = false;
-		void* first_ip = frame->ip;
+		BAN::ScopeGuard alloc_restore([temp] { g_safe_user_alloc_nonexisting = temp; });
-		void* last_ip = 0;
-		bool first = true;
 
 		BAN::Formatter::print(Debug::putchar, "\e[36mStack trace:\r\n");
 
 		if (ip != 0)
 			BAN::Formatter::print(Debug::putchar, "    {}\r\n", reinterpret_cast<void*>(ip));
 
-		while (frame)
+		stackframe frame;
+		if (!safe_user_memcpy(&frame, reinterpret_cast<void*>(bp), sizeof(stackframe)))
+			;
+		else for (size_t depth = 0; depth < 64; depth++)
 		{
-			if (!PageTable::is_valid_pointer((vaddr_t)frame))
+			BAN::Formatter::print(Debug::putchar, "    {}\r\n", reinterpret_cast<void*>(frame.ip));
-			{
+			if (frame.bp == nullptr || !safe_user_memcpy(&frame, frame.bp, sizeof(stackframe)))
-				derrorln("invalid pointer {H}", (vaddr_t)frame);
 				break;
 		}
 
-			if (PageTable::current().is_page_free((vaddr_t)frame & PAGE_ADDR_MASK))
-			{
-				derrorln("    {} not mapped", frame);
-				break;
-			}
-
-			BAN::Formatter::print(Debug::putchar, "    {}\r\n", (void*)frame->ip);
-
-			if (!first && frame->ip == first_ip)
-			{
-				derrorln("looping kernel panic :(");
-				break;
-			}
-			else if (!first && frame->ip == last_ip)
-			{
-				derrorln("repeating stack trace");
-				break;
-			}
-
-			last_ip = frame->ip;
-			frame = frame->bp;
-			first = false;
-		}
 		BAN::Formatter::print(Debug::putchar, "\e[m");
 	}
 

kernel/kernel/Device/Device.cpp

@@ -11,6 +11,7 @@ namespace Kernel
 			MUST(DevFileSystem::get().allocate_inode(create_inode_info(mode, uid, gid))),
 			create_inode_info(mode, uid, gid)
 		)
-	{ }
+	{
-
+		m_kind |= InodeKind::DEVICE;
+	}
 }

kernel/kernel/Device/FramebufferDevice.cpp

@@ -52,13 +52,14 @@ namespace Kernel
 	FramebufferDevice::FramebufferDevice(mode_t mode, uid_t uid, gid_t gid, dev_t rdev, paddr_t paddr, uint32_t width, uint32_t height, uint32_t pitch, uint8_t bpp)
 		: CharacterDevice(mode, uid, gid)
 		, m_name(MUST(BAN::String::formatted("fb{}", minor(rdev))))
-		, m_rdev(rdev)
 		, m_video_memory_paddr(paddr)
 		, m_width(width)
 		, m_height(height)
 		, m_pitch(pitch)
 		, m_bpp(bpp)
-	{ }
+	{
+		m_rdev = rdev;
+	}
 
 	FramebufferDevice::~FramebufferDevice()
 	{
@@ -84,10 +85,10 @@ namespace Kernel
 
 		m_video_buffer = TRY(VirtualRange::create_to_vaddr_range(
 			PageTable::kernel(),
-			KERNEL_OFFSET, UINTPTR_MAX,
+			{ KERNEL_OFFSET, UINTPTR_MAX },
 			BAN::Math::div_round_up<size_t>(m_width * m_height * (BANAN_FB_BPP / 8), PAGE_SIZE) * PAGE_SIZE,
 			PageTable::Flags::ReadWrite | PageTable::Flags::Present,
-			true, false
+			false
 		));
 
 		return {};
@@ -355,52 +356,8 @@ namespace Kernel
 
 		void do_msync(uint32_t first_pixel, uint32_t pixel_count)
 		{
-			if (!Processor::get_should_print_cpu_load())
-				return m_framebuffer->sync_pixels_linear(first_pixel, pixel_count);
-
-			const uint32_t fb_width = m_framebuffer->width();
-
-			// If we are here (in FramebufferMemoryRegion), our terminal driver is FramebufferTerminalDriver
-			ASSERT(g_terminal_driver->has_font());
-			const auto& font = g_terminal_driver->font();
-
-			const uint32_t x = first_pixel % fb_width;
-			const uint32_t y = first_pixel / fb_width;
-
-			const uint32_t load_w = 16                 * font.width();
-			const uint32_t load_h = Processor::count() * font.height();
-
-			if (y >= load_h || x + pixel_count <= fb_width - load_w)
-				return m_framebuffer->sync_pixels_linear(first_pixel, pixel_count);
-
-			if (x >= fb_width - load_w && x + pixel_count <= fb_width)
-				return;
-
-			if (x < fb_width - load_w)
-				m_framebuffer->sync_pixels_linear(first_pixel, fb_width - load_w - x);
-
-			if (x + pixel_count > fb_width)
-			{
-				const uint32_t past_last_pixel = first_pixel + pixel_count;
-
-				first_pixel = (y + 1) * fb_width;
-				pixel_count = past_last_pixel - first_pixel;
-
-				const uint32_t cpu_load_end = load_h * fb_width;
-
-				while (pixel_count && first_pixel < cpu_load_end)
-				{
-					m_framebuffer->sync_pixels_linear(first_pixel, BAN::Math::min(pixel_count, fb_width - load_w));
-
-					const uint32_t advance = BAN::Math::min(pixel_count, fb_width);
-					pixel_count -= advance;
-					first_pixel += advance;
-				}
-
-				if (pixel_count)
 			m_framebuffer->sync_pixels_linear(first_pixel, pixel_count);
 		}
-		}
 
 	private:
 		BAN::RefPtr<FramebufferDevice> m_framebuffer;

kernel/kernel/Epoll.cpp

@@ -14,9 +14,27 @@ namespace Kernel
 		return BAN::RefPtr<Epoll>::adopt(epoll_ptr);
 	}
 
+	Epoll::Epoll()
+	{
+		m_ino = 0;
+		m_mode = Mode::IRUSR | Mode::IWUSR;
+		m_nlink = 0;
+		m_uid = 0;
+		m_gid = 0;
+		m_size = 0;
+		m_atime = {};
+		m_mtime = {};
+		m_ctime = {};
+		m_blksize = PAGE_SIZE;
+		m_blocks = 0;
+		m_dev = 0;
+		m_rdev = 0;
+		m_kind = InodeKind::EPOLL;
+	}
+
 	Epoll::~Epoll()
 	{
-		for (auto [inode, _] : m_listening_events)
+		for (auto& [inode, _] : m_listening_events)
 			inode->del_epoll(this);
 	}
 
@@ -44,7 +62,7 @@ namespace Kernel
 
 				if (!contains_inode)
 					TRY(inode->add_epoll(this));
-				it->value.add_fd(fd, event);
+				TRY(it->value.add_fd(fd, event));
 
 				SpinLockGuard _(m_ready_lock);
 				auto ready_it = m_ready_events.find(inode);
@@ -144,9 +162,8 @@ namespace Kernel
 
 					{
 						uint32_t listen_mask = EPOLLHUP | EPOLLERR;
-						for (size_t fd = 0; fd < listen.events.size(); fd++)
+						for (const auto& [_, events] : listen.events)
-							if (listen.has_fd(fd))
+							listen_mask |= events.events;
-								listen_mask |= listen.events[fd].events;
 						events &= listen_mask;
 					}
 
@@ -154,8 +171,6 @@ namespace Kernel
 						REMOVE_IT();
 
 					{
-						LockGuard inode_locker(inode->m_mutex);
-
 #define CHECK_EVENT_BIT(mask, func) \
 						if ((events & mask) && !inode->func()) \
 							events &= ~mask;
@@ -171,11 +186,10 @@ namespace Kernel
 
 #undef REMOVE_IT
 
-					for (size_t fd = 0; fd < listen.events.size() && event_count < event_span.size(); fd++)
+					for (auto& [_, listen_event] : listen.events)
 					{
-						if (!listen.has_fd(fd))
+						if (event_count >= event_span.size())
-							continue;
+							break;
-						auto& listen_event = listen.events[fd];
 
 						const auto new_events = (listen_event.events | EPOLLHUP | EPOLLERR) & events;
 						if (new_events == 0)

kernel/kernel/FS/DevFS/FileSystem.cpp

@@ -102,7 +102,7 @@ namespace Kernel
 				uint64_t next_sync_ms { sync_interval_ms };
 				while (true)
 				{
-					LockGuard _(devfs->m_device_lock);
+					devfs->m_device_lock.lock();
 					while (!devfs->m_should_sync)
 					{
 						const uint64_t current_ms = SystemTimer::get().ms_since_boot();
@@ -110,12 +110,17 @@ namespace Kernel
 							break;
 						devfs->m_sync_thread_blocker.block_with_timeout_ms(next_sync_ms - current_ms, &devfs->m_device_lock);
 					}
-
+					BAN::Vector<BAN::RefPtr<StorageDevice>> storage_devices;
 					for (auto& device : devfs->m_devices)
 						if (device->is_storage_device())
-							if (auto ret = static_cast<StorageDevice*>(device.ptr())->sync_disk_cache(); ret.is_error())
+							MUST(storage_devices.push_back(static_cast<StorageDevice*>(device.ptr())));
+					devfs->m_device_lock.unlock();
+
+					for (auto& device : storage_devices)
+						if (auto ret = device->sync_disk_cache(); ret.is_error())
 							dwarnln("disk sync: {}", ret.error());
 
+					LockGuard _(devfs->m_device_lock);
 					next_sync_ms = SystemTimer::get().ms_since_boot() + sync_interval_ms;
 					devfs->m_should_sync = false;
 					devfs->m_sync_done.unblock();
@@ -148,7 +153,7 @@ namespace Kernel
 		MUST(static_cast<TmpDirectoryInode*>(root_inode().ptr())->link_inode(*device, device->name()));
 		MUST(m_devices.push_back(device));
 
-		dprintln("Added device /dev/{}", device->name());
+		dprintln_if(DEBUG_DEVFS, "Added device /dev/{}", device->name());
 	}
 
 	void DevFileSystem::remove_device(BAN::RefPtr<Device> device)
@@ -165,7 +170,7 @@ namespace Kernel
 			}
 		}
 
-		dprintln("Removed device /dev/{}", device->name());
+		dprintln_if(DEBUG_DEVFS, "Removed device /dev/{}", device->name());
 	}
 
 	void DevFileSystem::add_inode(BAN::StringView path, BAN::RefPtr<TmpInode> inode)

kernel/kernel/FS/EventFD.cpp

@@ -1,4 +1,5 @@
 #include <kernel/FS/EventFD.h>
+#include <kernel/Lock/LockGuard.h>
 
 #include <sys/epoll.h>
 
@@ -13,21 +14,44 @@ namespace Kernel
 		return BAN::RefPtr<Inode>(BAN::RefPtr<EventFD>::adopt(eventfd_ptr));
 	}
 
+	EventFD::EventFD(uint64_t initval, bool is_semaphore)
+		: m_is_semaphore(is_semaphore)
+		, m_value(initval)
+	{
+		m_ino = 0;
+		m_mode = Mode::IFCHR | Mode::IRUSR | Mode::IWUSR;
+		m_nlink = 0;
+		m_uid = 0;
+		m_gid = 0;
+		m_size = 0;
+		m_atime = {};
+		m_mtime = {};
+		m_ctime = {};
+		m_blksize = 8;
+		m_blocks = 0;
+		m_dev = 0;
+		m_rdev = 0;
+	}
+
 	BAN::ErrorOr<size_t> EventFD::read_impl(off_t, BAN::ByteSpan buffer)
 	{
 		if (buffer.size() < sizeof(uint64_t))
 			return BAN::Error::from_errno(EINVAL);
 
+		LockGuard _(m_mutex);
+
 		while (m_value == 0)
 			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &m_mutex));
 
-		const uint64_t read_value = m_is_semaphore ? 1 : m_value;
+		const uint64_t read_value = m_is_semaphore ? 1 : m_value.load();
 		m_value -= read_value;
 
 		buffer.as<uint64_t>() = read_value;
 
 		epoll_notify(EPOLLOUT);
 
+		m_thread_blocker.unblock();
+
 		return sizeof(uint64_t);
 	}
 
@@ -40,6 +64,8 @@ namespace Kernel
 		if (write_value == UINT64_MAX)
 			return BAN::Error::from_errno(EINVAL);
 
+		LockGuard _(m_mutex);
+
 		while (m_value + write_value < m_value)
 			TRY(Thread::current().block_or_eintr_indefinite(m_thread_blocker, &m_mutex));
 
@@ -48,6 +74,8 @@ namespace Kernel
 		if (m_value > 0)
 			epoll_notify(EPOLLIN);
 
+		m_thread_blocker.unblock();
+
 		return sizeof(uint64_t);
 	}
 

kernel/kernel/FS/Ext2/FileSystem.cpp

@@ -167,10 +167,36 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2FS::initialize_root_inode()
 	{
-		m_root_inode = TRY(Ext2Inode::create(*this, Ext2::Enum::ROOT_INO));
+		m_root_inode = TRY(open_inode(Ext2::Enum::ROOT_INO));
 		return {};
 	}
 
+	BAN::ErrorOr<BAN::RefPtr<Ext2Inode>> Ext2FS::open_inode(ino_t ino)
+	{
+		LockGuard _(m_inode_cache_lock);
+
+		auto it = m_inode_cache.find(ino);
+		if (it != m_inode_cache.end())
+			return it->value;
+
+		auto inode_location = TRY(locate_inode(ino));
+
+		auto block_buffer = TRY(get_block_buffer());
+		TRY(read_block(inode_location.block, block_buffer));
+
+		auto& inode = block_buffer.span().slice(inode_location.offset).as<Ext2::Inode>();
+
+		auto result = TRY(BAN::RefPtr<Ext2Inode>::create(*this, inode, ino));
+		TRY(m_inode_cache.insert(ino, result));
+		return result;
+	}
+
+	void Ext2FS::remove_from_cache(ino_t ino)
+	{
+		LockGuard _(m_inode_cache_lock);
+		m_inode_cache.remove(ino);
+	}
+
 	BAN::ErrorOr<uint32_t> Ext2FS::create_inode(const Ext2::Inode& ext2_inode)
 	{
 		auto bgd_buffer = TRY(m_buffer_manager.get_buffer());
@@ -281,7 +307,7 @@ namespace Kernel
 		auto& inode = inode_buffer.span().slice(inode_location.offset).as<Ext2::Inode>();
 #if EXT2_VERIFY_NO_BLOCKS
 		static const char zero_buffer[sizeof(inode.block)] {};
-		ASSERT(memcmp(inode.block, zero_buffer, sizeof(inode.block)) == 0);
+		ASSERT(memcmp(inode.block.block, zero_buffer, sizeof(inode.block)) == 0);
 #endif
 		bool is_directory = Inode::Mode(inode.mode).ifdir();
 		memset(&inode, 0x00, m_superblock.inode_size);
@@ -307,31 +333,27 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2FS::read_block(uint32_t block, BlockBufferWrapper& buffer)
 	{
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		const uint32_t block_size = this->block_size();
 		const uint32_t sectors_per_block = block_size / sector_size;
 
 		ASSERT(block >= superblock().first_data_block + 1);
 		ASSERT(buffer.size() >= block_size);
-		TRY(m_block_device->read_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 
+		TRY(m_block_device->read_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 		return {};
 	}
 
 	BAN::ErrorOr<void> Ext2FS::write_block(uint32_t block, const BlockBufferWrapper& buffer)
 	{
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		const uint32_t block_size = this->block_size();
 		const uint32_t sectors_per_block = block_size / sector_size;
 
 		ASSERT(block >= superblock().first_data_block + 1);
 		ASSERT(buffer.size() >= block_size);
-		TRY(m_block_device->write_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 
+		TRY(m_block_device->write_blocks(block * sectors_per_block, sectors_per_block, buffer.span()));
 		return {};
 	}
 
@@ -339,8 +361,6 @@ namespace Kernel
 	{
 		auto superblock_buffer = TRY(get_block_buffer());
 
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		ASSERT(1024 % sector_size == 0);
 
@@ -364,8 +384,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Ext2FS::sync_block(uint32_t block)
 	{
-		LockGuard _(m_mutex);
-
 		const uint32_t sector_size = m_block_device->blksize();
 		const uint32_t block_size = this->block_size();
 		const uint32_t sectors_per_block = block_size / sector_size;

kernel/kernel/FS/FAT/Inode.cpp

@@ -23,27 +23,30 @@ namespace Kernel
 		return BAN::to_unix_time(ban_time);
 	}
 
-	blksize_t FATInode::blksize() const
+	static timespec fat_date_to_timespec(FAT::Date date, FAT::Time time)
 	{
-		return m_fs.inode_block_size(this);
+		const time_t epoch = fat_date_to_epoch(date, time);
-	}
-
-	timespec FATInode::atime() const
-	{
-		const time_t epoch = fat_date_to_epoch(m_entry.last_access_date, {});
 		return timespec { .tv_sec = epoch, .tv_nsec = 0 };
 	}
 
-	timespec FATInode::mtime() const
+	FATInode::FATInode(FATFS& fs, const FAT::DirectoryEntry& entry, ino_t ino, uint32_t block_count)
+		: m_fs(fs)
+		, m_entry(entry)
+		, m_block_count(block_count)
 	{
-		const time_t epoch = fat_date_to_epoch(m_entry.write_date, m_entry.write_time);
+		m_ino = ino;
-		return timespec { .tv_sec = epoch, .tv_nsec = 0 };
+		m_mode = ((m_entry.attr & FAT::FileAttr::DIRECTORY) ? Mode::IFDIR : Mode::IFREG) | 0777;
-	}
+		m_nlink = 1;
-
+		m_uid = 0;
-	timespec FATInode::ctime() const
+		m_gid = 0;
-	{
+		m_size = m_entry.file_size;
-		const time_t epoch = fat_date_to_epoch(m_entry.creation_date, m_entry.creation_time);
+		m_blksize = fs.inode_block_size(this);
-		return timespec { .tv_sec = epoch, .tv_nsec = 0 };
+		m_atime = fat_date_to_timespec(m_entry.last_access_date, {});
+		m_mtime = fat_date_to_timespec(m_entry.write_date, m_entry.write_time);
+		m_ctime = fat_date_to_timespec(m_entry.creation_date, m_entry.creation_time);
+		m_blocks = m_block_count;
+		m_dev = 0;
+		m_rdev = 0;
 	}
 
 	const FileSystem* FATInode::filesystem() const

kernel/kernel/FS/Inode.cpp

@@ -5,6 +5,7 @@
 #include <kernel/Memory/FileBackedRegion.h>
 
 #include <fcntl.h>
+#include <sys/ioctl.h>
 #include <sys/statvfs.h>
 
 namespace Kernel
@@ -62,7 +63,6 @@ namespace Kernel
 
 	BAN::ErrorOr<BAN::RefPtr<Inode>> Inode::find_inode(BAN::StringView name)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		return find_inode_impl(name);
@@ -70,7 +70,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::list_next_inodes(off_t offset, struct dirent* list, size_t list_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		return list_next_inodes_impl(offset, list, list_len);
@@ -78,7 +77,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::create_file(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (Mode(mode).ifdir())
@@ -90,7 +88,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::create_directory(BAN::StringView name, mode_t mode, uid_t uid, gid_t gid)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (!Mode(mode).ifdir())
@@ -102,7 +99,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::link_inode(BAN::StringView name, BAN::RefPtr<Inode> inode)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (inode->mode().ifdir())
@@ -116,7 +112,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::rename_inode(BAN::RefPtr<Inode> old_parent, BAN::StringView old_name, BAN::StringView new_name)
 	{
-		LockGuard _(m_mutex);
 		if (!this->mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (!old_parent->mode().ifdir())
@@ -130,7 +125,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::unlink(BAN::StringView name)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifdir())
 			return BAN::Error::from_errno(ENOTDIR);
 		if (name == "."_sv || name == ".."_sv)
@@ -142,7 +136,6 @@ namespace Kernel
 
 	BAN::ErrorOr<BAN::String> Inode::link_target()
 	{
-		LockGuard _(m_mutex);
 		if (!mode().iflnk())
 			return BAN::Error::from_errno(EINVAL);
 		return link_target_impl();
@@ -150,7 +143,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::set_link_target(BAN::StringView target)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().iflnk())
 			return BAN::Error::from_errno(EINVAL);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
@@ -160,7 +152,6 @@ namespace Kernel
 
 	BAN::ErrorOr<long> Inode::accept(sockaddr* address, socklen_t* address_len, int flags)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return accept_impl(address, address_len, flags);
@@ -168,7 +159,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::bind(const sockaddr* address, socklen_t address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return bind_impl(address, address_len);
@@ -176,7 +166,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::connect(const sockaddr* address, socklen_t address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return connect_impl(address, address_len);
@@ -184,7 +173,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::listen(int backlog)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return listen_impl(backlog);
@@ -192,7 +180,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::recvmsg(msghdr& message, int flags)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return recvmsg_impl(message, flags);
@@ -200,7 +187,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::sendmsg(const msghdr& message, int flags)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return sendmsg_impl(message, flags);
@@ -208,7 +194,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::getsockname(sockaddr* address, socklen_t* address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return getsockname_impl(address, address_len);
@@ -216,7 +201,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::getpeername(sockaddr* address, socklen_t* address_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return getpeername_impl(address, address_len);
@@ -224,7 +208,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::getsockopt(int level, int option, void* value, socklen_t* value_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return getsockopt_impl(level, option, value, value_len);
@@ -232,7 +215,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::setsockopt(int level, int option, const void* value, socklen_t value_len)
 	{
-		LockGuard _(m_mutex);
 		if (!mode().ifsock())
 			return BAN::Error::from_errno(ENOTSOCK);
 		return setsockopt_impl(level, option, value, value_len);
@@ -240,7 +222,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::read(off_t offset, BAN::ByteSpan buffer)
 	{
-		LockGuard _(m_mutex);
 		if (mode().ifdir())
 			return BAN::Error::from_errno(EISDIR);
 		return read_impl(offset, buffer);
@@ -248,7 +229,6 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Inode::write(off_t offset, BAN::ConstByteSpan buffer)
 	{
-		LockGuard _(m_mutex);
 		if (mode().ifdir())
 			return BAN::Error::from_errno(EISDIR);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
@@ -258,7 +238,6 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::truncate(size_t size)
 	{
-		LockGuard _(m_mutex);
 		if (mode().ifdir())
 			return BAN::Error::from_errno(EISDIR);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
@@ -268,66 +247,93 @@ namespace Kernel
 
 	BAN::ErrorOr<void> Inode::chmod(mode_t mode)
 	{
-		ASSERT((mode & Inode::Mode::TYPE_MASK) == 0);
-		LockGuard _(m_mutex);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
 			return BAN::Error::from_errno(EROFS);
-		return chmod_impl(mode);
+
+		ASSERT((mode & Inode::Mode::TYPE_MASK) == 0);
+		mode |= m_mode & Inode::Mode::TYPE_MASK;
+
+		const auto old_mode = m_mode.exchange(mode);
+
+		if (auto ret = sync_inode(SyncType::Mode); ret.is_error())
+		{
+			m_mode.compare_exchange(mode, old_mode);
+			return ret.release_error();
+		}
+
+		return {};
 	}
 
 	BAN::ErrorOr<void> Inode::chown(uid_t uid, gid_t gid)
 	{
-		LockGuard _(m_mutex);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
 			return BAN::Error::from_errno(EROFS);
-		return chown_impl(uid, gid);
+
+		// TODO: unify uid and gid to a single atomic operation.
+		//       this needs 64 bit atomic support from 32 bit target
+
+		const auto old_uid = m_uid.exchange(uid);
+		const auto old_gid = m_gid.exchange(gid);
+
+		if (auto ret = sync_inode(SyncType::UidGid); ret.is_error())
+		{
+			m_uid.compare_exchange(uid, old_uid);
+			m_gid.compare_exchange(gid, old_gid);
+			return ret.release_error();
+		}
+
+		return {};
 	}
 
 	BAN::ErrorOr<void> Inode::utimens(const timespec times[2])
 	{
-		LockGuard _(m_mutex);
 		if (auto* fs = filesystem(); fs && (fs->flag() & ST_RDONLY))
 			return BAN::Error::from_errno(EROFS);
-		return utimens_impl(times);
+
+		// TODO: make these atomic
+
+		const auto old_atime = m_atime;
+		const auto old_mtime = m_mtime;
+
+		m_atime = times[0];
+		m_mtime = times[1];
+
+		if (auto ret = sync_inode(SyncType::Times); ret.is_error())
+		{
+			m_atime = old_atime;
+			m_mtime = old_mtime;
+			return ret.release_error();
+		}
+
+		return {};
 	}
 
 	BAN::ErrorOr<void> Inode::fsync()
 	{
-		LockGuard _(m_mutex);
+		// TODO: should we sync MAP_SHARED data?
-		if (auto shared = m_shared_region.lock())
+		TRY(sync_inode(SyncType::General));
-			for (size_t i = 0; i < shared->pages.size(); i++)
+		TRY(sync_data());
-				shared->sync(i);
+		return {};
-		return fsync_impl();
-	}
-
-	bool Inode::can_read() const
-	{
-		LockGuard _(m_mutex);
-		return can_read_impl();
-	}
-
-	bool Inode::can_write() const
-	{
-		LockGuard _(m_mutex);
-		return can_write_impl();
-	}
-
-	bool Inode::has_error() const
-	{
-		LockGuard _(m_mutex);
-		return has_error_impl();
-	}
-
-	bool Inode::has_hungup() const
-	{
-		LockGuard _(m_mutex);
-		return has_hungup_impl();
 	}
 
 	BAN::ErrorOr<long> Inode::ioctl(int request, void* arg)
 	{
-		LockGuard _(m_mutex);
+		auto ret = ioctl_impl(request, arg);
-		return ioctl_impl(request, arg);
+		if (!ret.is_error() || ret.error().get_error_code() != ENOTSUP)
+			return BAN::move(ret);
+
+		switch (request)
+		{
+			case TIOCGWINSZ:
+			case TIOCSWINSZ:
+			case TCGETS:
+			case TCSETS:
+			case TCSETSW:
+			case TCSETSF:
+				return BAN::Error::from_errno(EINVAL);
+			default:
+				return BAN::Error::from_errno(ENOTSUP);
+		}
 	}
 
 	BAN::ErrorOr<void> Inode::add_epoll(class Epoll* epoll)

kernel/kernel/FS/Pipe.cpp

@@ -1,3 +1,4 @@
+#include <BAN/HashMap.h>
 #include <kernel/FS/Pipe.h>
 #include <kernel/Lock/LockGuard.h>
 #include <kernel/Thread.h>
@@ -5,30 +6,131 @@
 
 #include <fcntl.h>
 #include <sys/epoll.h>
+#include <sys/ioctl.h>
 
 namespace Kernel
 {
 
 	static constexpr size_t s_pipe_buffer_size = 0x10000;
 
-	BAN::ErrorOr<BAN::RefPtr<Inode>> Pipe::create(const Credentials& credentials)
+	static Mutex s_named_pipe_mutex;
+	static BAN::HashMap<BAN::RefPtr<Inode>, BAN::WeakPtr<Pipe>> s_named_pipes;
+
+	BAN::ErrorOr<BAN::RefPtr<Inode>> Pipe::open(BAN::RefPtr<Inode> inode, int status_flags)
 	{
-		auto* pipe_ptr = new Pipe(credentials);
+		BAN::RefPtr<Pipe> pipe;
+
+		{
+			LockGuard _(s_named_pipe_mutex);
+
+			auto it = s_named_pipes.find(inode);
+			if (it == s_named_pipes.end())
+				it = TRY(s_named_pipes.insert(inode, {}));
+
+			if (!(pipe = it->value.lock()))
+			{
+				// FIXME: these should probably reference the underlying inode(?)
+				const struct stat st {
+					.st_dev     = inode->dev(),
+					.st_ino     = inode->ino(),
+					.st_mode    = inode->mode().mode,
+					.st_nlink   = inode->nlink(),
+					.st_uid     = inode->uid(),
+					.st_gid     = inode->gid(),
+					.st_rdev    = inode->rdev(),
+					.st_size    = inode->size(),
+					.st_atim    = inode->atime(),
+					.st_mtim    = inode->mtime(),
+					.st_ctim    = inode->ctime(),
+					.st_blksize = inode->blksize(),
+					.st_blocks  = inode->blocks(),
+				};
+
+				auto* pipe_ptr = new Pipe(st);
+				if (pipe_ptr == nullptr)
+					return BAN::Error::from_errno(ENOMEM);
+				pipe = BAN::RefPtr<Pipe>::adopt(pipe_ptr);
+				pipe->m_buffer = TRY(ByteRingBuffer::create(s_pipe_buffer_size));
+				pipe->m_named_inode = inode;
+
+				it->value = TRY(pipe->get_weak_ptr());
+			}
+		}
+
+		LockGuard _(pipe->m_mutex);
+
+		if (status_flags & O_RDONLY)
+			pipe->m_reading_count++;
+		if (status_flags & O_WRONLY)
+			pipe->m_writing_count++;
+
+		if (status_flags & O_NONBLOCK)
+		{
+			if ((status_flags & O_WRONLY) && pipe->m_writing_count == 0)
+				return BAN::Error::from_errno(ENXIO);
+			return BAN::RefPtr<Inode>(pipe);
+		}
+
+		auto& block_value = (status_flags & O_WRONLY) ? pipe->m_reading_count : pipe->m_writing_count;
+		while (block_value == 0)
+			TRY(Thread::current().block_or_eintr_indefinite(pipe->m_thread_blocker, &pipe->m_mutex));
+		return BAN::RefPtr<Inode>(pipe);
+	}
+
+	BAN::ErrorOr<BAN::RefPtr<Inode>> Pipe::create(uid_t uid, gid_t gid)
+	{
+		const timespec current_time = SystemTimer::get().real_time();
+		const struct stat st {
+			.st_dev     = 0, // FIXME
+			.st_ino     = 0, // FIXME
+			.st_mode    = Mode::IFIFO | Mode::IRUSR | Mode::IWUSR,
+			.st_nlink   = 0,
+			.st_uid     = uid,
+			.st_gid     = gid,
+			.st_rdev    = 0, // FIXME
+			.st_size    = 0,
+			.st_atim    = current_time,
+			.st_mtim    = current_time,
+			.st_ctim    = current_time,
+			.st_blksize = PAGE_SIZE,
+			.st_blocks  = 0,
+		};
+
+		auto* pipe_ptr = new Pipe(st);
 		if (pipe_ptr == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
 		auto pipe = BAN::RefPtr<Pipe>::adopt(pipe_ptr);
 		pipe->m_buffer = TRY(ByteRingBuffer::create(s_pipe_buffer_size));
+		pipe->m_reading_count++;
+		pipe->m_writing_count++;
 		return BAN::RefPtr<Inode>(pipe);
 	}
 
-	Pipe::Pipe(const Credentials& credentials)
+	Pipe::Pipe(const struct stat& st)
-		: m_uid(credentials.euid())
-		, m_gid(credentials.egid())
 	{
-		timespec current_time = SystemTimer::get().real_time();
+		m_ino     = st.st_ino;
-		m_atime = current_time;
+		m_mode    = st.st_mode;
-		m_mtime = current_time;
+		m_nlink   = st.st_nlink;
-		m_ctime = current_time;
+		m_uid     = st.st_uid;
+		m_gid     = st.st_gid;
+		m_size    = st.st_size;
+		m_atime   = st.st_atim;
+		m_mtime   = st.st_mtim;
+		m_ctime   = st.st_ctim;
+		m_blksize = st.st_blksize;
+		m_blocks  = st.st_blocks;
+		m_dev     = st.st_dev;
+		m_rdev    = st.st_rdev;
+
+		m_kind |= InodeKind::PIPE;
+	}
+
+	Pipe::~Pipe()
+	{
+		if (!m_named_inode)
+			return;
+		LockGuard _(s_named_pipe_mutex);
+		s_named_pipes.remove(m_named_inode);
 	}
 
 	void Pipe::on_clone(int status_flags)
@@ -71,8 +173,48 @@ namespace Kernel
 		m_thread_blocker.unblock();
 	}
 
+	BAN::ErrorOr<void> Pipe::sync_inode(SyncType type)
+	{
+		if (!m_named_inode)
+			return {};
+
+		switch (type)
+		{
+			case SyncType::General:
+				break;
+			case SyncType::Mode:
+				TRY(m_named_inode->chmod(m_mode));
+				break;
+			case SyncType::UidGid:
+				TRY(m_named_inode->chown(m_uid, m_gid));
+				break;
+			case SyncType::Times:
+				const timespec times[] { m_atime, m_mtime };
+				TRY(m_named_inode->utimens(times));
+				break;
+		}
+
+		m_mode = m_named_inode->mode().mode;
+
+		m_uid = m_named_inode->uid();
+		m_gid = m_named_inode->gid();
+
+		m_atime = m_named_inode->atime();
+		m_mtime = m_named_inode->mtime();
+		m_ctime = m_named_inode->ctime();
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> Pipe::sync_data()
+	{
+		return {};
+	}
+
 	BAN::ErrorOr<size_t> Pipe::read_impl(off_t, BAN::ByteSpan buffer)
 	{
+		LockGuard _(m_mutex);
+
 		while (m_buffer->empty())
 		{
 			if (m_writing_count == 0)
@@ -95,6 +237,8 @@ namespace Kernel
 
 	BAN::ErrorOr<size_t> Pipe::write_impl(off_t, BAN::ConstByteSpan buffer)
 	{
+		LockGuard _(m_mutex);
+
 		while (m_buffer->full())
 		{
 			if (m_reading_count == 0)
@@ -119,4 +263,9 @@ namespace Kernel
 		return to_copy;
 	}
 
+	BAN::ErrorOr<void> Pipe::truncate_impl(size_t)
+	{
+		return BAN::Error::from_errno(ENODEV);
+	}
+
 }

kernel/kernel/FS/ProcFS/FileSystem.cpp

@@ -16,22 +16,23 @@ namespace Kernel
 		MUST(s_instance->TmpFileSystem::initialize(0555, 0, 0));
 
 		auto meminfo_inode = MUST(ProcROInode::create_new(
-			[](off_t offset, BAN::ByteSpan buffer) -> size_t
+			[](off_t offset, BAN::ByteSpan buffer, void*) -> BAN::ErrorOr<size_t>
 			{
 				ASSERT(offset >= 0);
 				if ((size_t)offset >= sizeof(full_meminfo_t))
 					return 0;
 
-				full_meminfo_t meminfo;
+				const full_meminfo_t meminfo {
-				meminfo.page_size = PAGE_SIZE;
+					.page_size = PAGE_SIZE,
-				meminfo.free_pages = Heap::get().free_pages();
+					.free_pages = Heap::get().free_pages(),
-				meminfo.used_pages = Heap::get().used_pages();
+					.used_pages = Heap::get().used_pages(),
+				};
 
 				size_t bytes = BAN::Math::min<size_t>(sizeof(full_meminfo_t) - offset, buffer.size());
 				memcpy(buffer.data(), (uint8_t*)&meminfo + offset, bytes);
 				return bytes;
 			},
-			*s_instance, 0444, 0, 0
+			*s_instance, nullptr, 0444, 0, 0
 		));
 		MUST(static_cast<TmpDirectoryInode*>(s_instance->root_inode().ptr())->link_inode(*meminfo_inode, "meminfo"_sv));
 
@@ -48,6 +49,35 @@ namespace Kernel
 		MUST(static_cast<TmpDirectoryInode*>(s_instance->root_inode().ptr())->link_inode(*self_inode, "self"_sv));
 	}
 
+	void ProcFileSystem::post_scheduler_initialize()
+	{
+		MUST(s_instance->root_inode()->create_directory("cpu"_sv, Inode::Mode::IFDIR | 0555, 0, 0));
+		auto cpu_directory = MUST(s_instance->root_inode()->find_inode("cpu"_sv));
+
+		for (size_t i = 0; i < Processor::count(); i++)
+		{
+			auto cpu_inode = MUST(ProcROInode::create_new(
+				[](off_t offset, BAN::ByteSpan buffer, void* index_ptr) -> BAN::ErrorOr<size_t>
+				{
+					ASSERT(offset >= 0);
+
+					const size_t index = reinterpret_cast<uintptr_t>(index_ptr);
+					const auto load_stats = Processor::get_load_stats(index);
+
+					auto string = TRY(BAN::String::formatted("{} {}", load_stats.ns_idle, load_stats.ns_total));
+					if (static_cast<size_t>(offset) >= string.size())
+						return 0;
+
+					const size_t bytes = BAN::Math::min<size_t>(string.size() - offset, buffer.size());
+					memcpy(buffer.data(), string.data() + offset, bytes);
+					return bytes;
+				},
+				*s_instance, reinterpret_cast<void*>(i), 0444, 0, 0
+			));
+			MUST(cpu_directory->link_inode(MUST(BAN::String::formatted("{}", i)), cpu_inode));
+		}
+	}
+
 	ProcFileSystem& ProcFileSystem::get()
 	{
 		ASSERT(s_instance);