WindowServer: Fix xbanan access check

Instead of calculating bit-by-bit crc32, we now calculate a lookup table
during compile time. The old crc32 calculation was taking almost 50% of
the decompression time.

Also handle multiple symbols at once without outputting to user. It is
much more efficient to output many bytes instead of the up to 258 that a
single symbol can decode to :^)

.gitignore

@@ -3,3 +3,4 @@
 build/
 base/
 script/fakeroot-context
+tools/update-image-perms

BAN/include/BAN/Debug.h

@@ -9,29 +9,35 @@
 #include <BAN/Formatter.h>
 #include <stdio.h>
 
-#define __debug_putchar [](int c) { putc(c, stddbg); }
+#define __debug_putchar [](int c) { putc_unlocked(c, stddbg); }
 
 #define dprintln(...)											\
 	do {														\
+		flockfile(stddbg);										\
 		BAN::Formatter::print(__debug_putchar, __VA_ARGS__);	\
 		BAN::Formatter::print(__debug_putchar,"\n");			\
 		fflush(stddbg);											\
+		funlockfile(stddbg);									\
 	} while (false)
 
 #define dwarnln(...)											\
 	do {														\
+		flockfile(stddbg);										\
 		BAN::Formatter::print(__debug_putchar, "\e[33m");		\
 		BAN::Formatter::print(__debug_putchar, __VA_ARGS__);	\
 		BAN::Formatter::print(__debug_putchar, "\e[m\n");		\
 		fflush(stddbg);											\
+		funlockfile(stddbg);									\
 	} while(false)
 
 #define derrorln(...)											\
 	do {														\
+		flockfile(stddbg);										\
 		BAN::Formatter::print(__debug_putchar, "\e[31m");		\
 		BAN::Formatter::print(__debug_putchar, __VA_ARGS__);	\
 		BAN::Formatter::print(__debug_putchar, "\e[m\n");		\
 		fflush(stddbg);											\
+		funlockfile(stddbg);									\
 	} while(false)
 
 #define dprintln_if(cond, ...)		\

BAN/include/BAN/Heap.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include <BAN/Iteration.h>
+#include <BAN/Iterators.h>
 #include <BAN/Swap.h>
 #include <BAN/Traits.h>
 

BAN/include/BAN/IPv4.h

@@ -9,6 +9,10 @@ namespace BAN
 
 	struct IPv4Address
 	{
+		constexpr IPv4Address()
+			: IPv4Address(0)
+		{ }
+
 		constexpr IPv4Address(uint32_t u32_address)
 		{
 			raw = u32_address;

BAN/include/BAN/MacroUtils.h

@@ -0,0 +1,46 @@
+#pragma once
+
+#define _ban_count_args_impl(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, ...) _9
+#define _ban_count_args(...) _ban_count_args_impl(__VA_ARGS__ __VA_OPT__(,) 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)
+
+#define _ban_concat_impl(a, b) a##b
+#define _ban_concat(a, b) _ban_concat_impl(a, b)
+
+#define _ban_stringify_impl(x) #x
+#define _ban_stringify(x) _ban_stringify_impl(x)
+
+#define _ban_fe_0(f)
+#define _ban_fe_1(f, _0)                                 f(0, _0)
+#define _ban_fe_2(f, _0, _1)                             f(0, _0) f(1, _1)
+#define _ban_fe_3(f, _0, _1, _2)                         f(0, _0) f(1, _1) f(2, _2)
+#define _ban_fe_4(f, _0, _1, _2, _3)                     f(0, _0) f(1, _1) f(2, _2) f(3, _3)
+#define _ban_fe_5(f, _0, _1, _2, _3, _4)                 f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4)
+#define _ban_fe_6(f, _0, _1, _2, _3, _4, _5)             f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5)
+#define _ban_fe_7(f, _0, _1, _2, _3, _4, _5, _6)         f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5) f(6, _6)
+#define _ban_fe_8(f, _0, _1, _2, _3, _4, _5, _6, _7)     f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5) f(6, _6) f(7, _7)
+#define _ban_fe_9(f, _0, _1, _2, _3, _4, _5, _6, _7, _8) f(0, _0) f(1, _1) f(2, _2) f(3, _3) f(4, _4) f(5, _5) f(6, _6) f(7, _7) f(8, _8)
+#define _ban_for_each(f, ...) _ban_concat(_ban_fe_, _ban_count_args(__VA_ARGS__))(f __VA_OPT__(,) __VA_ARGS__)
+
+#define _ban_fe_comma_0(f)
+#define _ban_fe_comma_1(f, _0)                                 f(0, _0)
+#define _ban_fe_comma_2(f, _0, _1)                             f(0, _0), f(1, _1)
+#define _ban_fe_comma_3(f, _0, _1, _2)                         f(0, _0), f(1, _1), f(2, _2)
+#define _ban_fe_comma_4(f, _0, _1, _2, _3)                     f(0, _0), f(1, _1), f(2, _2), f(3, _3)
+#define _ban_fe_comma_5(f, _0, _1, _2, _3, _4)                 f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4)
+#define _ban_fe_comma_6(f, _0, _1, _2, _3, _4, _5)             f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5)
+#define _ban_fe_comma_7(f, _0, _1, _2, _3, _4, _5, _6)         f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5), f(6, _6)
+#define _ban_fe_comma_8(f, _0, _1, _2, _3, _4, _5, _6, _7)     f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5), f(6, _6), f(7, _7)
+#define _ban_fe_comma_9(f, _0, _1, _2, _3, _4, _5, _6, _7, _8) f(0, _0), f(1, _1), f(2, _2), f(3, _3), f(4, _4), f(5, _5), f(6, _6), f(7, _7), f(8, _8)
+#define _ban_for_each_comma(f, ...) _ban_concat(_ban_fe_comma_, _ban_count_args(__VA_ARGS__))(f __VA_OPT__(,) __VA_ARGS__)
+
+#define _ban_get_0(a0, ...)                                     a0
+#define _ban_get_1(a0, a1, ...)                                 a1
+#define _ban_get_2(a0, a1, a2, ...)                             a2
+#define _ban_get_3(a0, a1, a2, a3, ...)                         a3
+#define _ban_get_4(a0, a1, a2, a3, a4, ...)                     a4
+#define _ban_get_5(a0, a1, a2, a3, a4, a5, ...)                 a5
+#define _ban_get_6(a0, a1, a2, a3, a4, a5, a6, ...)             a6
+#define _ban_get_7(a0, a1, a2, a3, a4, a5, a6, a7, ...)         a7
+#define _ban_get_8(a0, a1, a2, a3, a4, a5, a6, a7, a8, ...)     a8
+#define _ban_get_9(a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, ...) a9
+#define _ban_get(n, ...) _ban_concat(_ban_get_, n)(__VA_ARGS__)

BAN/include/BAN/Math.h

@@ -36,12 +36,11 @@ namespace BAN::Math
 	template<integral T>
 	inline constexpr T gcd(T a, T b)
 	{
-		T t;
 		while (b)
 		{
-			t = b;
+			T temp = b;
 			b = a % b;
-			a = t;
+			a = temp;
 		}
 		return a;
 	}
@@ -66,25 +65,20 @@ namespace BAN::Math
 		return (x & (x - 1)) == 0;
 	}
 
-	template<BAN::integral T>
+	template<integral T>
-	static constexpr bool will_multiplication_overflow(T a, T b)
+	__attribute__((always_inline))
+	inline constexpr bool will_multiplication_overflow(T a, T b)
 	{
-		if (a == 0 || b == 0)
+		T dummy;
-			return false;
+		return __builtin_mul_overflow(a, b, &dummy);
-		if ((a > 0) == (b > 0))
-			return a > BAN::numeric_limits<T>::max() / b;
-		else
-			return a < BAN::numeric_limits<T>::min() / b;
 	}
 
-	template<BAN::integral T>
+	template<integral T>
-	static constexpr bool will_addition_overflow(T a, T b)
+	__attribute__((always_inline))
+	inline constexpr bool will_addition_overflow(T a, T b)
 	{
-		if (a > 0 && b > 0)
+		T dummy;
-			return a > BAN::numeric_limits<T>::max() - b;
+		return __builtin_add_overflow(a, b, &dummy);
-		if (a < 0 && b < 0)
-			return a < BAN::numeric_limits<T>::min() - b;
-		return false;
 	}
 
 	template<typename T>
@@ -98,6 +92,19 @@ namespace BAN::Math
 		return sizeof(T) * 8 - __builtin_clzll(x) - 1;
 	}
 
+// This is ugly but my clangd does not like including
+// intrinsic headers at all
+#if !defined(__SSE__) || !defined(__SSE2__)
+#pragma GCC push_options
+#ifndef __SSE__
+#pragma GCC target("sse")
+#endif
+#ifndef __SSE2__
+#pragma GCC target("sse2")
+#endif
+#define BAN_MATH_POP_OPTIONS
+#endif
+
 	template<floating_point T>
 	inline constexpr T floor(T x)
 	{
@@ -159,7 +166,23 @@ namespace BAN::Math
 			"jne 1b;"
 			: "+t"(a)
 			: "u"(b)
-			: "ax"
+			: "ax", "cc"
+		);
+		return a;
+	}
+
+	template<floating_point T>
+	inline constexpr T remainder(T a, T b)
+	{
+		asm(
+			"1:"
+			"fprem1;"
+			"fnstsw %%ax;"
+			"testb $4, %%ah;"
+			"jne 1b;"
+			: "+t"(a)
+			: "u"(b)
+			: "ax", "cc"
 		);
 		return a;
 	}
@@ -167,7 +190,7 @@ namespace BAN::Math
 	template<floating_point T>
 	static T modf(T x, T* iptr)
 	{
-		const T frac = BAN::Math::fmod<T>(x, 1);
+		const T frac = BAN::Math::fmod<T>(x, (T)1.0);
 		*iptr = x - frac;
 		return frac;
 	}
@@ -175,15 +198,15 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T frexp(T num, int* exp)
 	{
-		if (num == 0.0)
+		if (num == (T)0.0)
 		{
 			*exp = 0;
-			return 0.0;
+			return (T)0.0;
 		}
 
-		T _exp;
+		T e;
-		asm("fxtract" : "+t"(num), "=u"(_exp));
+		asm("fxtract" : "+t"(num), "=u"(e));
-		*exp = (int)_exp + 1;
+		*exp = (int)e + 1;
 		return num / (T)2.0;
 	}
 
@@ -251,6 +274,7 @@ namespace BAN::Math
 			"fstp %%st(1);"
 			: "+t"(x)
 		);
+
 		return x;
 	}
 
@@ -263,18 +287,9 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T pow(T x, T y)
 	{
-		asm(
+		if (x == (T)0.0)
-			"fyl2x;"
+			return (T)0.0;
-			"fld1;"
+		return exp2<T>(y * log2<T>(x));
-			"fld %%st(1);"
-			"fprem;"
-			"f2xm1;"
-			"faddp;"
-			"fscale;"
-			: "+t"(x), "+u"(y)
-		);
-
-		return x;
 	}
 
 	template<floating_point T>
@@ -310,16 +325,27 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T sqrt(T x)
 	{
-		asm("fsqrt" : "+t"(x));
+		if constexpr(BAN::is_same_v<T, float>)
-		return x;
+		{
+			using v4sf = float __attribute__((vector_size(16)));
+			return __builtin_ia32_sqrtss((v4sf) { x, 0.0f, 0.0f, 0.0f })[0];
+		}
+		else if constexpr(BAN::is_same_v<T, double>)
+		{
+			using v2df = double __attribute__((vector_size(16)));
+			return __builtin_ia32_sqrtsd((v2df) { x, 0.0 })[0];
+		}
+		else if constexpr(BAN::is_same_v<T, long double>)
+		{
+			asm("fsqrt" : "+t"(x));
+			return x;
+		}
 	}
 
 	template<floating_point T>
 	inline constexpr T cbrt(T value)
 	{
-		if (value == 0.0)
+		return pow<T>(value, (T)1.0 / (T)3.0);
-			return 0.0;
-		return pow<T>(value, 1.0 / 3.0);
 	}
 
 	template<floating_point T>
@@ -346,30 +372,21 @@ namespace BAN::Math
 	inline constexpr T tan(T x)
 	{
 		T one, ret;
-		asm(
+		asm("fptan" : "=t"(one), "=u"(ret) : "0"(x));
-			"fptan"
-			: "=t"(one), "=u"(ret)
-			: "0"(x)
-		);
 		return ret;
 	}
 
 	template<floating_point T>
 	inline constexpr T atan2(T y, T x)
 	{
-		asm(
+		asm("fpatan" : "+t"(x) : "u"(y) : "st(1)");
-			"fpatan"
-			: "+t"(x)
-			: "u"(y)
-			: "st(1)"
-		);
 		return x;
 	}
 
 	template<floating_point T>
 	inline constexpr T atan(T x)
 	{
-		return atan2<T>(x, 1.0);
+		return atan2<T>(x, (T)1.0);
 	}
 
 	template<floating_point T>
@@ -378,10 +395,10 @@ namespace BAN::Math
 		if (x == (T)0.0)
 			return (T)0.0;
 		if (x == (T)1.0)
-			return  numbers::pi_v<T> / (T)2.0;
+			return +numbers::pi_v<T> / (T)2.0;
 		if (x == (T)-1.0)
 			return -numbers::pi_v<T> / (T)2.0;
-		return (T)2.0 * atan<T>(x / (T(1.0) + sqrt<T>((T)1.0 - x * x)));
+		return (T)2.0 * atan<T>(x / ((T)1.0 + sqrt<T>((T)1.0 - x * x)));
 	}
 
 	template<floating_point T>
@@ -411,7 +428,7 @@ namespace BAN::Math
 	template<floating_point T>
 	inline constexpr T tanh(T x)
 	{
-		const T exp_px = exp<T>(x);
+		const T exp_px = exp<T>(+x);
 		const T exp_nx = exp<T>(-x);
 		return (exp_px - exp_nx) / (exp_px + exp_nx);
 	}
@@ -440,4 +457,9 @@ namespace BAN::Math
 		return sqrt<T>(x * x + y * y);
 	}
 
+#ifdef BAN_MATH_POP_OPTIONS
+#undef BAN_MATH_POP_OPTIONS
+#pragma GCC pop_options
+#endif
+
 }

BAN/include/BAN/New.h

@@ -9,10 +9,12 @@
 namespace BAN
 {
 	#if defined(__is_kernel)
-		static constexpr void*(&allocator)(size_t) = kmalloc;
+		static constexpr void*(*allocator)(size_t) = kmalloc;
-		static constexpr void(&deallocator)(void*) = kfree;
+		static constexpr void*(*reallocator)(void*, size_t) = nullptr;
+		static constexpr void(*deallocator)(void*) = kfree;
 	#else
-		static constexpr void*(&allocator)(size_t) = malloc;
+		static constexpr void*(*allocator)(size_t) = malloc;
-		static constexpr void(&deallocator)(void*) = free;
+		static constexpr void*(*reallocator)(void*, size_t) = realloc;
+		static constexpr void(*deallocator)(void*) = free;
 	#endif
 }

BAN/include/BAN/Span.h

@@ -69,7 +69,6 @@ namespace BAN
 
 		value_type* data() const
 		{
-			ASSERT(m_data);
 			return m_data;
 		}
 
@@ -84,7 +83,6 @@ namespace BAN
 
 		Span slice(size_type start, size_type length = ~size_type(0)) const
 		{
-			ASSERT(m_data);
 			ASSERT(start <= m_size);
 			if (length == ~size_type(0))
 				length = m_size - start;

BAN/include/BAN/String.h

@@ -14,6 +14,7 @@ namespace BAN
 	{
 	public:
 		using size_type = size_t;
+		using value_type = char;
 		using iterator = IteratorSimple<char, String>;
 		using const_iterator = ConstIteratorSimple<char, String>;
 		static constexpr size_type sso_capacity = 15;
@@ -352,10 +353,9 @@ namespace BAN::Formatter
 {
 
 	template<typename F>
-	void print_argument(F putc, const String& string, const ValueFormat&)
+	void print_argument(F putc, const String& string, const ValueFormat& format)
 	{
-		for (String::size_type i = 0; i < string.size(); i++)
+		print_argument(putc, string.sv(), format);
-			putc(string[i]);
 	}
 
 }

BAN/include/BAN/StringView.h

@@ -14,6 +14,7 @@ namespace BAN
 	{
 	public:
 		using size_type = size_t;
+		using value_type = char;
 		using const_iterator = ConstIteratorSimple<char, StringView>;
 
 	public:
@@ -246,10 +247,12 @@ namespace BAN::Formatter
 {
 
 	template<typename F>
-	void print_argument(F putc, const StringView& sv, const ValueFormat&)
+	void print_argument(F putc, const StringView& sv, const ValueFormat& format)
 	{
 		for (StringView::size_type i = 0; i < sv.size(); i++)
 			putc(sv[i]);
+		for (int i = sv.size(); i < format.fill; i++)
+			putc(' ');
 	}
 
 }

BAN/include/BAN/Traits.h

@@ -61,6 +61,9 @@ namespace BAN
 	template<typename T> struct is_move_constructible { static constexpr bool value = is_constructible_v<T, T&&>; };
 	template<typename T> inline constexpr bool is_move_constructible_v = is_move_constructible<T>::value;
 
+	template<typename T> struct is_trivially_copyable { static constexpr bool value = __is_trivially_copyable(T); };
+	template<typename T> inline constexpr bool is_trivially_copyable_v = is_trivially_copyable<T>::value;
+
 	template<typename T> struct is_integral { static constexpr bool value = requires (T t, T* p, void (*f)(T)) { reinterpret_cast<T>(t); f(0); p + t; }; };
 	template<typename T> inline constexpr bool is_integral_v = is_integral<T>::value;
 	template<typename T> concept integral = is_integral_v<T>;

BAN/include/BAN/Variant.h

@@ -126,14 +126,16 @@ namespace BAN
 		Variant(Variant&& other)
 			: m_index(other.m_index)
 		{
-			detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+			if (other.has_value())
+				detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 			other.clear();
 		}
 
 		Variant(const Variant& other)
 			: m_index(other.m_index)
 		{
-			detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+			if (other.has_value())
+				detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 		}
 
 		template<typename T>
@@ -157,12 +159,13 @@ namespace BAN
 
 		Variant& operator=(Variant&& other)
 		{
-			if (m_index == other.m_index)
+			if (m_index == other.m_index && m_index != invalid_index())
 				detail::move_assign<Ts...>(m_index, other.m_storage, m_storage);
 			else
 			{
 				clear();
-				detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+				if (other.has_value())
+					detail::move_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 				m_index = other.m_index;
 			}
 			other.clear();
@@ -171,12 +174,13 @@ namespace BAN
 
 		Variant& operator=(const Variant& other)
 		{
-			if (m_index == other.m_index)
+			if (m_index == other.m_index && m_index != invalid_index())
 				detail::copy_assign<Ts...>(m_index, other.m_storage, m_storage);
 			else
 			{
 				clear();
-				detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
+				if (other.has_value())
+					detail::copy_construct<Ts...>(other.m_index, other.m_storage, m_storage);
 				m_index = other.m_index;
 			}
 			return *this;

BAN/include/BAN/Vector.h

@@ -381,19 +381,46 @@ namespace BAN
 	template<typename T>
 	ErrorOr<void> Vector<T>::ensure_capacity(size_type size)
 	{
+		static_assert(alignof(T) <= alignof(max_align_t), "over aligned types not supported");
+
 		if (m_capacity >= size)
 			return {};
-		size_type new_cap = BAN::Math::max<size_type>(size, m_capacity * 2);
+
-		T* new_data = (T*)BAN::allocator(new_cap * sizeof(T));
+		const size_type new_cap = BAN::Math::max<size_type>(size, m_capacity * 2);
-		if (new_data == nullptr)
+
-			return Error::from_errno(ENOMEM);
+		if constexpr (BAN::is_trivially_copyable_v<T>)
-		for (size_type i = 0; i < m_size; i++)
 		{
-			new (new_data + i) T(move(m_data[i]));
+			if constexpr (BAN::reallocator)
-			m_data[i].~T();
+			{
+				auto* new_data = static_cast<T*>(BAN::reallocator(m_data, new_cap * sizeof(T)));
+				if (new_data == nullptr)
+					return Error::from_errno(ENOMEM);
+				m_data = new_data;
+			}
+			else
+			{
+				auto* new_data = static_cast<T*>(BAN::allocator(new_cap * sizeof(T)));
+				if (new_data == nullptr)
+					return Error::from_errno(ENOMEM);
+				memcpy(new_data, m_data, m_size * sizeof(T));
+				BAN::deallocator(m_data);
+				m_data = new_data;
+			}
 		}
-		BAN::deallocator(m_data);
+		else
-		m_data = new_data;
+		{
+			auto* new_data = static_cast<T*>(BAN::allocator(new_cap * sizeof(T)));
+			if (new_data == nullptr)
+				return Error::from_errno(ENOMEM);
+			for (size_type i = 0; i < m_size; i++)
+			{
+				new (new_data + i) T(move(m_data[i]));
+				m_data[i].~T();
+			}
+			BAN::deallocator(m_data);
+			m_data = new_data;
+		}
+
 		m_capacity = new_cap;
 		return {};
 	}

kernel/CMakeLists.txt

@@ -9,6 +9,8 @@ set(KERNEL_SOURCES
 	kernel/APIC.cpp
 	kernel/Audio/AC97/Controller.cpp
 	kernel/Audio/Controller.cpp
+	kernel/Audio/HDAudio/AudioFunctionGroup.cpp
+	kernel/Audio/HDAudio/Controller.cpp
 	kernel/BootInfo.cpp
 	kernel/CPUID.cpp
 	kernel/Credentials.cpp
@@ -23,6 +25,7 @@ set(KERNEL_SOURCES
 	kernel/Epoll.cpp
 	kernel/Errors.cpp
 	kernel/FS/DevFS/FileSystem.cpp
+	kernel/FS/EventFD.cpp
 	kernel/FS/Ext2/FileSystem.cpp
 	kernel/FS/Ext2/Inode.cpp
 	kernel/FS/FAT/FileSystem.cpp
@@ -48,6 +51,7 @@ set(KERNEL_SOURCES
 	kernel/InterruptController.cpp
 	kernel/kernel.cpp
 	kernel/Lock/SpinLock.cpp
+	kernel/Memory/ByteRingBuffer.cpp
 	kernel/Memory/DMARegion.cpp
 	kernel/Memory/FileBackedRegion.cpp
 	kernel/Memory/Heap.cpp
@@ -108,6 +112,7 @@ set(KERNEL_SOURCES
 	kernel/USB/Controller.cpp
 	kernel/USB/Device.cpp
 	kernel/USB/HID/HIDDriver.cpp
+	kernel/USB/HID/Joystick.cpp
 	kernel/USB/HID/Keyboard.cpp
 	kernel/USB/HID/Mouse.cpp
 	kernel/USB/Hub/HubDriver.cpp
@@ -134,6 +139,8 @@ if("${BANAN_ARCH}" STREQUAL "x86_64")
 		arch/x86_64/Signal.S
 		arch/x86_64/Syscall.S
 		arch/x86_64/Thread.S
+		arch/x86_64/User.S
+		arch/x86_64/Yield.S
 	)
 elseif("${BANAN_ARCH}" STREQUAL "i686")
 	set(KERNEL_SOURCES
@@ -144,6 +151,8 @@ elseif("${BANAN_ARCH}" STREQUAL "i686")
 		arch/i686/Signal.S
 		arch/i686/Syscall.S
 		arch/i686/Thread.S
+		arch/i686/User.S
+		arch/i686/Yield.S
 	)
 else()
 	message(FATAL_ERROR "unsupported architecure ${BANAN_ARCH}")
@@ -159,10 +168,7 @@ set(BAN_SOURCES
 set(KLIBC_SOURCES
 	klibc/ctype.cpp
 	klibc/string.cpp
-
+	klibc/arch/${BANAN_ARCH}/string.S
-	# Ehhh don't do this but for now libc uses the same stuff kernel can use
-	# This won't work after libc starts using sse implemetations tho
-	../userspace/libraries/LibC/arch/${BANAN_ARCH}/string.S
 )
 
 set(LIBDEFLATE_SOURCE

kernel/arch/i686/PageTable.cpp

@@ -21,6 +21,11 @@ namespace Kernel
 
 	SpinLock PageTable::s_fast_page_lock;
 
+	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
+	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
+
+	static bool s_is_post_heap_done = false;
+
 	static PageTable* s_kernel = nullptr;
 	static bool s_has_nxe = false;
 	static bool s_has_pge = false;
@@ -67,7 +72,7 @@ namespace Kernel
 
 	void PageTable::initialize_post_heap()
 	{
-		// NOTE: this is no-op as our 32 bit target does not use hhdm
+		s_is_post_heap_done = true;
 	}
 
 	void PageTable::initial_load()
@@ -141,9 +146,9 @@ namespace Kernel
 	}
 
 	template<typename T>
-	static vaddr_t P2V(const T paddr)
+	static uint64_t* P2V(const T paddr)
 	{
-		return (paddr_t)paddr - g_boot_info.kernel_paddr + KERNEL_OFFSET;
+		return reinterpret_cast<uint64_t*>(reinterpret_cast<paddr_t>(paddr) - g_boot_info.kernel_paddr + KERNEL_OFFSET);
 	}
 
 	void PageTable::initialize_kernel()
@@ -193,13 +198,18 @@ namespace Kernel
 	{
 		constexpr uint64_t pdpte = (fast_page() >> 30) & 0x1FF;
 		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
+		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
 		ASSERT(pdpt[pdpte] & Flags::Present);
 
-		uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte]) & PAGE_ADDR_MASK);
+		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 		ASSERT(!(pd[pde] & Flags::Present));
 		pd[pde] = V2P(allocate_zeroed_page_aligned_page()) | Flags::ReadWrite | Flags::Present;
+
+		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+		ASSERT(pt[pte] == 0);
+		pt[pte] = Flags::Reserved;
 	}
 
 	void PageTable::map_fast_page(paddr_t paddr)
@@ -214,14 +224,14 @@ namespace Kernel
 		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
 		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(s_kernel->m_highest_paging_struct));
+		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
-		uint64_t* pd   = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
+		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
 
 		ASSERT(!(pt[pte] & Flags::Present));
 		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
 
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	void PageTable::unmap_fast_page()
@@ -234,14 +244,14 @@ namespace Kernel
 		constexpr uint64_t pde   = (fast_page() >> 21) & 0x1FF;
 		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(s_kernel->m_highest_paging_struct));
+		uint64_t* pdpt = P2V(s_kernel->m_highest_paging_struct);
-		uint64_t* pd   = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
+		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
 
 		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = 0;
+		pt[pte] = Flags::Reserved;
 
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -263,7 +273,7 @@ namespace Kernel
 		m_highest_paging_struct = V2P(kmalloc(32, 32, true));
 		ASSERT(m_highest_paging_struct);
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
 		pdpt[0] = 0;
 		pdpt[1] = 0;
 		pdpt[2] = 0;
@@ -276,18 +286,17 @@ namespace Kernel
 		if (m_highest_paging_struct == 0)
 			return;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
-
 		for (uint32_t pdpte = 0; pdpte < 3; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+			uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 			for (uint32_t pde = 0; pde < 512; pde++)
 			{
 				if (!(pd[pde] & Flags::Present))
 					continue;
-				kfree(reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK)));
+				kfree(P2V(pd[pde] & s_page_addr_mask));
 			}
 			kfree(pd);
 		}
@@ -298,15 +307,43 @@ namespace Kernel
 	{
 		SpinLockGuard _(m_lock);
 		ASSERT(m_highest_paging_struct < 0x100000000);
-		const uint32_t pdpt_lo = m_highest_paging_struct;
+		asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
-		asm volatile("movl %0, %%cr3" :: "r"(pdpt_lo));
 		Processor::set_current_page_table(this);
 	}
 
-	void PageTable::invalidate(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::invalidate_range(vaddr_t vaddr, size_t pages, bool send_smp_message)
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
-		asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory");
+
+		const bool is_userspace = (vaddr < KERNEL_OFFSET);
+		if (is_userspace && this != &PageTable::current())
+			;
+		else if (pages <= 32 || !s_is_post_heap_done)
+		{
+			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
+				asm volatile("invlpg (%0)" :: "r"(vaddr));
+		}
+		else if (is_userspace || !s_has_pge)
+		{
+			asm volatile("movl %0, %%cr3" :: "r"(static_cast<uint32_t>(m_highest_paging_struct)));
+		}
+		else
+		{
+			asm volatile(
+				"movl %%cr4, %%eax;"
+
+				"andl $~0x80, %%eax;"
+				"movl %%eax, %%cr4;"
+
+				"movl %0, %%cr3;"
+
+				"orl $0x80, %%eax;"
+				"movl %%eax, %%cr4;"
+				:
+				: "r"(static_cast<uint32_t>(m_highest_paging_struct))
+				: "eax"
+			);
+		}
 
 		if (send_smp_message)
 		{
@@ -314,13 +351,14 @@ namespace Kernel
 				.type = Processor::SMPMessage::Type::FlushTLB,
 				.flush_tlb = {
 					.vaddr      = vaddr,
-					.page_count = 1
+					.page_count = pages,
+					.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
 				}
 			});
 		}
 	}
 
-	void PageTable::unmap_page(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr % PAGE_SIZE == 0);
@@ -339,12 +377,16 @@ namespace Kernel
 		if (is_page_free(vaddr))
 			Kernel::panic("trying to unmap unmapped page 0x{H}", vaddr);
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
-		uint64_t* pd   = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
-		uint64_t* pt   = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
+		uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
+
+		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
 
 		pt[pte] = 0;
-		invalidate(vaddr, send_smp_message);
+
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::unmap_range(vaddr_t vaddr, size_t size)
@@ -356,17 +398,10 @@ namespace Kernel
 		SpinLockGuard _(m_lock);
 		for (vaddr_t page = 0; page < page_count; page++)
 			unmap_page(vaddr + page * PAGE_SIZE, false);
-
+		invalidate_range(vaddr, page_count, true);
-		Processor::broadcast_smp_message({
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = page_count
-			}
-		});
 	}
 
-	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool send_smp_message)
+	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -401,11 +436,11 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		uint64_t* pdpt = P2V(m_highest_paging_struct);
 		if (!(pdpt[pdpte] & Flags::Present))
 			pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present;
 
-		uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 		if ((pd[pde] & uwr_flags) != uwr_flags)
 		{
 			if (!(pd[pde] & Flags::Present))
@@ -416,10 +451,14 @@ namespace Kernel
 		if (!(flags & Flags::Present))
 			uwr_flags &= ~Flags::Present;
 
-		uint64_t* pt = reinterpret_cast<uint64_t*>(P2V(pd[pde] & PAGE_ADDR_MASK));
+		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+
+		const paddr_t old_paddr = pt[pte] & s_page_addr_mask;
+
 		pt[pte] = paddr | uwr_flags | extra_flags;
 
-		invalidate(vaddr, send_smp_message);
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
@@ -433,14 +472,49 @@ namespace Kernel
 		SpinLockGuard _(m_lock);
 		for (size_t page = 0; page < page_count; page++)
 			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false);
+		invalidate_range(vaddr, page_count, true);
+	}
 
-		Processor::broadcast_smp_message({
+	void PageTable::remove_writable_from_range(vaddr_t vaddr, size_t size)
-			.type = Processor::SMPMessage::Type::FlushTLB,
+	{
-			.flush_tlb = {
+		ASSERT(vaddr);
-				.vaddr      = vaddr,
+		ASSERT(vaddr % PAGE_SIZE == 0);
-				.page_count = page_count
+
+		uint32_t pdpte = (vaddr >> 30) & 0x1FF;
+		uint32_t pde   = (vaddr >> 21) & 0x1FF;
+		uint32_t pte   = (vaddr >> 12) & 0x1FF;
+
+		const uint32_t e_pdpte = ((vaddr + size - 1) >> 30) & 0x1FF;
+		const uint32_t e_pde   = ((vaddr + size - 1) >> 21) & 0x1FF;
+		const uint32_t e_pte   = ((vaddr + size - 1) >> 12) & 0x1FF;
+
+		SpinLockGuard _(m_lock);
+
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
+		for (; pdpte <= e_pdpte; pdpte++)
+		{
+			if (!(pdpt[pdpte] & Flags::Present))
+				continue;
+			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+			for (; pde < 512; pde++)
+			{
+				if (pdpte == e_pdpte && pde > e_pde)
+					break;
+				if (!(pd[pde] & Flags::ReadWrite))
+					continue;
+				uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+				for (; pte < 512; pte++)
+				{
+					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
+						break;
+					pt[pte] &= ~static_cast<uint64_t>(Flags::ReadWrite);
+				}
+				pte = 0;
 			}
-		});
+			pde = 0;
+		}
+
+		invalidate_range(vaddr, size / PAGE_SIZE, true);
 	}
 
 	uint64_t PageTable::get_page_data(vaddr_t vaddr) const
@@ -453,15 +527,15 @@ namespace Kernel
 
 		SpinLockGuard _(m_lock);
 
-		uint64_t* pdpt = (uint64_t*)P2V(m_highest_paging_struct);
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
 		if (!(pdpt[pdpte] & Flags::Present))
 			return 0;
 
-		uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK);
+		const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 		if (!(pd[pde] & Flags::Present))
 			return 0;
 
-		uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK);
+		const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
 		if (!(pt[pte] & Flags::Used))
 			return 0;
 
@@ -475,8 +549,7 @@ namespace Kernel
 
 	paddr_t PageTable::physical_address_of(vaddr_t vaddr) const
 	{
-		uint64_t page_data = get_page_data(vaddr);
+		return get_page_data(vaddr) & s_page_addr_mask;
-		return (page_data & PAGE_ADDR_MASK) & ~(1ull << 63);
 	}
 
 	bool PageTable::is_page_free(vaddr_t vaddr) const
@@ -518,13 +591,8 @@ namespace Kernel
 			return false;
 		for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
 			reserve_page(vaddr + offset, true, false);
-		Processor::broadcast_smp_message({
+		invalidate_range(vaddr, bytes / PAGE_SIZE, true);
-			.type = Processor::SMPMessage::Type::FlushTLB,
+
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = bytes / PAGE_SIZE,
-			}
-		});
 		return true;
 	}
 
@@ -537,48 +605,47 @@ namespace Kernel
 		if (size_t rem = last_address % PAGE_SIZE)
 			last_address -= rem;
 
-		const uint32_t s_pdpte = (first_address >> 30) & 0x1FF;
+		uint32_t pdpte = (first_address >> 30) & 0x1FF;
-		const uint32_t s_pde   = (first_address >> 21) & 0x1FF;
+		uint32_t pde   = (first_address >> 21) & 0x1FF;
-		const uint32_t s_pte   = (first_address >> 12) & 0x1FF;
+		uint32_t pte   = (first_address >> 12) & 0x1FF;
 
-		const uint32_t e_pdpte = (last_address >> 30) & 0x1FF;
+		const uint32_t e_pdpte = ((last_address - 1) >> 30) & 0x1FF;
-		const uint32_t e_pde   = (last_address >> 21) & 0x1FF;
+		const uint32_t e_pde   = ((last_address - 1) >> 21) & 0x1FF;
-		const uint32_t e_pte   = (last_address >> 12) & 0x1FF;
+		const uint32_t e_pte   = ((last_address - 1) >> 12) & 0x1FF;
 
 		SpinLockGuard _(m_lock);
 
 		// Try to find free page that can be mapped without
 		// allocations (page table with unused entries)
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
-		for (uint32_t pdpte = s_pdpte; pdpte < 4; pdpte++)
+		for (; pdpte <= e_pdpte; pdpte++)
 		{
-			if (pdpte > e_pdpte)
-				break;
 			if (!(pdpt[pdpte] & Flags::Present))
 				continue;
-			uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte] & PAGE_ADDR_MASK));
+			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
-			for (uint32_t pde = s_pde; pde < 512; pde++)
+			for (; pde < 512; pde++)
 			{
 				if (pdpte == e_pdpte && pde > e_pde)
 					break;
 				if (!(pd[pde] & Flags::Present))
 					continue;
-				uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK);
+				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
-				for (uint32_t pte = s_pte; pte < 512; pte++)
+				for (; pte < 512; pte++)
 				{
-					if (pdpte == e_pdpte && pde == e_pde && pte >= e_pte)
+					if (pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 						break;
-					if (!(pt[pte] & Flags::Used))
+					if (pt[pte] & Flags::Used)
-					{
+						continue;
-						vaddr_t vaddr = 0;
+					vaddr_t vaddr = 0;
-						vaddr |= (vaddr_t)pdpte << 30;
+					vaddr |= (vaddr_t)pdpte << 30;
-						vaddr |= (vaddr_t)pde   << 21;
+					vaddr |= (vaddr_t)pde   << 21;
-						vaddr |= (vaddr_t)pte   << 12;
+					vaddr |= (vaddr_t)pte   << 12;
-						ASSERT(reserve_page(vaddr));
+					ASSERT(reserve_page(vaddr));
-						return vaddr;
+					return vaddr;
-					}
 				}
+				pte = 0;
 			}
+			pde = 0;
 		}
 
 		// Find any free page
@@ -591,7 +658,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	vaddr_t PageTable::reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address)
@@ -624,7 +691,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	static void dump_range(vaddr_t start, vaddr_t end, PageTable::flags_t flags)
@@ -647,7 +714,7 @@ namespace Kernel
 		flags_t flags = 0;
 		vaddr_t start = 0;
 
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
+		const uint64_t* pdpt = P2V(m_highest_paging_struct);
 		for (uint32_t pdpte = 0; pdpte < 4; pdpte++)
 		{
 			if (!(pdpt[pdpte] & Flags::Present))
@@ -656,7 +723,7 @@ namespace Kernel
 				start = 0;
 				continue;
 			}
-			uint64_t* pd = (uint64_t*)P2V(pdpt[pdpte] & PAGE_ADDR_MASK);
+			const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 			for (uint64_t pde = 0; pde < 512; pde++)
 			{
 				if (!(pd[pde] & Flags::Present))
@@ -665,7 +732,7 @@ namespace Kernel
 					start = 0;
 					continue;
 				}
-				uint64_t* pt = (uint64_t*)P2V(pd[pde] & PAGE_ADDR_MASK);
+				const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
 				for (uint64_t pte = 0; pte < 512; pte++)
 				{
 					if (parse_flags(pt[pte]) != flags)

kernel/arch/i686/Signal.S

@@ -1,12 +1,13 @@
 .section .userspace, "ax"
 
 // stack contains
-//    return address
+//    (4 bytes)  return address (on return stack)
-//    return stack
+//    (4 bytes)  return stack
-//    return rflags
+//    (4 bytes)  return rflags
-//    siginfo_t
+//    (8 bytes)  restore sigmask
-//    signal number
+//    (36 bytes) siginfo_t
-//    signal handler
+//    (4 bytes)  signal number
+//    (4 bytes)  signal handler
 
 .global signal_trampoline
 signal_trampoline:
@@ -18,11 +19,9 @@ signal_trampoline:
 	pushl %eax
 	pushl %ebp
 
-	movl 76(%esp), %eax // return sp
+	movl 80(%esp), %eax
-	addl $4, %eax       //   return address
+	pushl %eax; addl $4, (%esp)
-	movl 80(%esp), %ebx // return ip
+	pushl (%eax)
-	pushl %eax;
-	pushl %ebx
 
 	// FIXME: populate these
 	xorl %eax, %eax
@@ -57,9 +56,17 @@ signal_trampoline:
 
 	// restore stack
 	movl %ebp, %esp
-	addl $32, %esp
+	addl $24, %esp
+
+	// restore sigmask
+	movl $83, %eax      // SYS_SIGPROCMASK
+	movl $3,  %ebx      // SIG_SETMASK
+	leal 72(%esp), %ecx // set
+	xorl %edx, %edx     // oset
+	int $0xF0
 
 	// restore registers
+	addl $8, %esp
 	popl %ebp
 	popl %eax
 	popl %ebx
@@ -68,8 +75,8 @@ signal_trampoline:
 	popl %edi
 	popl %esi
 
-	// skip handler, number, siginfo_t
+	// skip handler, number, siginfo_t, sigmask
-	addl $44, %esp
+	addl $52, %esp
 
 	// restore flags
 	popf

kernel/arch/i686/Syscall.S

@@ -15,9 +15,7 @@ asm_syscall_handler:
 	andl $-16, %esp
 
 	# push arguments
-	subl $4, %esp
+	subl $8, %esp
-	pushl %ebp
-	addl $24, (%esp)
 	pushl %edi
 	pushl %esi
 	pushl %edx
@@ -65,7 +63,7 @@ sys_fork_trampoline:
 
 	call read_ip
 	testl %eax, %eax
-	jz .reload_stack
+	jz .done
 
 	movl %esp, %ebx
 
@@ -81,9 +79,3 @@ sys_fork_trampoline:
 	popl %ebx
 	popl %ebp
 	ret
-
- .reload_stack:
-	call get_thread_start_sp
-	movl %eax, %esp
-	xorl %eax, %eax
-	jmp .done

kernel/arch/i686/Thread.S

@@ -7,9 +7,6 @@ read_ip:
 # void start_kernel_thread()
 .global start_kernel_thread
 start_kernel_thread:
-	call get_thread_start_sp
-	movl %eax, %esp
-
 	# STACK LAYOUT
 	#   on_exit arg
 	#   on_exit func
@@ -34,11 +31,6 @@ start_kernel_thread:
 
 .global start_userspace_thread
 start_userspace_thread:
-	call load_thread_sse
-
-	call get_thread_start_sp
-	movl %eax, %esp
-
 	movw $(0x20 | 3), %bx
 	movw %bx, %ds
 	movw %bx, %es

kernel/arch/i686/User.S

@@ -0,0 +1,54 @@
+# bool safe_user_memcpy(void*, const void*, size_t)
+.global safe_user_memcpy
+.global safe_user_memcpy_end
+.global safe_user_memcpy_fault
+safe_user_memcpy:
+	xorl %eax, %eax
+	xchgl 4(%esp), %edi
+	xchgl 8(%esp), %esi
+	movl 12(%esp), %ecx
+	movl %edi, %edx
+	rep movsb
+	movl 4(%esp), %edi
+	movl 8(%esp), %esi
+	incl %eax
+safe_user_memcpy_fault:
+	ret
+safe_user_memcpy_end:
+
+# bool safe_user_strncpy(void*, const void*, size_t)
+.global safe_user_strncpy
+.global safe_user_strncpy_end
+.global safe_user_strncpy_fault
+safe_user_strncpy:
+	xchgl 4(%esp), %edi
+	xchgl 8(%esp), %esi
+	movl 12(%esp), %ecx
+
+	testl %ecx, %ecx
+	jz safe_user_strncpy_fault
+
+ .safe_user_strncpy_loop:
+	movb (%esi), %al
+	movb %al, (%edi)
+	testb %al, %al
+	jz .safe_user_strncpy_done
+
+	incl %edi
+	incl %esi
+	decl %ecx
+	jnz .safe_user_strncpy_loop
+
+ safe_user_strncpy_fault:
+	xorl %eax, %eax
+	jmp .safe_user_strncpy_return
+
+ .safe_user_strncpy_done:
+	movl $1, %eax
+
+ .safe_user_strncpy_return:
+	movl 4(%esp), %edi
+	movl 8(%esp), %esi
+	ret
+
+safe_user_strncpy_end:

kernel/arch/i686/Yield.S

@@ -0,0 +1,25 @@
+.global asm_yield_trampoline
+asm_yield_trampoline:
+	leal 4(%esp), %ecx
+	movl 4(%esp), %esp
+
+	pushl -4(%ecx)
+	pushl %ecx
+	pushl %eax
+	pushl %ebx
+	pushl %esi
+	pushl %edi
+	pushl %ebp
+
+	pushl %esp
+	call scheduler_on_yield
+	addl $4, %esp
+
+	popl %ebp
+	popl %edi
+	popl %esi
+	popl %ebx
+	popl %eax
+	movl 4(%esp), %ecx
+	movl 0(%esp), %esp
+	jmp *%ecx

kernel/arch/i686/boot.S

@@ -11,13 +11,14 @@
 
 .code32
 
-# multiboot2 header
+// video mode info, page align modules
+.set multiboot_flags, (1 << 2) | (1 << 0)
+
 .section .multiboot, "aw"
-	.align 8
 multiboot_start:
 	.long 0x1BADB002
-	.long (1 << 2) # page align modules
+	.long multiboot_flags
-	.long -(0x1BADB002 + (1 << 2))
+	.long -(0x1BADB002 + multiboot_flags)
 
 	.long 0
 	.long 0
@@ -30,7 +31,8 @@ multiboot_start:
 	.long FB_HEIGHT
 	.long FB_BPP
 multiboot_end:
-	.align 8
+
+.section .multiboot2, "aw"
 multiboot2_start:
 	.long 0xE85250D6
 	.long 0
@@ -66,7 +68,6 @@ multiboot2_start:
 multiboot2_end:
 
 .section .bananboot, "aw"
-	.align 8
 bananboot_start:
 	.long 0xBABAB007
 	.long -(0xBABAB007 + FB_WIDTH + FB_HEIGHT + FB_BPP)
@@ -184,6 +185,13 @@ enable_sse:
 	movl %eax, %cr4
 	ret
 
+enable_tsc:
+	# allow userspace to use RDTSC
+	movl %cr4, %ecx
+	andl $0xFFFFFFFB, %ecx
+	movl %ecx, %cr4
+	ret
+
 initialize_paging:
 	# enable PAE
 	movl %cr4, %ecx
@@ -226,6 +234,7 @@ gdt_flush:
 	# do processor initialization
 	call check_requirements
 	call enable_sse
+	call enable_tsc
 	call initialize_paging
 
 	# load higher half stack pointer
@@ -302,6 +311,7 @@ ap_protected_mode:
 	movb $1, AP_V2P(ap_stack_loaded)
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
+	leal V2P(enable_tsc),        %ecx; call *%ecx
 	leal V2P(initialize_paging), %ecx; call *%ecx
 
 	# load boot gdt and enter long mode

kernel/arch/i686/interrupts.S

@@ -1,21 +1,20 @@
-.macro maybe_load_kernel_segments, n
+.macro intr_header, n
-	cmpb $0x08, \n(%esp)
+	pushal
-	je 1f
+	testb $3, \n+8*4(%esp)
-
+	jz 1f
 	movw $0x10, %ax
 	movw %ax, %ds
 	movw %ax, %es
 	movw %ax, %fs
-
 	movw $0x28, %ax
 	movw %ax, %gs
-1:
+1:	cld
 .endm
 
-.macro maybe_load_userspace_segments, n
+.macro intr_footer, n
-	cmpb $0x08, \n(%esp)
+	testb $3, \n+8*4(%esp)
-	je 1f
+	jz 1f
-
+	call cpp_check_signal
 	movw $(0x20 | 3), %bx
 	movw %bx, %ds
 	movw %bx, %es
@@ -23,14 +22,11 @@
 	movw %bx, %fs
 	movw $(0x38 | 3), %bx
 	movw %bx, %gs
-1:
+1:	popal
 .endm
 
 isr_stub:
-	pushal
+	intr_header 12
-	maybe_load_kernel_segments 44
-	cld
-
 	movl %cr0, %eax; pushl %eax
 	movl %cr2, %eax; pushl %eax
 	movl %cr3, %eax; pushl %eax
@@ -58,15 +54,12 @@ isr_stub:
 	movl %ebp, %esp
 	addl $24, %esp
 
-	maybe_load_userspace_segments 44
+	intr_footer 12
-	popal
 	addl $8, %esp
 	iret
 
 irq_stub:
-	pushal
+	intr_header 12
-	maybe_load_kernel_segments 44
-	cld
 
 	movl 32(%esp), %edi # interrupt number
 
@@ -79,38 +72,13 @@ irq_stub:
 
 	movl %ebp, %esp
 
-	maybe_load_userspace_segments 44
+	intr_footer 12
-	popal
 	addl $8, %esp
 	iret
 
-.global asm_yield_handler
-asm_yield_handler:
-	# This can only be called from kernel, so no segment saving is needed
-	pushal
-	cld
-
-	leal 32(%esp), %edi # interrupt stack ptr
-	movl %esp,     %esi # interrupt registers ptr
-
-	movl %esp, %ebp
-	andl $-16, %esp
-
-	subl $8, %esp
-	pushl %esi
-	pushl %edi
-	call cpp_yield_handler
-
-	movl %ebp, %esp
-
-	popal
-	iret
-
 .global asm_ipi_handler
 asm_ipi_handler:
-	pushal
+	intr_header 4
-	maybe_load_kernel_segments 36
-	cld
 
 	movl %esp, %ebp
 	andl $-16, %esp
@@ -119,15 +87,12 @@ asm_ipi_handler:
 
 	movl %ebp, %esp
 
-	maybe_load_userspace_segments 36
+	intr_footer 4
-	popal
 	iret
 
 .global asm_timer_handler
 asm_timer_handler:
-	pushal
+	intr_header 4
-	maybe_load_kernel_segments 36
-	cld
 
 	movl %esp, %ebp
 	andl $-16, %esp
@@ -136,8 +101,7 @@ asm_timer_handler:
 
 	movl %ebp, %esp
 
-	maybe_load_userspace_segments 36
+	intr_footer 4
-	popal
 	iret
 
 .macro isr n

kernel/arch/i686/linker.ld

@@ -11,6 +11,7 @@ SECTIONS
 	{
 		g_kernel_execute_start = .;
 		*(.multiboot)
+		*(.multiboot2)
 		*(.bananboot)
 		*(.text.*)
 	}

kernel/arch/x86_64/PageTable.cpp

@@ -23,7 +23,7 @@ namespace Kernel
 	SpinLock PageTable::s_fast_page_lock;
 
 	static constexpr vaddr_t s_hhdm_offset = 0xFFFF800000000000;
-	static bool s_is_hddm_initialized = false;
+	static bool s_is_post_heap_done = false;
 
 	constexpr uint64_t s_page_flag_mask = 0x8000000000000FFF;
 	constexpr uint64_t s_page_addr_mask = ~s_page_flag_mask;
@@ -376,7 +376,7 @@ namespace Kernel
 		V2P                               = &FuncsHHDM::V2P;
 		P2V                               = &FuncsHHDM::P2V;
 
-		s_is_hddm_initialized = true;
+		s_is_post_heap_done = true;
 
 		// This is a hack to unmap fast page. fast page pt is copied
 		// while it is mapped, so we need to manually unmap it
@@ -485,6 +485,7 @@ namespace Kernel
 		constexpr uint64_t pml4e = (uc_vaddr >> 39) & 0x1FF;
 		constexpr uint64_t pdpte = (uc_vaddr >> 30) & 0x1FF;
 		constexpr uint64_t pde   = (uc_vaddr >> 21) & 0x1FF;
+		constexpr uint64_t pte   = (uc_vaddr >> 12) & 0x1FF;
 
 		uint64_t* pml4 = P2V(m_highest_paging_struct);
 		ASSERT(!(pml4[pml4e] & Flags::Present));
@@ -497,6 +498,10 @@ namespace Kernel
 		uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
 		ASSERT(!(pd[pde] & Flags::Present));
 		pd[pde] = allocate_zeroed_page_aligned_page() | Flags::ReadWrite | Flags::Present;
+
+		uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+		ASSERT(pt[pte] == 0);
+		pt[pte] = Flags::Reserved;
 	}
 
 	void PageTable::map_fast_page(paddr_t paddr)
@@ -521,7 +526,7 @@ namespace Kernel
 		ASSERT(!(pt[pte] & Flags::Present));
 		pt[pte] = paddr | Flags::ReadWrite | Flags::Present;
 
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	void PageTable::unmap_fast_page()
@@ -542,9 +547,9 @@ namespace Kernel
 		      uint64_t* pt   = P2V(pd[pde] & s_page_addr_mask);
 
 		ASSERT(pt[pte] & Flags::Present);
-		pt[pte] = 0;
+		pt[pte] = Flags::Reserved;
 
-		invalidate(fast_page(), false);
+		asm volatile("invlpg (%0)" :: "r"(fast_page()) : "memory");
 	}
 
 	BAN::ErrorOr<PageTable*> PageTable::create_userspace()
@@ -612,10 +617,39 @@ namespace Kernel
 		Processor::set_current_page_table(this);
 	}
 
-	void PageTable::invalidate(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::invalidate_range(vaddr_t vaddr, size_t pages, bool send_smp_message)
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
-		asm volatile("invlpg (%0)" :: "r"(vaddr) : "memory");
+
+		const bool is_userspace = (vaddr < KERNEL_OFFSET);
+		if (is_userspace && this != &PageTable::current())
+			;
+		else if (pages <= 32 || !s_is_post_heap_done)
+		{
+			for (size_t i = 0; i < pages; i++, vaddr += PAGE_SIZE)
+				asm volatile("invlpg (%0)" :: "r"(vaddr));
+		}
+		else if (is_userspace || !s_has_pge)
+		{
+			asm volatile("movq %0, %%cr3" :: "r"(m_highest_paging_struct));
+		}
+		else
+		{
+			asm volatile(
+				"movq %%cr4, %%rax;"
+
+				"andq $~0x80, %%rax;"
+				"movq %%rax, %%cr4;"
+
+				"movq %0, %%cr3;"
+
+				"orq $0x80, %%rax;"
+				"movq %%rax, %%cr4;"
+				:
+				: "r"(m_highest_paging_struct)
+				: "rax"
+			);
+		}
 
 		if (send_smp_message)
 		{
@@ -623,13 +657,14 @@ namespace Kernel
 				.type = Processor::SMPMessage::Type::FlushTLB,
 				.flush_tlb = {
 					.vaddr      = vaddr,
-					.page_count = 1
+					.page_count = pages,
+					.page_table = vaddr < KERNEL_OFFSET ? this : nullptr,
 				}
 			});
 		}
 	}
 
-	void PageTable::unmap_page(vaddr_t vaddr, bool send_smp_message)
+	void PageTable::unmap_page(vaddr_t vaddr, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -658,30 +693,27 @@ namespace Kernel
 		uint64_t* pd   = P2V(pdpt[pdpte] & s_page_addr_mask);
 		uint64_t* pt   = P2V(pd[pde]     & s_page_addr_mask);
 
+		const paddr_t old_paddr = pt[pte] & PAGE_ADDR_MASK;
+
 		pt[pte] = 0;
-		invalidate(vaddr, send_smp_message);
+
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::unmap_range(vaddr_t vaddr, size_t size)
 	{
 		ASSERT(vaddr % PAGE_SIZE == 0);
 
-		size_t page_count = range_page_count(vaddr, size);
+		const size_t page_count = range_page_count(vaddr, size);
 
 		SpinLockGuard _(m_lock);
 		for (vaddr_t page = 0; page < page_count; page++)
 			unmap_page(vaddr + page * PAGE_SIZE, false);
-
+		invalidate_range(vaddr, page_count, true);
-		Processor::broadcast_smp_message({
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = page_count
-			}
-		});
 	}
 
-	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool send_smp_message)
+	void PageTable::map_page_at(paddr_t paddr, vaddr_t vaddr, flags_t flags, MemoryType memory_type, bool invalidate)
 	{
 		ASSERT(vaddr);
 		ASSERT(vaddr != fast_page());
@@ -742,9 +774,12 @@ namespace Kernel
 		if (!(flags & Flags::Present))
 			uwr_flags &= ~Flags::Present;
 
+		const paddr_t old_paddr = pt[pte] & PAGE_ADDR_MASK;
+
 		pt[pte] = paddr | uwr_flags | extra_flags;
 
-		invalidate(vaddr, send_smp_message);
+		if (invalidate && old_paddr != 0)
+			invalidate_page(vaddr, true);
 	}
 
 	void PageTable::map_range_at(paddr_t paddr, vaddr_t vaddr, size_t size, flags_t flags, MemoryType memory_type)
@@ -760,14 +795,66 @@ namespace Kernel
 		SpinLockGuard _(m_lock);
 		for (size_t page = 0; page < page_count; page++)
 			map_page_at(paddr + page * PAGE_SIZE, vaddr + page * PAGE_SIZE, flags, memory_type, false);
+		invalidate_range(vaddr, page_count, true);
+	}
 
-		Processor::broadcast_smp_message({
+	void PageTable::remove_writable_from_range(vaddr_t vaddr, size_t size)
-			.type = Processor::SMPMessage::Type::FlushTLB,
+	{
-			.flush_tlb = {
+		ASSERT(vaddr);
-				.vaddr      = vaddr,
+		ASSERT(vaddr % PAGE_SIZE == 0);
-				.page_count = page_count
+
+		ASSERT(is_canonical(vaddr));
+		ASSERT(is_canonical(vaddr + size - 1));
+
+		const vaddr_t uc_vaddr_start = uncanonicalize(vaddr);
+		const vaddr_t uc_vaddr_end   = uncanonicalize(vaddr + size - 1);
+
+		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
+		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
+		uint16_t pde   = (uc_vaddr_start >> 21) & 0x1FF;
+		uint16_t pte   = (uc_vaddr_start >> 12) & 0x1FF;
+
+		const uint16_t e_pml4e = (uc_vaddr_end >> 39) & 0x1FF;
+		const uint16_t e_pdpte = (uc_vaddr_end >> 30) & 0x1FF;
+		const uint16_t e_pde   = (uc_vaddr_end >> 21) & 0x1FF;
+		const uint16_t e_pte   = (uc_vaddr_end >> 12) & 0x1FF;
+
+		SpinLockGuard _(m_lock);
+
+		const uint64_t* pml4 = P2V(m_highest_paging_struct);
+		for (; pml4e <= e_pml4e; pml4e++)
+		{
+			if (!(pml4[pml4e] & Flags::ReadWrite))
+				continue;
+			const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
+			for (; pdpte < 512; pdpte++)
+			{
+				if (pml4e == e_pml4e && pdpte > e_pdpte)
+					break;
+				if (!(pdpt[pdpte] & Flags::ReadWrite))
+					continue;
+				const uint64_t* pd = P2V(pdpt[pdpte] & s_page_addr_mask);
+				for (; pde < 512; pde++)
+				{
+					if (pml4e == e_pml4e && pdpte == e_pdpte && pde > e_pde)
+						break;
+					if (!(pd[pde] & Flags::ReadWrite))
+						continue;
+					uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
+					for (; pte < 512; pte++)
+					{
+						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
+							break;
+						pt[pte] &= ~static_cast<uint64_t>(Flags::ReadWrite);
+					}
+					pte = 0;
+				}
+				pde = 0;
 			}
-		});
+			pdpte = 0;
+		}
+
+		invalidate_range(vaddr, size / PAGE_SIZE, true);
 	}
 
 	uint64_t PageTable::get_page_data(vaddr_t vaddr) const
@@ -814,13 +901,13 @@ namespace Kernel
 		return page_data & s_page_addr_mask;
 	}
 
-	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool send_smp_message)
+	bool PageTable::reserve_page(vaddr_t vaddr, bool only_free, bool invalidate)
 	{
 		SpinLockGuard _(m_lock);
 		ASSERT(vaddr % PAGE_SIZE == 0);
 		if (only_free && !is_page_free(vaddr))
 			return false;
-		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, send_smp_message);
+		map_page_at(0, vaddr, Flags::Reserved, MemoryType::Normal, invalidate);
 		return true;
 	}
 
@@ -835,13 +922,7 @@ namespace Kernel
 			return false;
 		for (size_t offset = 0; offset < bytes; offset += PAGE_SIZE)
 			reserve_page(vaddr + offset, true, false);
-		Processor::broadcast_smp_message({
+		invalidate_range(vaddr, bytes / PAGE_SIZE, true);
-			.type = Processor::SMPMessage::Type::FlushTLB,
-			.flush_tlb = {
-				.vaddr      = vaddr,
-				.page_count = bytes / PAGE_SIZE,
-			}
-		});
 		return true;
 	}
 
@@ -855,9 +936,9 @@ namespace Kernel
 			last_address -= rem;
 
 		ASSERT(is_canonical(first_address));
-		ASSERT(is_canonical(last_address));
+		ASSERT(is_canonical(last_address - 1));
 		const vaddr_t uc_vaddr_start = uncanonicalize(first_address);
-		const vaddr_t uc_vaddr_end   = uncanonicalize(last_address);
+		const vaddr_t uc_vaddr_end   = uncanonicalize(last_address - 1);
 
 		uint16_t pml4e = (uc_vaddr_start >> 39) & 0x1FF;
 		uint16_t pdpte = (uc_vaddr_start >> 30) & 0x1FF;
@@ -874,10 +955,8 @@ namespace Kernel
 		// Try to find free page that can be mapped without
 		// allocations (page table with unused entries)
 		const uint64_t* pml4 = P2V(m_highest_paging_struct);
-		for (; pml4e < 512; pml4e++)
+		for (; pml4e <= e_pml4e; pml4e++)
 		{
-			if (pml4e > e_pml4e)
-				break;
 			if (!(pml4[pml4e] & Flags::Present))
 				continue;
 			const uint64_t* pdpt = P2V(pml4[pml4e] & s_page_addr_mask);
@@ -897,22 +976,24 @@ namespace Kernel
 					const uint64_t* pt = P2V(pd[pde] & s_page_addr_mask);
 					for (; pte < 512; pte++)
 					{
-						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte >= e_pte)
+						if (pml4e == e_pml4e && pdpte == e_pdpte && pde == e_pde && pte > e_pte)
 							break;
-						if (!(pt[pte] & Flags::Used))
+						if (pt[pte] & Flags::Used)
-						{
+							continue;
-							vaddr_t vaddr = 0;
+						vaddr_t vaddr = 0;
-							vaddr |= static_cast<uint64_t>(pml4e) << 39;
+						vaddr |= static_cast<uint64_t>(pml4e) << 39;
-							vaddr |= static_cast<uint64_t>(pdpte) << 30;
+						vaddr |= static_cast<uint64_t>(pdpte) << 30;
-							vaddr |= static_cast<uint64_t>(pde)   << 21;
+						vaddr |= static_cast<uint64_t>(pde)   << 21;
-							vaddr |= static_cast<uint64_t>(pte)   << 12;
+						vaddr |= static_cast<uint64_t>(pte)   << 12;
-							vaddr = canonicalize(vaddr);
+						vaddr = canonicalize(vaddr);
-							ASSERT(reserve_page(vaddr));
+						ASSERT(reserve_page(vaddr));
-							return vaddr;
+						return vaddr;
-						}
 					}
+					pte = 0;
 				}
+				pde = 0;
 			}
+			pdpte = 0;
 		}
 
 		for (vaddr_t uc_vaddr = uc_vaddr_start; uc_vaddr < uc_vaddr_end; uc_vaddr += PAGE_SIZE)
@@ -924,7 +1005,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	vaddr_t PageTable::reserve_free_contiguous_pages(size_t page_count, vaddr_t first_address, vaddr_t last_address)
@@ -937,7 +1018,7 @@ namespace Kernel
 			last_address -= rem;
 
 		ASSERT(is_canonical(first_address));
-		ASSERT(is_canonical(last_address));
+		ASSERT(is_canonical(last_address - 1));
 
 		SpinLockGuard _(m_lock);
 
@@ -966,7 +1047,7 @@ namespace Kernel
 			}
 		}
 
-		ASSERT_NOT_REACHED();
+		return 0;
 	}
 
 	bool PageTable::is_page_free(vaddr_t page) const

kernel/arch/x86_64/Signal.S

@@ -1,12 +1,13 @@
 .section .userspace, "ax"
 
 // stack contains
-//    return address
+//    (8 bytes)  return address (on return stack)
-//    return stack
+//    (8 bytes)  return stack
-//    return rflags
+//    (8 bytes)  return rflags
-//    siginfo_t
+//    (8 bytes)  restore sigmask
-//    signal number
+//    (56 bytes) siginfo_t
-//    signal handler
+//    (8 bytes)  signal number
+//    (8 bytes)  signal handler
 
 .global signal_trampoline
 signal_trampoline:
@@ -26,11 +27,9 @@ signal_trampoline:
 	pushq %rax
 	pushq %rbp
 
-	movq 200(%rsp), %rax  // return sp
+	movq 208(%rsp), %rax
-	addq $(128 + 8), %rax //   red-zone and return address
+	pushq %rax; addq $(128 + 8), (%rsp)
-	movq 208(%rsp), %rbx  // return ip
+	pushq (%rax)
-	pushq %rax;
-	pushq %rbx
 
 	// FIXME: populate these
 	xorq %rax, %rax
@@ -59,9 +58,17 @@ signal_trampoline:
 
 	// restore stack
 	movq %rbp, %rsp
-	addq $56, %rsp
+	addq $40, %rsp
+
+	// restore sigmask
+	movq $83, %rdi       // SYS_SIGPROCMASK
+	movq $3,  %rsi       // SIG_SETMASK
+	leaq 192(%rsp), %rdx // set
+	xorq %r10, %r10      // oset
+	syscall
 
 	// restore registers
+	addq $16, %rsp
 	popq %rbp
 	popq %rax
 	popq %rbx
@@ -78,13 +85,13 @@ signal_trampoline:
 	popq %r14
 	popq %r15
 
-	// skip handler, number, siginfo_t
+	// skip handler, number, siginfo_t, sigmask
-	addq $72, %rsp
+	addq $80, %rsp
 
 	// restore flags
 	popfq
 
 	movq (%rsp), %rsp
 
-	// return over red-zone and siginfo_t
+	// return over red-zone
 	ret $128

kernel/arch/x86_64/Syscall.S

@@ -1,50 +1,26 @@
-// arguments in RAX, RBX, RCX, RDX, RSI, RDI
-// System V ABI: RDI, RSI, RDX, RCX, R8, R9
 .global asm_syscall_handler
 asm_syscall_handler:
 	swapgs
-	pushq %rbx
+
-	pushq %rcx
+	movq %rsp, %rax
-	pushq %rdx
+	movq %gs:8, %rsp
-	pushq %rdi
+
-	pushq %rsi
+	pushq $(0x20 | 3)
-	pushq %rbp
+	pushq %rax
-	pushq %r8
-	pushq %r9
-	pushq %r10
 	pushq %r11
-	pushq %r12
+	pushq $(0x28 | 3)
-	pushq %r13
+	pushq %rcx
-	pushq %r14
+	subq $8, %rsp
-	pushq %r15
-	cld
 
-	movq %rsi, %r8
+	movq %r10, %rcx
-	movq %rdi, %r9
-	movq %rax, %rdi
-	movq %rbx, %rsi
-	xchgq %rcx, %rdx
-	leaq 112(%rsp), %rbx
-	pushq %rbx
 	call cpp_syscall_handler
-	addq $8, %rsp
 
-	popq %r15
+	movq  8(%rsp), %rcx
-	popq %r14
+	movq 24(%rsp), %r11
-	popq %r13
+	movq 32(%rsp), %rsp
-	popq %r12
+
-	popq %r11
-	popq %r10
-	popq %r9
-	popq %r8
-	popq %rbp
-	popq %rsi
-	popq %rdi
-	popq %rdx
-	popq %rcx
-	popq %rbx
 	swapgs
-	iretq
+	sysretq
 
 .global sys_fork_trampoline
 sys_fork_trampoline:
@@ -57,7 +33,7 @@ sys_fork_trampoline:
 
 	call read_ip
 	testq %rax, %rax
-	je .reload_stack
+	jz .done
 
 	movq %rax, %rsi
 	movq %rsp, %rdi
@@ -71,9 +47,3 @@ sys_fork_trampoline:
 	popq %rbp
 	popq %rbx
 	ret
-
- .reload_stack:
-	call get_thread_start_sp
-	movq %rax, %rsp
-	xorq %rax, %rax
-	jmp .done

kernel/arch/x86_64/Thread.S

@@ -7,9 +7,6 @@ read_ip:
 # void start_kernel_thread()
 .global start_kernel_thread
 start_kernel_thread:
-	call get_thread_start_sp
-	movq %rax, %rsp
-
 	# STACK LAYOUT
 	#   on_exit arg
 	#   on_exit func
@@ -27,11 +24,5 @@ start_kernel_thread:
 
 .global start_userspace_thread
 start_userspace_thread:
-	call load_thread_sse
-
-	call get_thread_start_sp
-	movq %rax, %rsp
-
 	swapgs
-
 	iretq

kernel/arch/x86_64/User.S

@@ -0,0 +1,87 @@
+# bool safe_user_memcpy(void*, const void*, size_t)
+.global safe_user_memcpy
+.global safe_user_memcpy_end
+.global safe_user_memcpy_fault
+safe_user_memcpy:
+	xorq %rax, %rax
+	movq %rdx, %rcx
+	rep movsb
+	incq %rax
+safe_user_memcpy_fault:
+	ret
+safe_user_memcpy_end:
+
+# bool safe_user_strncpy(void*, const void*, size_t)
+.global safe_user_strncpy
+.global safe_user_strncpy_end
+.global safe_user_strncpy_fault
+safe_user_strncpy:
+	movq %rdx, %rcx
+	testq %rcx, %rcx
+	jz safe_user_strncpy_fault
+
+ .safe_user_strncpy_align_loop:
+	testb $0x7, %sil
+	jz .safe_user_strncpy_align_done
+
+	movb (%rsi), %al
+	movb %al, (%rdi)
+	testb %al, %al
+	jz .safe_user_strncpy_done
+
+	incq %rdi
+	incq %rsi
+	decq %rcx
+	jnz .safe_user_strncpy_align_loop
+	jmp safe_user_strncpy_fault
+
+ .safe_user_strncpy_align_done:
+	movq $0x0101010101010101, %r8
+	movq $0x8080808080808080, %r9
+
+ .safe_user_strncpy_qword_loop:
+	cmpq $8, %rcx
+	jb .safe_user_strncpy_qword_done
+
+	movq (%rsi), %rax
+	movq %rax, %r10
+	movq %rax, %r11
+
+	# https://graphics.stanford.edu/~seander/bithacks.html#ZeroInWord
+	subq %r8, %r10
+	notq %r11
+	andq %r11, %r10
+	andq %r9, %r10
+	jnz .safe_user_strncpy_byte_loop
+
+	movq %rax, (%rdi)
+
+	addq $8, %rdi
+	addq $8, %rsi
+	subq $8, %rcx
+	jnz .safe_user_strncpy_qword_loop
+	jmp safe_user_strncpy_fault
+
+ .safe_user_strncpy_qword_done:
+	testq %rcx, %rcx
+	jz safe_user_strncpy_fault
+
+ .safe_user_strncpy_byte_loop:
+	movb (%rsi), %al
+	movb %al, (%rdi)
+	testb %al, %al
+	jz .safe_user_strncpy_done
+
+	incq %rdi
+	incq %rsi
+	decq %rcx
+	jnz .safe_user_strncpy_byte_loop
+
+ safe_user_strncpy_fault:
+	xorq %rax, %rax
+	ret
+
+ .safe_user_strncpy_done:
+	movb $1, %al
+	ret
+safe_user_strncpy_end:

kernel/arch/x86_64/Yield.S

@@ -0,0 +1,29 @@
+.global asm_yield_trampoline
+asm_yield_trampoline:
+	leaq 8(%rsp), %rcx
+	movq %rdi, %rsp
+
+	subq $8, %rsp
+	pushq -8(%rcx)
+	pushq %rcx
+	pushq %rax
+	pushq %rbx
+	pushq %rbp
+	pushq %r12
+	pushq %r13
+	pushq %r14
+	pushq %r15
+
+	movq %rsp, %rdi
+	call scheduler_on_yield
+
+	popq %r15
+	popq %r14
+	popq %r13
+	popq %r12
+	popq %rbp
+	popq %rbx
+	popq %rax
+	movq 8(%rsp), %rcx
+	movq 0(%rsp), %rsp
+	jmp *%rcx

kernel/arch/x86_64/boot.S

@@ -11,26 +11,28 @@
 
 .code32
 
-# multiboot2 header
+// custom addresses, video mode info, page align modules
+.set multiboot_flags, (1 << 16) | (1 << 2) | (1 << 0)
+
 .section .multiboot, "aw"
-	.align 8
 multiboot_start:
 	.long 0x1BADB002
-	.long (1 << 2) # page align modules
+	.long multiboot_flags
-	.long -(0x1BADB002 + (1 << 2))
+	.long -(0x1BADB002 + multiboot_flags)
 
-	.long 0
+	.long V2P(multiboot_start)
-	.long 0
+	.long V2P(g_kernel_start)
-	.long 0
+	.long V2P(g_kernel_bss_start)
-	.long 0
+	.long V2P(g_kernel_end)
-	.long 0
+	.long V2P(_start)
 
 	.long 0
 	.long FB_WIDTH
 	.long FB_HEIGHT
 	.long FB_BPP
 multiboot_end:
-	.align 8
+
+.section .multiboot2, "aw"
 multiboot2_start:
 	.long 0xE85250D6
 	.long 0
@@ -66,7 +68,6 @@ multiboot2_start:
 multiboot2_end:
 
 .section .bananboot, "aw"
-	.align 8
 bananboot_start:
 	.long 0xBABAB007
 	.long -(0xBABAB007 + FB_WIDTH + FB_HEIGHT + FB_BPP)
@@ -179,6 +180,13 @@ enable_sse:
 	movl %eax, %cr4
 	ret
 
+enable_tsc:
+	# allow userspace to use RDTSC
+	movl %cr4, %ecx
+	andl $0xFFFFFFFB, %ecx
+	movl %ecx, %cr4
+	ret
+
 initialize_paging:
 	# enable PAE
 	movl %cr4, %ecx
@@ -215,6 +223,7 @@ _start:
 
 	call check_requirements
 	call enable_sse
+	call enable_tsc
 	call initialize_paging
 
 	# flush gdt and jump to 64 bit
@@ -301,6 +310,7 @@ ap_protected_mode:
 	movb $1, AP_V2P(ap_stack_loaded)
 
 	leal V2P(enable_sse),        %ecx; call *%ecx
+	leal V2P(enable_tsc),        %ecx; call *%ecx
 	leal V2P(initialize_paging), %ecx; call *%ecx
 
 	# load boot gdt and enter long mode

kernel/arch/x86_64/interrupts.S

@@ -1,12 +1,4 @@
-.macro swapgs_if_necessary, n
+.macro intr_header, n
-	cmpb $0x08, \n(%rsp)
-	je 1f
-	swapgs
-1:
-.endm
-
-.macro pushaq, n
-	swapgs_if_necessary \n
 	pushq %rax
 	pushq %rcx
 	pushq %rdx
@@ -22,10 +14,18 @@
 	pushq %r13
 	pushq %r14
 	pushq %r15
+	testb $3, \n+15*8(%rsp)
+	jz 1f
+	swapgs
+1:	cld
 .endm
 
-.macro popaq, n
+.macro intr_footer, n
-	popq %r15
+	testb $3, \n+15*8(%rsp)
+	jz 1f
+	call cpp_check_signal
+	swapgs
+1:	popq %r15
 	popq %r14
 	popq %r13
 	popq %r12
@@ -40,12 +40,10 @@
 	popq %rdx
 	popq %rcx
 	popq %rax
-	swapgs_if_necessary \n
 .endm
 
 isr_stub:
-	pushaq 24
+	intr_header 24
-	cld
 	movq %cr0, %rax; pushq %rax
 	movq %cr2, %rax; pushq %rax
 	movq %cr3, %rax; pushq %rax
@@ -58,43 +56,33 @@ isr_stub:
 	call cpp_isr_handler
 	addq $32, %rsp
 
-	popaq 24
+	intr_footer 24
 	addq $16, %rsp
 	iretq
 
 irq_stub:
-	pushaq 24
+	intr_header 24
-	cld
+	xorq %rbp, %rbp
 	movq 120(%rsp), %rdi	# irq number
 	call cpp_irq_handler
-	popaq 24
+	intr_footer 24
 	addq $16, %rsp
 	iretq
 
-.global asm_yield_handler
-asm_yield_handler:
-	pushaq 8
-	cld
-	leaq 120(%rsp), %rdi # interrupt stack ptr
-	movq     %rsp,  %rsi # interrupt register ptr
-	call cpp_yield_handler
-	popaq 8
-	iretq
-
 .global asm_ipi_handler
 asm_ipi_handler:
-	pushaq 8
+	intr_header 8
-	cld
+	xorq %rbp, %rbp
 	call cpp_ipi_handler
-	popaq 8
+	intr_footer 8
 	iretq
 
 .global asm_timer_handler
 asm_timer_handler:
-	pushaq 8
+	intr_header 8
-	cld
+	xorq %rbp, %rbp
 	call cpp_timer_handler
-	popaq 8
+	intr_footer 8
 	iretq
 
 .macro isr n

kernel/arch/x86_64/linker.ld

@@ -11,6 +11,7 @@ SECTIONS
 	{
 		g_kernel_execute_start = .;
 		*(.multiboot)
+		*(.multiboot2)
 		*(.bananboot)
 		*(.text.*)
 	}
@@ -43,6 +44,7 @@ SECTIONS
 	}
 	.bss ALIGN(4K) : AT(ADDR(.bss) - KERNEL_OFFSET)
 	{
+		g_kernel_bss_start = .;
 		*(COMMON)
 		*(.bss)
 		g_kernel_writable_end = .;

kernel/include/kernel/API/SharedPage.h

@@ -0,0 +1,37 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace Kernel::API
+{
+
+	enum SharedPageFeature : uint32_t
+	{
+		SPF_GETTIME = 1 << 0,
+	};
+
+	struct SharedPage
+	{
+		uint8_t __sequence[0x100];
+
+		uint32_t features;
+
+		struct
+		{
+			uint8_t shift;
+			uint64_t mult;
+			uint64_t realtime_seconds;
+		} gettime_shared;
+
+		struct
+		{
+			struct
+			{
+				uint32_t seq;
+				uint64_t last_ns;
+				uint64_t last_tsc;
+			} gettime_local;
+		} cpus[];
+	};
+
+}

kernel/include/kernel/API/Syscall.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <BAN/MacroUtils.h>
+
+#if defined(__x86_64__)
+	#define _kas_instruction "syscall"
+	#define _kas_result rax
+	#define _kas_arguments rdi, rsi, rdx, r10, r8, r9
+	#define _kas_globbers rcx, rdx, rdi, rsi, r8, r9, r10, r11
+#elif defined(__i686__)
+	#define _kas_instruction "int $0xF0"
+	#define _kas_result eax
+	#define _kas_arguments eax, ebx, ecx, edx, esi, edi
+	#define _kas_globbers
+#endif
+
+#define _kas_argument_var(index, value) register long _kas_a##index asm(_ban_stringify(_ban_get(index, _kas_arguments))) = (long)value;
+#define _kas_dummy_var(index, value) register long _kas_d##index asm(#value);
+#define _kas_input(index, _) "r"(_kas_a##index)
+#define _kas_output(index, _) , "=r"(_kas_d##index)
+#define _kas_globber(_, value) #value
+
+#define _kas_syscall(...) ({ \
+		register long _kas_ret asm(_ban_stringify(_kas_result)); \
+		_ban_for_each(_kas_argument_var, __VA_ARGS__) \
+		_ban_for_each(_kas_dummy_var, _kas_globbers) \
+		asm volatile( \
+			_kas_instruction \
+			: "=r"(_kas_ret) _ban_for_each(_kas_output, _kas_globbers) \
+			: _ban_for_each_comma(_kas_input, __VA_ARGS__) \
+			: "cc", "memory"); \
+		(void)_kas_a0; /* require 1 argument */ \
+		_kas_ret; \
+	})

kernel/include/kernel/Audio/AC97/Controller.h

@@ -6,10 +6,10 @@
 namespace Kernel
 {
 
-	class AC97AudioController : public AudioController, public Interruptable
+	class AC97AudioController final : public AudioController, public Interruptable
 	{
 	public:
-		static BAN::ErrorOr<BAN::RefPtr<AC97AudioController>> create(PCI::Device& pci_device);
+		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
 
 		void handle_irq() override;
 
@@ -19,11 +19,19 @@ namespace Kernel
 		uint32_t get_channels() const override { return 2; }
 		uint32_t get_sample_rate() const override { return 48000; }
 
+		uint32_t get_total_pins() const override { return 1; }
+		uint32_t get_current_pin() const override { return 0; }
+		BAN::ErrorOr<void> set_current_pin(uint32_t pin) override { if (pin != 0) return BAN::Error::from_errno(EINVAL); return {}; }
+
+		BAN::ErrorOr<void> set_volume_mdB(int32_t) override;
+
 	private:
 		AC97AudioController(PCI::Device& pci_device)
 			: m_pci_device(pci_device)
 		{ }
 
+		uint32_t get_volume_data() const;
+
 		BAN::ErrorOr<void> initialize();
 		BAN::ErrorOr<void> initialize_bld();
 		BAN::ErrorOr<void> initialize_interrupts();

kernel/include/kernel/Audio/Controller.h

@@ -1,29 +1,39 @@
 #pragma once
 
 #include <kernel/Device/Device.h>
+#include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/PCI.h>
 
+#include <sys/ioctl.h>
+
 namespace Kernel
 {
 
 	class AudioController : public CharacterDevice
 	{
 	public:
-		static BAN::ErrorOr<BAN::RefPtr<AudioController>> create(PCI::Device& pci_device);
+		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
 
 		dev_t rdev() const override { return m_rdev; }
 		BAN::StringView name() const override { return m_name; }
 
 	protected:
 		AudioController();
+		BAN::ErrorOr<void> initialize();
 
 		virtual void handle_new_data() = 0;
 
 		virtual uint32_t get_channels() const = 0;
 		virtual uint32_t get_sample_rate() const = 0;
 
+		virtual uint32_t get_total_pins() const = 0;
+		virtual uint32_t get_current_pin() const = 0;
+		virtual BAN::ErrorOr<void> set_current_pin(uint32_t) = 0;
+
+		virtual BAN::ErrorOr<void> set_volume_mdB(int32_t) = 0;
+
 		bool can_read_impl() const override { return false; }
-		bool can_write_impl() const override { SpinLockGuard _(m_spinlock); return m_sample_data_size < m_sample_data_capacity; }
+		bool can_write_impl() const override { SpinLockGuard _(m_spinlock); return !m_sample_data->full(); }
 		bool has_error_impl() const override { return false; }
 		bool has_hungup_impl() const override { return false; }
 
@@ -36,9 +46,9 @@ namespace Kernel
 		mutable SpinLock m_spinlock;
 
 		static constexpr size_t m_sample_data_capacity = 1 << 20;
-		uint8_t m_sample_data[m_sample_data_capacity];
+		BAN::UniqPtr<ByteRingBuffer> m_sample_data;
-		size_t m_sample_data_head { 0 };
+
-		size_t m_sample_data_size { 0 };
+		snd_volume_info m_volume_info {};
 
 	private:
 		const dev_t m_rdev;

kernel/include/kernel/Audio/HDAudio/AudioFunctionGroup.h

@@ -0,0 +1,80 @@
+#pragma once
+
+#include <kernel/Audio/Controller.h>
+#include <kernel/Audio/HDAudio/Controller.h>
+
+namespace Kernel
+{
+
+	class HDAudioController;
+
+	class HDAudioFunctionGroup final : public AudioController
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<HDAudioFunctionGroup>> create(BAN::RefPtr<HDAudioController>, uint8_t cid, HDAudio::AFGNode&&);
+
+		void on_stream_interrupt(uint8_t stream_index);
+
+	protected:
+		// FIXME: allow setting these :D
+		uint32_t get_channels() const override { return 2; }
+		uint32_t get_sample_rate() const override { return 48000; }
+
+		uint32_t get_total_pins() const override;
+		uint32_t get_current_pin() const override;
+		BAN::ErrorOr<void> set_current_pin(uint32_t) override;
+
+		BAN::ErrorOr<void> set_volume_mdB(int32_t) override;
+
+		void handle_new_data() override;
+
+	private:
+		HDAudioFunctionGroup(BAN::RefPtr<HDAudioController> controller, uint8_t cid, HDAudio::AFGNode&& afg_node)
+			: m_controller(controller)
+			, m_afg_node(BAN::move(afg_node))
+			, m_cid(cid)
+		{ }
+		~HDAudioFunctionGroup();
+
+		BAN::ErrorOr<void> initialize();
+		BAN::ErrorOr<void> initialize_stream();
+		BAN::ErrorOr<void> initialize_output();
+
+		BAN::ErrorOr<void> enable_output_path(uint8_t index);
+		BAN::ErrorOr<void> disable_output_path(uint8_t index);
+
+		void reset_stream();
+
+		BAN::ErrorOr<void> recurse_output_paths(const HDAudio::AFGWidget& widget, BAN::Vector<const HDAudio::AFGWidget*>& path);
+
+		uint16_t get_format_data() const;
+
+		size_t bdl_offset() const;
+
+		void queue_bdl_data();
+
+	private:
+		// use 6 512 sample BDL entries
+		// each entry is ~10.7 ms at 48 kHz
+		// -> total buffered audio is 64 ms
+		static constexpr size_t m_bdl_entry_sample_frames = 512;
+		static constexpr size_t m_bdl_entry_count         = 6;
+
+		BAN::RefPtr<HDAudioController> m_controller;
+		const HDAudio::AFGNode m_afg_node;
+		const uint8_t m_cid;
+
+		BAN::Vector<BAN::Vector<const HDAudio::AFGWidget*>> m_output_paths;
+		BAN::Vector<const HDAudio::AFGWidget*> m_output_pins;
+		size_t m_output_path_index { SIZE_MAX };
+
+		uint8_t m_stream_id    { 0xFF };
+		uint8_t m_stream_index { 0xFF };
+		BAN::UniqPtr<DMARegion> m_bdl_region;
+
+		size_t m_bdl_head { 0 };
+		size_t m_bdl_tail { 0 };
+		bool m_stream_running { false };
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/Controller.h

@@ -0,0 +1,77 @@
+#pragma once
+
+#include <kernel/Audio/Controller.h>
+#include <kernel/Audio/HDAudio/Definitions.h>
+#include <kernel/Memory/DMARegion.h>
+
+namespace Kernel
+{
+
+	class HDAudioController : public Interruptable, public BAN::RefCounted<HDAudioController>
+	{
+	public:
+		static BAN::ErrorOr<void> create(PCI::Device& pci_device);
+
+		BAN::ErrorOr<uint32_t> send_command(HDAudio::CORBEntry);
+
+		uint8_t get_stream_index(HDAudio::StreamType type, uint8_t index) const;
+
+		BAN::ErrorOr<uint8_t> allocate_stream_id();
+		void deallocate_stream_id(uint8_t id);
+
+		BAN::ErrorOr<uint8_t> allocate_stream(HDAudio::StreamType type, void* afg);
+		void deallocate_stream(uint8_t index);
+
+		PCI::BarRegion& bar0() { return *m_bar0; }
+
+		bool is_64bit() const { return m_is64bit; }
+
+		void handle_irq() override;
+
+	private:
+		HDAudioController(PCI::Device& pci_device)
+			: m_pci_device(pci_device)
+		{ }
+
+		BAN::ErrorOr<void> initialize();
+		BAN::ErrorOr<void> initialize_ring_buffers();
+
+		BAN::ErrorOr<void> reset_controller();
+
+		BAN::ErrorOr<HDAudio::Codec> initialize_codec(uint8_t codec);
+		BAN::ErrorOr<HDAudio::AFGNode> initialize_node(uint8_t codec, uint8_t node);
+		BAN::ErrorOr<HDAudio::AFGWidget> initialize_widget(uint8_t codec, uint8_t node);
+
+	private:
+		struct RingBuffer
+		{
+			vaddr_t vaddr;
+			uint32_t index;
+			uint32_t size;
+		};
+
+	private:
+		PCI::Device& m_pci_device;
+		BAN::UniqPtr<PCI::BarRegion> m_bar0;
+		bool m_is64bit { false };
+
+		bool m_use_immediate_command { false };
+
+		uint8_t m_output_streams { 0 };
+		uint8_t m_input_streams { 0 };
+		uint8_t m_bidir_streams { 0 };
+		void* m_allocated_streams[30] {};
+
+		// NOTE: stream ids are from 1 to 15
+		uint16_t m_allocated_stream_ids { 0 };
+
+		Mutex m_command_mutex;
+		SpinLock m_rb_lock;
+		ThreadBlocker m_rb_blocker;
+
+		RingBuffer m_corb;
+		RingBuffer m_rirb;
+		BAN::UniqPtr<DMARegion> m_ring_buffer_region;
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/Definitions.h

@@ -0,0 +1,90 @@
+#pragma once
+
+#include <BAN/Vector.h>
+
+namespace Kernel::HDAudio
+{
+
+	struct CORBEntry
+	{
+		union {
+			struct {
+				uint32_t data          : 8;
+				uint32_t command       : 12;
+				uint32_t node_index    : 8;
+				uint32_t codec_address : 4;
+			};
+			uint32_t raw;
+		};
+	};
+	static_assert(sizeof(CORBEntry) == sizeof(uint32_t));
+
+	struct BDLEntry
+	{
+		paddr_t address;
+		uint32_t length;
+		uint32_t ioc;
+	};
+	static_assert(sizeof(BDLEntry) == 16);
+
+	struct AFGWidget
+	{
+		enum class Type
+		{
+			OutputConverter,
+			InputConverter,
+			Mixer,
+			Selector,
+			PinComplex,
+			Power,
+			VolumeKnob,
+			BeepGenerator,
+		};
+
+		Type type;
+		uint8_t id;
+
+		union
+		{
+			struct
+			{
+				bool input;
+				bool output;
+				bool display; // HDMI or DP
+				uint32_t config;
+			} pin_complex;
+		};
+
+		struct Amplifier
+		{
+			uint8_t offset;
+			uint8_t num_steps;
+			uint8_t step_size;
+			bool mute;
+		};
+
+		BAN::Optional<Amplifier> output_amplifier;
+
+		BAN::Vector<uint16_t> connections;
+	};
+
+	struct AFGNode
+	{
+		uint8_t id;
+		BAN::Vector<AFGWidget> widgets;
+	};
+
+	struct Codec
+	{
+		uint8_t id;
+		BAN::Vector<AFGNode> nodes;
+	};
+
+	enum class StreamType
+	{
+		Input,
+		Output,
+		Bidirectional,
+	};
+
+}

kernel/include/kernel/Audio/HDAudio/Registers.h

@@ -0,0 +1,50 @@
+#pragma once
+
+#include <stdint.h>
+
+namespace Kernel::HDAudio
+{
+
+	enum Regs : uint8_t
+	{
+		GCAP = 0x00,
+		VMIN = 0x02,
+		VMAJ = 0x03,
+		GCTL = 0x08,
+		STATESTS = 0x0E,
+
+		INTCTL = 0x20,
+		INTSTS = 0x24,
+
+		CORBLBASE = 0x40,
+		CORBUBASE = 0x44,
+		CORBWP    = 0x48,
+		CORBRP    = 0x4A,
+		CORBCTL   = 0x4C,
+		CORBSTS   = 0x4D,
+		CORBSIZE  = 0x4E,
+
+		RIRBLBASE = 0x50,
+		RIRBUBASE = 0x54,
+		RIRBWP    = 0x58,
+		RINTCNT   = 0x5A,
+		RIRBCTL   = 0x5C,
+		RIRBSTS   = 0x5D,
+		RIRBSIZE  = 0x5E,
+
+		ICOI = 0x60,
+		ICII = 0x64,
+		ICIS = 0x68,
+
+		SDCTL   = 0x00,
+		SDSTS   = 0x03,
+		SDLPIB  = 0x04,
+		SDCBL   = 0x08,
+		SDLVI   = 0x0C,
+		SDFIFOD = 0x10,
+		SDFMT   = 0x12,
+		SDBDPL  = 0x18,
+		SDBDPU  = 0x1C,
+	};
+
+}

kernel/include/kernel/BootInfo.h

@@ -44,7 +44,7 @@ namespace Kernel
 	struct BootModule
 	{
 		paddr_t start;
-		size_t size;
+		uint64_t size;
 	};
 
 	struct BootInfo

kernel/include/kernel/CPUID.h

@@ -81,5 +81,6 @@ namespace CPUID
 	bool has_pge();
 	bool has_pat();
 	bool has_1gib_pages();
+	bool has_invariant_tsc();
 
 }

kernel/include/kernel/Debug.h

@@ -70,10 +70,13 @@
 #define DEBUG_USB_MOUSE 0
 #define DEBUG_USB_MASS_STORAGE 0
 
+#define DEBUG_HDAUDIO 0
+
 
 namespace Debug
 {
 	void dump_stack_trace();
+	void dump_stack_trace(uintptr_t ip, uintptr_t bp);
 	void dump_qr_code();
 
 	void putchar(char);

kernel/include/kernel/Device/DeviceNumbers.h

@@ -16,6 +16,7 @@ namespace Kernel
 		Debug,
 		Keyboard,
 		Mouse,
+		Joystick,
 		SCSI,
 		NVMeController,
 		NVMeNamespace,

kernel/include/kernel/Device/FramebufferDevice.h

@@ -37,6 +37,8 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 
+		BAN::ErrorOr<long> ioctl_impl(int cmd, void* arg) override;
+
 		virtual bool can_read_impl() const override { return true; }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return false; }

kernel/include/kernel/FS/EventFD.h

@@ -0,0 +1,52 @@
+#pragma once
+
+#include <kernel/FS/Inode.h>
+
+namespace Kernel
+{
+
+	class EventFD final : public Inode
+	{
+	public:
+		static BAN::ErrorOr<BAN::RefPtr<Inode>> create(uint64_t initval, bool semaphore);
+
+		ino_t ino() const override { return 0; }
+		Mode mode() const override { return { Mode::IFCHR | Mode::IRUSR | Mode::IWUSR }; }
+		nlink_t nlink() const override { return ref_count(); }
+		uid_t uid() const override { return 0; }
+		gid_t gid() const override { return 0; }
+		off_t size() const override { return 0; }
+		timespec atime() const override { return {}; }
+		timespec mtime() const override { return {}; }
+		timespec ctime() const override { return {}; }
+		blksize_t blksize() const override { return 8; }
+		blkcnt_t blocks() const override { return 0; }
+		dev_t dev() const override { return 0; }
+		dev_t rdev() const override { return 0; }
+
+		const FileSystem* filesystem() const override { return nullptr; }
+
+	protected:
+		BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
+		BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
+		BAN::ErrorOr<void> fsync_impl() final override { return {}; }
+
+		bool can_read_impl() const override { return m_value > 0; }
+		bool can_write_impl() const override { return m_value < UINT64_MAX - 1; }
+		bool has_error_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
+
+	private:
+		EventFD(uint64_t initval, bool is_semaphore)
+			: m_is_semaphore(is_semaphore)
+			, m_value(initval)
+		{ }
+
+	private:
+		const bool m_is_semaphore;
+		uint64_t m_value;
+
+		ThreadBlocker m_thread_blocker;
+	};
+
+}

kernel/include/kernel/FS/Ext2/FileSystem.h

@@ -28,36 +28,28 @@ namespace Kernel
 			BAN_NON_COPYABLE(BlockBufferWrapper);
 
 		public:
-			BlockBufferWrapper(BAN::Span<uint8_t> buffer, bool* used, Mutex* mutex, ThreadBlocker* blocker)
+			BlockBufferWrapper(BAN::Span<uint8_t> buffer, void (*callback)(void*, const uint8_t*), void* argument)
 				: m_buffer(buffer)
-				, m_used(used)
+				, m_callback(callback)
-				, m_mutex(mutex)
+				, m_argument(argument)
-				, m_blocker(blocker)
+			{ }
-			{
-				ASSERT(m_used && *m_used);
-			}
 			BlockBufferWrapper(BlockBufferWrapper&& other) { *this = BAN::move(other); }
 			~BlockBufferWrapper()
 			{
-				if (m_used == nullptr)
+				if (m_callback == nullptr)
 					return;
-				m_mutex->lock();
+				m_callback(m_argument, m_buffer.data());
-				*m_used = false;
-				m_blocker->unblock();
-				m_mutex->unlock();
 			}
 
 			BlockBufferWrapper& operator=(BlockBufferWrapper&& other)
 			{
 				this->m_buffer = other.m_buffer;
-				this->m_used = other.m_used;
+				this->m_callback = other.m_callback;
-				this->m_mutex = other.m_mutex;
+				this->m_argument = other.m_argument;
-				this->m_blocker = other.m_blocker;
 
 				other.m_buffer = {};
-				other.m_used = nullptr;
+				other.m_callback = nullptr;
-				other.m_mutex = nullptr;
+				other.m_argument = nullptr;
-				other.m_blocker = nullptr;
 
 				return *this;
 			}
@@ -75,9 +67,8 @@ namespace Kernel
 
 		private:
 			BAN::Span<uint8_t> m_buffer;
-			bool* m_used;
+			void (*m_callback)(void*, const uint8_t*);
-			Mutex* m_mutex;
+			void* m_argument;
-			ThreadBlocker* m_blocker;
 		};
 
 	public:
@@ -130,6 +121,9 @@ namespace Kernel
 
 			BAN::ErrorOr<void> initialize(size_t block_size);
 
+		private:
+			void destroy_callback(const uint8_t* buffer_ptr);
+
 		private:
 			struct BlockBuffer
 			{
@@ -137,10 +131,20 @@ namespace Kernel
 				bool used { false };
 			};
 
+			struct ThreadInfo
+			{
+				pid_t tid { 0 };
+				size_t buffers { 0 };
+			};
+
 		private:
+			static constexpr size_t max_threads = 8;
+			static constexpr size_t max_buffers_per_thread = 6;
+
 			Mutex m_buffer_mutex;
 			ThreadBlocker m_buffer_blocker;
-			BAN::Array<BlockBuffer, 16> m_buffers;
+			BAN::Array<BlockBuffer, max_threads * max_buffers_per_thread> m_buffers;
+			BAN::Array<ThreadInfo, max_threads> m_thread_infos;
 		};
 
 	private:

kernel/include/kernel/FS/Ext2/Inode.h

@@ -61,8 +61,8 @@ namespace Kernel
 		// NOTE: the inode might have more blocks than what this suggests if it has been shrinked
 		uint32_t max_used_data_block_count() const { return size() / blksize(); }
 
-		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block(uint32_t block, uint32_t index, uint32_t depth);
+		BAN::ErrorOr<BAN::Optional<uint32_t>> block_from_indirect_block(uint32_t& block, uint32_t index, uint32_t depth, bool allocate);
-		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index(uint32_t data_block_index);
+		BAN::ErrorOr<BAN::Optional<uint32_t>> fs_block_of_data_block_index(uint32_t data_block_index, bool allocate);
 
 		BAN::ErrorOr<void> link_inode_to_directory(Ext2Inode&, BAN::StringView name);
 		BAN::ErrorOr<void> remove_inode_from_directory(BAN::StringView name, bool cleanup_directory);
@@ -70,10 +70,9 @@ namespace Kernel
 
 		BAN::ErrorOr<void> cleanup_indirect_block(uint32_t block, uint32_t depth);
 		BAN::ErrorOr<void> cleanup_default_links();
+		BAN::ErrorOr<void> cleanup_data_blocks();
 		BAN::ErrorOr<void> cleanup_from_fs();
 
-		BAN::ErrorOr<uint32_t> allocate_new_block_to_indirect_block(uint32_t& block, uint32_t index, uint32_t depth);
-		BAN::ErrorOr<uint32_t> allocate_new_block(uint32_t data_block_index);
 		BAN::ErrorOr<void> sync();
 
 		uint32_t block_group() const;
@@ -86,6 +85,26 @@ namespace Kernel
 		{}
 		static BAN::ErrorOr<BAN::RefPtr<Ext2Inode>> create(Ext2FS&, uint32_t);
 
+	private:
+		struct ScopedSync
+		{
+			ScopedSync(Ext2Inode& inode)
+				: inode(inode)
+				, inode_info(inode.m_inode)
+			{ }
+
+			~ScopedSync()
+			{
+				if (memcmp(&inode.m_inode, &inode_info, sizeof(Ext2::Inode)) == 0)
+					return;
+				if (auto ret = inode.sync(); ret.is_error())
+					dwarnln("failed to sync inode: {}", ret.error());
+			}
+
+			Ext2Inode& inode;
+			Ext2::Inode inode_info;
+		};
+
 	private:
 		Ext2FS& m_fs;
 		Ext2::Inode m_inode;

kernel/include/kernel/FS/Inode.h

@@ -113,6 +113,8 @@ namespace Kernel
 		BAN::ErrorOr<size_t> recvmsg(msghdr& message, int flags);
 		BAN::ErrorOr<void> getsockname(sockaddr* address, socklen_t* address_len);
 		BAN::ErrorOr<void> getpeername(sockaddr* address, socklen_t* address_len);
+		BAN::ErrorOr<void> getsockopt(int level, int option, void* value, socklen_t* value_len);
+		BAN::ErrorOr<void> setsockopt(int level, int option, const void* value, socklen_t value_len);
 
 		// General API
 		BAN::ErrorOr<size_t> read(off_t, BAN::ByteSpan buffer);
@@ -161,6 +163,8 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> sendmsg_impl(const msghdr&, int)								{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> getsockname_impl(sockaddr*, socklen_t*)							{ return BAN::Error::from_errno(ENOTSUP); }
 		virtual BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*)							{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> getsockopt_impl(int, int, void*, socklen_t*)						{ return BAN::Error::from_errno(ENOTSUP); }
+		virtual BAN::ErrorOr<void> setsockopt_impl(int, int, const void*, socklen_t)				{ return BAN::Error::from_errno(ENOTSUP); }
 
 		// General API
 		virtual BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan)		{ return BAN::Error::from_errno(ENOTSUP); }

kernel/include/kernel/FS/Pipe.h

@@ -2,6 +2,7 @@
 
 #include <BAN/Array.h>
 #include <kernel/FS/Inode.h>
+#include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/ThreadBlocker.h>
 
 namespace Kernel
@@ -38,7 +39,7 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> write_impl(off_t, BAN::ConstByteSpan) override;
 		virtual BAN::ErrorOr<void> fsync_impl() final override { return {}; }
 
-		virtual bool can_read_impl() const override { return m_buffer_size > 0; }
+		virtual bool can_read_impl() const override { return !m_buffer->empty(); }
 		virtual bool can_write_impl() const override { return true; }
 		virtual bool has_error_impl() const override { return m_reading_count == 0; }
 		virtual bool has_hungup_impl() const override { return m_writing_count == 0; }
@@ -54,9 +55,7 @@ namespace Kernel
 		timespec m_ctime {};
 		ThreadBlocker m_thread_blocker;
 
-		BAN::Array<uint8_t, PAGE_SIZE> m_buffer;
+		BAN::UniqPtr<ByteRingBuffer> m_buffer;
-		BAN::Atomic<size_t> m_buffer_size { 0 };
-		size_t m_buffer_tail { 0 };
 
 		BAN::Atomic<uint32_t> m_writing_count { 1 };
 		BAN::Atomic<uint32_t> m_reading_count { 1 };

kernel/include/kernel/FS/USTARModule.h

@@ -1,12 +1,11 @@
 #pragma once
 
 #include <kernel/BootInfo.h>
-#include <kernel/FS/FileSystem.h>
+#include <kernel/FS/Inode.h>
 
 namespace Kernel
 {
 
-	bool is_ustar_boot_module(const BootModule&);
+	BAN::ErrorOr<bool> unpack_boot_module_into_directory(BAN::RefPtr<Inode>, const BootModule&);
-	BAN::ErrorOr<void> unpack_boot_module_into_filesystem(BAN::RefPtr<FileSystem>, const BootModule&);
 
 }

kernel/include/kernel/GDT.h

@@ -151,8 +151,8 @@ namespace Kernel
 
 	private:
 #if ARCH(x86_64)
-		BAN::Array<SegmentDescriptor, 7> m_gdt; // null, kernel code, kernel data, user code, user data, tss low, tss high
+		BAN::Array<SegmentDescriptor, 8> m_gdt; // null, kernel code, kernel data, user code (32 bit), user data, user code (64 bit), tss low, tss high
-		static constexpr uint16_t m_tss_offset = 0x28;
+		static constexpr uint16_t m_tss_offset = 0x30;
 #elif ARCH(i686)
 		BAN::Array<SegmentDescriptor, 9> m_gdt; // null, kernel code, kernel data, user code, user data, processor data, fsbase, gsbase, tss
 		static constexpr uint16_t m_tss_offset = 0x40;

kernel/include/kernel/IDT.h

@@ -18,10 +18,12 @@ namespace Kernel
 
 	constexpr uint8_t IRQ_VECTOR_BASE   = 0x20;
 	constexpr uint8_t IRQ_MSI_BASE      = 0x80;
-	constexpr uint8_t IRQ_SYSCALL       = 0xF0;
+	constexpr uint8_t IRQ_MSI_END       = 0xF0;
-	constexpr uint8_t IRQ_YIELD         = 0xF1;
+#if ARCH(i686)
-	constexpr uint8_t IRQ_IPI           = 0xF2;
+	constexpr uint8_t IRQ_SYSCALL       = 0xF0; // hard coded in kernel/API/Syscall.h
-	constexpr uint8_t IRQ_TIMER         = 0xF3;
+#endif
+	constexpr uint8_t IRQ_IPI           = 0xF1;
+	constexpr uint8_t IRQ_TIMER         = 0xF2;
 
 #if ARCH(x86_64)
 	struct GateDescriptor

kernel/include/kernel/Input/InputDevice.h

@@ -15,6 +15,7 @@ namespace Kernel
 		{
 			Mouse,
 			Keyboard,
+			Joystick,
 		};
 
 	public:

kernel/include/kernel/InterruptStack.h

@@ -27,7 +27,6 @@ namespace Kernel
 		uintptr_t r10;
 		uintptr_t r9;
 		uintptr_t r8;
-
 		uintptr_t rdi;
 		uintptr_t rsi;
 		uintptr_t rbp;
@@ -36,6 +35,18 @@ namespace Kernel
 		uintptr_t rcx;
 		uintptr_t rax;
 	};
+	struct YieldRegisters
+	{
+		uintptr_t r15;
+		uintptr_t r14;
+		uintptr_t r13;
+		uintptr_t r12;
+		uintptr_t rbp;
+		uintptr_t rbx;
+		uintptr_t ret;
+		uintptr_t sp;
+		uintptr_t ip;
+	};
 #elif ARCH(i686)
 	struct InterruptRegisters
 	{
@@ -48,6 +59,16 @@ namespace Kernel
 		uintptr_t ecx;
 		uintptr_t eax;
 	};
+	struct YieldRegisters
+	{
+		uintptr_t ebp;
+		uintptr_t edi;
+		uintptr_t esi;
+		uintptr_t ebx;
+		uintptr_t ret;
+		uintptr_t sp;
+		uintptr_t ip;
+	};
 #endif
 
 }

kernel/include/kernel/Lock/RWLock.h

@@ -0,0 +1,97 @@
+#pragma once
+
+#include <kernel/Lock/Mutex.h>
+#include <kernel/Lock/LockGuard.h>
+
+namespace Kernel
+{
+
+	class RWLock
+	{
+		BAN_NON_COPYABLE(RWLock);
+		BAN_NON_MOVABLE(RWLock);
+	public:
+		RWLock() = default;
+
+		void rd_lock()
+		{
+			LockGuard _(m_mutex);
+			while (m_writers_waiting > 0 || m_writer_active)
+				m_thread_blocker.block_indefinite(&m_mutex);
+			m_readers_active++;
+		}
+
+		void rd_unlock()
+		{
+			LockGuard _(m_mutex);
+			if (--m_readers_active == 0)
+				m_thread_blocker.unblock();
+		}
+
+		void wr_lock()
+		{
+			LockGuard _(m_mutex);
+			m_writers_waiting++;
+			while (m_readers_active > 0 || m_writer_active)
+				m_thread_blocker.block_indefinite(&m_mutex);
+			m_writers_waiting--;
+			m_writer_active = true;
+		}
+
+
+		void wr_unlock()
+		{
+			LockGuard _(m_mutex);
+			m_writer_active = false;
+			m_thread_blocker.unblock();
+		}
+
+	private:
+		Mutex m_mutex;
+		ThreadBlocker m_thread_blocker;
+		uint32_t m_readers_active  { 0 };
+		uint32_t m_writers_waiting { 0 };
+		bool m_writer_active { false };
+	};
+
+	class RWLockRDGuard
+	{
+		BAN_NON_COPYABLE(RWLockRDGuard);
+		BAN_NON_MOVABLE(RWLockRDGuard);
+	public:
+		RWLockRDGuard(RWLock& lock)
+			: m_lock(lock)
+		{
+			m_lock.rd_lock();
+		}
+
+		~RWLockRDGuard()
+		{
+			m_lock.rd_unlock();
+		}
+
+	private:
+		RWLock& m_lock;
+	};
+
+	class RWLockWRGuard
+	{
+		BAN_NON_COPYABLE(RWLockWRGuard);
+		BAN_NON_MOVABLE(RWLockWRGuard);
+	public:
+		RWLockWRGuard(RWLock& lock)
+			: m_lock(lock)
+		{
+			m_lock.wr_lock();
+		}
+
+		~RWLockWRGuard()
+		{
+			m_lock.wr_unlock();
+		}
+
+	private:
+		RWLock& m_lock;
+	};
+
+}

kernel/include/kernel/Memory/ByteRingBuffer.h

@@ -0,0 +1,76 @@
+#pragma once
+
+#include <BAN/ByteSpan.h>
+#include <BAN/UniqPtr.h>
+#include <BAN/Vector.h>
+
+#include <kernel/Memory/Types.h>
+
+namespace Kernel
+{
+
+	class ByteRingBuffer
+	{
+	public:
+		static BAN::ErrorOr<BAN::UniqPtr<ByteRingBuffer>> create(size_t size);
+		~ByteRingBuffer();
+
+		void push(BAN::ConstByteSpan data)
+		{
+			ASSERT(data.size() + m_size <= m_capacity);
+			uint8_t* buffer_head = reinterpret_cast<uint8_t*>(m_vaddr) + (m_tail + m_size) % m_capacity;
+			memcpy(buffer_head, data.data(), data.size());
+			m_size += data.size();
+		}
+
+		void pop(size_t size)
+		{
+			ASSERT(size <= m_size);
+			m_tail = (m_tail + size) % m_capacity;
+			m_size -= size;
+		}
+
+		void pop_back(size_t size)
+		{
+			ASSERT(size <= m_size);
+			m_size -= size;
+		}
+
+		BAN::ConstByteSpan get_data() const
+		{
+			const uint8_t* base = reinterpret_cast<const uint8_t*>(m_vaddr);
+			return { base + m_tail, m_size };
+		}
+
+		uint8_t front() const
+		{
+			ASSERT(!empty());
+			return reinterpret_cast<const uint8_t*>(m_vaddr)[m_tail];
+		}
+
+		uint8_t back() const
+		{
+			ASSERT(!empty());
+			return reinterpret_cast<const uint8_t*>(m_vaddr)[m_tail + m_size];
+		}
+
+		bool empty() const { return m_size == 0; }
+		bool full() const { return m_size == m_capacity; }
+		size_t free() const { return m_capacity - m_size; }
+		size_t size() const { return m_size; }
+		size_t capacity() const { return m_capacity; }
+
+	private:
+		ByteRingBuffer(size_t capacity)
+			: m_capacity(capacity)
+		{ }
+
+	private:
+		size_t m_size { 0 };
+		size_t m_tail { 0 };
+		const size_t m_capacity;
+
+		vaddr_t m_vaddr { 0 };
+	};
+
+}

kernel/include/kernel/Memory/Heap.h

@@ -18,6 +18,8 @@ namespace Kernel
 		static void initialize();
 		static Heap& get();
 
+		void release_boot_modules();
+
 		paddr_t take_free_page();
 		void release_page(paddr_t);
 

kernel/include/kernel/Memory/MemoryBackedRegion.h

@@ -28,6 +28,20 @@ namespace Kernel
 
 	private:
 		MemoryBackedRegion(PageTable&, size_t size, Type, PageTable::flags_t, int status_flags);
+
+	private:
+		struct PhysicalPage
+		{
+			PhysicalPage(paddr_t paddr)
+				: paddr(paddr)
+			{ }
+			~PhysicalPage();
+
+			BAN::Atomic<uint32_t> ref_count { 1 };
+			const paddr_t paddr;
+		};
+		BAN::Vector<PhysicalPage*> m_physical_pages;
+		Mutex m_mutex;
 	};
 
 }

kernel/include/kernel/Memory/PageTable.h

@@ -100,19 +100,21 @@ namespace Kernel
 		static BAN::ErrorOr<PageTable*> create_userspace();
 		~PageTable();
 
-		void unmap_page(vaddr_t, bool send_smp_message = true);
+		void unmap_page(vaddr_t, bool invalidate = true);
 		void unmap_range(vaddr_t, size_t bytes);
 
-		void map_page_at(paddr_t, vaddr_t, flags_t, MemoryType = MemoryType::Normal, bool send_smp_message = true);
+		void map_page_at(paddr_t, vaddr_t, flags_t, MemoryType = MemoryType::Normal, bool invalidate = true);
 		void map_range_at(paddr_t, vaddr_t, size_t bytes, flags_t, MemoryType = MemoryType::Normal);
 
+		void remove_writable_from_range(vaddr_t, size_t);
+
 		paddr_t physical_address_of(vaddr_t) const;
 		flags_t get_page_flags(vaddr_t) const;
 
 		bool is_page_free(vaddr_t) const;
 		bool is_range_free(vaddr_t, size_t bytes) const;
 
-		bool reserve_page(vaddr_t, bool only_free = true, bool send_smp_message = true);
+		bool reserve_page(vaddr_t, bool only_free = true, bool invalidate = true);
 		bool reserve_range(vaddr_t, size_t bytes, bool only_free = true);
 
 		vaddr_t reserve_free_page(vaddr_t first_address, vaddr_t last_address = UINTPTR_MAX);
@@ -121,6 +123,9 @@ namespace Kernel
 		void load();
 		void initial_load();
 
+		void invalidate_page(vaddr_t addr, bool send_smp_message) { invalidate_range(addr, 1, send_smp_message); }
+		void invalidate_range(vaddr_t addr, size_t pages, bool send_smp_message);
+
 		InterruptState lock() const { return m_lock.lock(); }
 		void unlock(InterruptState state) const { m_lock.unlock(state); }
 
@@ -133,8 +138,6 @@ namespace Kernel
 		void map_kernel_memory();
 		void prepare_fast_page();
 
-		static void invalidate(vaddr_t, bool send_smp_message);
-
 		static void map_fast_page(paddr_t);
 		static void unmap_fast_page();
 

kernel/include/kernel/Memory/PhysicalRange.h

@@ -10,7 +10,7 @@ namespace Kernel
 	class PhysicalRange
 	{
 	public:
-		PhysicalRange(paddr_t, size_t);
+		PhysicalRange(paddr_t, uint64_t);
 
 		paddr_t reserve_page();
 		void release_page(paddr_t);

kernel/include/kernel/Memory/Types.h

@@ -4,7 +4,7 @@
 
 #if ARCH(x86_64)
 	#define KERNEL_OFFSET 0xFFFFFFFF80000000
-	#define USERSPACE_END 0xFFFF800000000000
+	#define USERSPACE_END 0x800000000000
 #elif ARCH(i686)
 	#define KERNEL_OFFSET 0xC0000000
 	#define USERSPACE_END 0xC0000000

kernel/include/kernel/Memory/VirtualRange.h

@@ -20,8 +20,6 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> create_to_vaddr_range(PageTable&, vaddr_t vaddr_start, vaddr_t vaddr_end, size_t, PageTable::flags_t flags, bool preallocate_pages, bool add_guard_pages);
 		~VirtualRange();
 
-		BAN::ErrorOr<BAN::UniqPtr<VirtualRange>> clone(PageTable&);
-
 		vaddr_t vaddr() const { return m_vaddr + (m_has_guard_pages ? PAGE_SIZE : 0); }
 		size_t size() const { return m_size - (m_has_guard_pages ? 2 * PAGE_SIZE : 0); }
 		PageTable::flags_t flags() const { return m_flags; }

kernel/include/kernel/Networking/ARPTable.h

@@ -31,35 +31,18 @@ namespace Kernel
 
 	public:
 		static BAN::ErrorOr<BAN::UniqPtr<ARPTable>> create();
-		~ARPTable();
 
 		BAN::ErrorOr<BAN::MACAddress> get_mac_from_ipv4(NetworkInterface&, BAN::IPv4Address);
 
-		void add_arp_packet(NetworkInterface&, BAN::ConstByteSpan);
+		BAN::ErrorOr<void> handle_arp_packet(NetworkInterface&, BAN::ConstByteSpan);
 
 	private:
-		ARPTable();
+		ARPTable() = default;
-
-		void packet_handle_task();
-		BAN::ErrorOr<void> handle_arp_packet(NetworkInterface&, const ARPPacket&);
 
 	private:
-		struct PendingArpPacket
+		SpinLock m_arp_table_lock;
-		{
-			NetworkInterface& interface;
-			ARPPacket packet;
-		};
-
-	private:
-		SpinLock m_table_lock;
-		SpinLock m_pending_lock;
-
 		BAN::HashMap<BAN::IPv4Address, BAN::MACAddress> m_arp_table;
 
-		Thread*										m_thread { nullptr };
-		BAN::CircularQueue<PendingArpPacket, 128>	m_pending_packets;
-		ThreadBlocker								m_pending_thread_blocker;
-
 		friend class BAN::UniqPtr<ARPTable>;
 	};
 

kernel/include/kernel/Networking/E1000/E1000.h

@@ -23,14 +23,14 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<E1000>> create(PCI::Device&);
 		~E1000();
 
-		virtual BAN::MACAddress get_mac_address() const override { return m_mac_address; }
+		BAN::MACAddress get_mac_address() const override { return m_mac_address; }
 
-		virtual bool link_up() override { return m_link_up; }
+		bool link_up() override { return m_link_up; }
-		virtual int link_speed() override;
+		int link_speed() override;
 
-		virtual size_t payload_mtu() const override { return E1000_RX_BUFFER_SIZE - sizeof(EthernetHeader); }
+		size_t payload_mtu() const override { return E1000_RX_BUFFER_SIZE - sizeof(EthernetHeader); }
 
-		virtual void handle_irq() final override;
+		void handle_irq() final override;
 
 	protected:
 		E1000(PCI::Device& pci_device)
@@ -45,12 +45,12 @@ namespace Kernel
 		uint32_t read32(uint16_t reg);
 		void write32(uint16_t reg, uint32_t value);
 
-		virtual BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::ConstByteSpan) override;
+		BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::Span<const BAN::ConstByteSpan> payload) override;
 
-		virtual bool can_read_impl() const override { return false; }
+		bool can_read_impl() const override { return false; }
-		virtual bool can_write_impl() const override { return false; }
+		bool can_write_impl() const override { return false; }
-		virtual bool has_error_impl() const override { return false; }
+		bool has_error_impl() const override { return false; }
-		virtual bool has_hungup_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
 
 	private:
 		BAN::ErrorOr<void> read_mac_address();
@@ -61,7 +61,7 @@ namespace Kernel
 		void enable_link();
 		BAN::ErrorOr<void> enable_interrupt();
 
-		void handle_receive();
+		void receive_thread();
 
 	protected:
 		PCI::Device&					m_pci_device;
@@ -75,6 +75,10 @@ namespace Kernel
 		BAN::UniqPtr<DMARegion>	m_tx_descriptor_region;
 		SpinLock m_lock;
 
+		bool m_thread_should_die { false };
+		BAN::Atomic<bool> m_thread_is_dead { true };
+		ThreadBlocker m_thread_blocker;
+
 		BAN::MACAddress	m_mac_address {};
 		bool			m_link_up { false };
 

kernel/include/kernel/Networking/E1000/E1000E.h

@@ -12,8 +12,8 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<E1000E>> create(PCI::Device&);
 
 	protected:
-		virtual void detect_eeprom() override;
+		void detect_eeprom() override;
-		virtual uint32_t eeprom_read(uint8_t addr) override;
+		uint32_t eeprom_read(uint8_t addr) override;
 
 	private:
 		E1000E(PCI::Device& pci_device)

kernel/include/kernel/Networking/IPv4Layer.h

@@ -38,14 +38,13 @@ namespace Kernel
 
 	public:
 		static BAN::ErrorOr<BAN::UniqPtr<IPv4Layer>> create();
-		~IPv4Layer();
 
 		ARPTable& arp_table() { return *m_arp_table; }
 
-		void add_ipv4_packet(NetworkInterface&, BAN::ConstByteSpan);
+		BAN::ErrorOr<void> handle_ipv4_packet(NetworkInterface&, BAN::ConstByteSpan);
 
 		virtual void unbind_socket(uint16_t port) override;
-		virtual BAN::ErrorOr<void> bind_socket_to_unused(BAN::RefPtr<NetworkSocket>, const sockaddr* send_address, socklen_t send_address_len) override;
+		virtual BAN::ErrorOr<void> bind_socket_with_target(BAN::RefPtr<NetworkSocket>, const sockaddr* target_address, socklen_t target_address_len) override;
 		virtual BAN::ErrorOr<void> bind_socket_to_address(BAN::RefPtr<NetworkSocket>, const sockaddr* address, socklen_t address_len) override;
 		virtual BAN::ErrorOr<void> get_socket_address(BAN::RefPtr<NetworkSocket>, sockaddr* address, socklen_t* address_len) override;
 
@@ -55,33 +54,15 @@ namespace Kernel
 		virtual size_t header_size() const override { return sizeof(IPv4Header); }
 
 	private:
-		IPv4Layer();
+		IPv4Layer() = default;
 
-		void add_ipv4_header(BAN::ByteSpan packet, BAN::IPv4Address src_ipv4, BAN::IPv4Address dst_ipv4, uint8_t protocol) const;
+		BAN::ErrorOr<in_port_t> find_free_port();
-
-		void packet_handle_task();
-		BAN::ErrorOr<void> handle_ipv4_packet(NetworkInterface&, BAN::ByteSpan);
 
 	private:
-		struct PendingIPv4Packet
+		BAN::UniqPtr<ARPTable> m_arp_table;
-		{
-			NetworkInterface& interface;
-		};
 
-	private:
+		RecursiveSpinLock m_bound_socket_lock;
-		RecursiveSpinLock		m_bound_socket_lock;
+		BAN::HashMap<int, BAN::WeakPtr<NetworkSocket>> m_bound_sockets;
-
-		BAN::UniqPtr<ARPTable>	m_arp_table;
-		Thread*					m_thread { nullptr };
-
-		static constexpr size_t pending_packet_buffer_size = 128 * PAGE_SIZE;
-		BAN::UniqPtr<VirtualRange>					m_pending_packet_buffer;
-		BAN::CircularQueue<PendingIPv4Packet, 128>	m_pending_packets;
-		ThreadBlocker									m_pending_thread_blocker;
-		SpinLock									m_pending_lock;
-		size_t										m_pending_total_size { 0 };
-
-		BAN::HashMap<int, BAN::WeakPtr<NetworkSocket>>	m_bound_sockets;
 
 		friend class BAN::UniqPtr<IPv4Layer>;
 	};

kernel/include/kernel/Networking/Loopback.h

@@ -9,6 +9,7 @@ namespace Kernel
 	{
 	public:
 		static constexpr size_t buffer_size = BAN::numeric_limits<uint16_t>::max() + 1;
+		static constexpr size_t buffer_count = 32;
 
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<LoopbackInterface>> create();
@@ -24,8 +25,9 @@ namespace Kernel
 		LoopbackInterface()
 			: NetworkInterface(Type::Loopback)
 		{}
+		~LoopbackInterface();
 
-		BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::ConstByteSpan) override;
+		BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::Span<const BAN::ConstByteSpan> payload) override;
 
 		bool can_read_impl() const override { return false; }
 		bool can_write_impl() const override { return false; }
@@ -33,8 +35,27 @@ namespace Kernel
 		bool has_hungup_impl() const override { return false; }
 
 	private:
-		SpinLock m_buffer_lock;
+		void receive_thread();
+
+	private:
+		struct Descriptor
+		{
+			uint8_t* addr;
+			uint32_t size;
+			uint8_t state;
+		};
+
+	private:
+		Mutex m_buffer_lock;
 		BAN::UniqPtr<VirtualRange> m_buffer;
+
+		uint32_t m_buffer_tail { 0 };
+		uint32_t m_buffer_head { 0 };
+		Descriptor m_descriptors[buffer_count] {};
+
+		bool m_thread_should_die { false };
+		BAN::Atomic<bool> m_thread_is_dead { true };
+		ThreadBlocker m_thread_blocker;
 	};
 
 }

kernel/include/kernel/Networking/NetworkInterface.h

@@ -60,7 +60,11 @@ namespace Kernel
 		virtual dev_t rdev() const override { return m_rdev; }
 		virtual BAN::StringView name() const override { return m_name; }
 
-		virtual BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::ConstByteSpan) = 0;
+		BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::ConstByteSpan payload)
+		{
+			return send_bytes(destination, protocol, { &payload, 1 });
+		}
+		virtual BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::Span<const BAN::ConstByteSpan> payload) = 0;
 
 	private:
 		const Type m_type;

kernel/include/kernel/Networking/NetworkLayer.h

@@ -11,7 +11,7 @@ namespace Kernel
 		BAN::IPv4Address src_ipv4				{ 0 };
 		BAN::IPv4Address dst_ipv4				{ 0 };
 		BAN::NetworkEndian<uint16_t> protocol	{ 0 };
-		BAN::NetworkEndian<uint16_t> extra		{ 0 };
+		BAN::NetworkEndian<uint16_t> length		{ 0 };
 	};
 	static_assert(sizeof(PseudoHeader) == 12);
 
@@ -23,7 +23,7 @@ namespace Kernel
 		virtual ~NetworkLayer() {}
 
 		virtual void unbind_socket(uint16_t port) = 0;
-		virtual BAN::ErrorOr<void> bind_socket_to_unused(BAN::RefPtr<NetworkSocket>, const sockaddr* send_address, socklen_t send_address_len) = 0;
+		virtual BAN::ErrorOr<void> bind_socket_with_target(BAN::RefPtr<NetworkSocket>, const sockaddr* target_address, socklen_t target_address_len) = 0;
 		virtual BAN::ErrorOr<void> bind_socket_to_address(BAN::RefPtr<NetworkSocket>, const sockaddr* address, socklen_t address_len) = 0;
 		virtual BAN::ErrorOr<void> get_socket_address(BAN::RefPtr<NetworkSocket>, sockaddr* address, socklen_t* address_len) = 0;
 
@@ -36,6 +36,7 @@ namespace Kernel
 		NetworkLayer() = default;
 	};
 
-	uint16_t calculate_internet_checksum(BAN::ConstByteSpan packet, const PseudoHeader& pseudo_header);
+	uint16_t calculate_internet_checksum(BAN::ConstByteSpan buffer);
+	uint16_t calculate_internet_checksum(BAN::Span<const BAN::ConstByteSpan> buffers);
 
 }

kernel/include/kernel/Networking/NetworkSocket.h

@@ -5,6 +5,8 @@
 #include <kernel/Networking/NetworkInterface.h>
 #include <kernel/Networking/NetworkLayer.h>
 
+#include <netinet/in.h>
+
 namespace Kernel
 {
 
@@ -24,18 +26,30 @@ namespace Kernel
 		static constexpr uint16_t PORT_NONE = 0;
 
 	public:
-		void bind_interface_and_port(NetworkInterface*, uint16_t port);
+		void bind_address_and_port(const sockaddr*, socklen_t);
 		~NetworkSocket();
 
-		NetworkInterface& interface() { ASSERT(m_interface); return *m_interface; }
+		BAN::ErrorOr<BAN::RefPtr<NetworkInterface>> interface(const sockaddr* target, socklen_t target_len);
 
 		virtual size_t protocol_header_size() const = 0;
-		virtual void add_protocol_header(BAN::ByteSpan packet, uint16_t dst_port, PseudoHeader) = 0;
+		virtual void get_protocol_header(BAN::ByteSpan header, BAN::ConstByteSpan payload, uint16_t dst_port, PseudoHeader) = 0;
 		virtual NetworkProtocol protocol() const = 0;
 
 		virtual void receive_packet(BAN::ConstByteSpan, const sockaddr* sender, socklen_t sender_len) = 0;
 
-		bool is_bound() const { return m_interface != nullptr; }
+		bool is_bound() const { return m_address_len >= static_cast<socklen_t>(sizeof(sa_family_t)) && m_address.ss_family != AF_UNSPEC; }
+		in_port_t bound_port() const
+		{
+			ASSERT(is_bound());
+			ASSERT(m_address.ss_family == AF_INET && m_address_len >= static_cast<socklen_t>(sizeof(sockaddr_in)));
+			return BAN::network_endian_to_host(reinterpret_cast<const sockaddr_in*>(&m_address)->sin_port);
+		}
+
+		const sockaddr* address() const { return reinterpret_cast<const sockaddr*>(&m_address); }
+		socklen_t address_len() const { return m_address_len; }
+
+	private:
+		bool can_interface_send_to(const NetworkInterface&, const sockaddr*, socklen_t) const;
 
 	protected:
 		NetworkSocket(NetworkLayer&, const Socket::Info&);
@@ -45,9 +59,9 @@ namespace Kernel
 		virtual BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*) override = 0;
 
 	protected:
-		NetworkLayer&		m_network_layer;
+		NetworkLayer&    m_network_layer;
-		NetworkInterface*	m_interface	= nullptr;
+		sockaddr_storage m_address       { .ss_family = AF_UNSPEC, .ss_storage = {} };
-		uint16_t			m_port { PORT_NONE };
+		socklen_t        m_address_len   { 0 };
 	};
 
 }

kernel/include/kernel/Networking/RTL8169/RTL8169.h

@@ -29,9 +29,11 @@ namespace Kernel
 			: NetworkInterface(Type::Ethernet)
 			, m_pci_device(pci_device)
 		{ }
+		~RTL8169();
+
 		BAN::ErrorOr<void> initialize();
 
-		virtual BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::ConstByteSpan) override;
+		virtual BAN::ErrorOr<void> send_bytes(BAN::MACAddress destination, EtherType protocol, BAN::Span<const BAN::ConstByteSpan>) override;
 
 		virtual bool can_read_impl() const override { return false; }
 		virtual bool can_write_impl() const override { return false; }
@@ -47,7 +49,7 @@ namespace Kernel
 		void enable_link();
 		BAN::ErrorOr<void> enable_interrupt();
 
-		void handle_receive();
+		void receive_thread();
 
 	protected:
 		PCI::Device&					m_pci_device;
@@ -63,6 +65,9 @@ namespace Kernel
 		BAN::UniqPtr<DMARegion>	m_tx_descriptor_region;
 
 		SpinLock m_lock;
+
+		bool m_thread_should_die { false };
+		BAN::Atomic<bool> m_thread_is_dead { true };
 		ThreadBlocker m_thread_blocker;
 
 		uint32_t m_rx_current { 0 };

kernel/include/kernel/Networking/TCPSocket.h

@@ -4,7 +4,7 @@
 #include <BAN/Endianness.h>
 #include <BAN/Queue.h>
 #include <kernel/Lock/Mutex.h>
-#include <kernel/Memory/VirtualRange.h>
+#include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/Networking/NetworkInterface.h>
 #include <kernel/Networking/NetworkSocket.h>
 #include <kernel/Thread.h>
@@ -50,28 +50,30 @@ namespace Kernel
 		static BAN::ErrorOr<BAN::RefPtr<TCPSocket>> create(NetworkLayer&, const Info&);
 		~TCPSocket();
 
-		virtual NetworkProtocol protocol() const override { return NetworkProtocol::TCP; }
+		NetworkProtocol protocol() const override { return NetworkProtocol::TCP; }
 
-		virtual size_t protocol_header_size() const override { return sizeof(TCPHeader) + m_tcp_options_bytes; }
+		size_t protocol_header_size() const override { return sizeof(TCPHeader) + m_tcp_options_bytes; }
-		virtual void add_protocol_header(BAN::ByteSpan packet, uint16_t dst_port, PseudoHeader) override;
+		void get_protocol_header(BAN::ByteSpan header, BAN::ConstByteSpan payload, uint16_t dst_port, PseudoHeader) override;
 
 	protected:
-		virtual BAN::ErrorOr<long> accept_impl(sockaddr*, socklen_t*, int) override;
+		BAN::ErrorOr<long> accept_impl(sockaddr*, socklen_t*, int) override;
-		virtual BAN::ErrorOr<void> connect_impl(const sockaddr*, socklen_t) override;
+		BAN::ErrorOr<void> connect_impl(const sockaddr*, socklen_t) override;
-		virtual BAN::ErrorOr<void> listen_impl(int) override;
+		BAN::ErrorOr<void> listen_impl(int) override;
-		virtual BAN::ErrorOr<void> bind_impl(const sockaddr*, socklen_t) override;
+		BAN::ErrorOr<void> bind_impl(const sockaddr*, socklen_t) override;
-		virtual BAN::ErrorOr<size_t> recvmsg_impl(msghdr& message, int flags) override;
+		BAN::ErrorOr<size_t> recvmsg_impl(msghdr& message, int flags) override;
-		virtual BAN::ErrorOr<size_t> sendmsg_impl(const msghdr& message, int flags) override;
+		BAN::ErrorOr<size_t> sendmsg_impl(const msghdr& message, int flags) override;
-		virtual BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*) override;
+		BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*) override;
+		BAN::ErrorOr<void> getsockopt_impl(int, int, void*, socklen_t*) override;
+		BAN::ErrorOr<void> setsockopt_impl(int, int, const void*, socklen_t) override;
 
-		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) override;
+		BAN::ErrorOr<long> ioctl_impl(int, void*) override;
 
-		virtual void receive_packet(BAN::ConstByteSpan, const sockaddr* sender, socklen_t sender_len) override;
+		void receive_packet(BAN::ConstByteSpan, const sockaddr* sender, socklen_t sender_len) override;
 
-		virtual bool can_read_impl() const override;
+		bool can_read_impl() const override;
-		virtual bool can_write_impl() const override;
+		bool can_write_impl() const override;
-		virtual bool has_error_impl() const override { return false; }
+		bool has_error_impl() const override { return false; }
-		virtual bool has_hungup_impl() const override;
+		bool has_hungup_impl() const override;
 
 	private:
 		enum class State
@@ -91,33 +93,32 @@ namespace Kernel
 
 		struct RecvWindowInfo
 		{
-			uint32_t					start_seq	{ 0 }; // sequence number of first byte in buffer
+			uint32_t start_seq      { 0 }; // sequence number of first byte in buffer
 
-			bool						has_ghost_byte { false };
+			bool     has_ghost_byte { false };
 
-			uint32_t					data_size	{ 0 }; // number of bytes in this buffer
+			uint8_t  scale_shift    { 0 }; // window scale
-			uint8_t						scale_shift	{ 0 }; // window scale
+			BAN::UniqPtr<ByteRingBuffer> buffer;
-			BAN::UniqPtr<VirtualRange>	buffer;
 		};
 
 		struct SendWindowInfo
 		{
-			uint32_t					mss				{ 0 }; // maximum segment size
+			uint32_t mss             { 0 }; // maximum segment size
-			uint16_t					non_scaled_size	{ 0 }; // window size without scaling
+			uint16_t non_scaled_size { 0 }; // window size without scaling
-			uint8_t						scale_shift		{ 0 }; // window scale
+			uint8_t  scale_shift     { 0 }; // window scale
-			uint32_t 					scaled_size() const { return (uint32_t)non_scaled_size << scale_shift; }
+			uint32_t scaled_size() const { return (uint32_t)non_scaled_size << scale_shift; }
 
-			uint32_t					start_seq		{ 0 }; // sequence number of first byte in buffer
+			uint32_t start_seq       { 0 }; // sequence number of first byte in buffer
-			uint32_t					current_seq		{ 0 }; // sequence number of next send
+			uint32_t current_seq     { 0 }; // sequence number of next send
-			uint32_t					current_ack		{ 0 }; // sequence number aknowledged by connection
+			uint32_t current_ack     { 0 }; // sequence number aknowledged by connection
 
-			uint64_t					last_send_ms	{ 0 }; // last send time, used for retransmission timeout
+			uint64_t last_send_ms    { 0 }; // last send time, used for retransmission timeout
 
-			bool						has_ghost_byte { false };
+			bool     has_ghost_byte  { false };
+			bool     had_zero_window { false };
 
-			uint32_t					data_size		{ 0 }; // number of bytes in this buffer
+			uint32_t sent_size       { 0 }; // number of bytes in this buffer that have been sent
-			uint32_t					sent_size		{ 0 }; // number of bytes in this buffer that have been sent
+			BAN::UniqPtr<ByteRingBuffer> buffer;
-			BAN::UniqPtr<VirtualRange>	buffer;
 		};
 
 		struct ConnectionInfo
@@ -131,6 +132,8 @@ namespace Kernel
 		{
 			ConnectionInfo target;
 			uint32_t target_start_seq;
+			uint16_t maximum_seqment_size;
+			uint8_t window_scale;
 		};
 
 		struct ListenKey
@@ -165,8 +168,17 @@ namespace Kernel
 		State m_next_state		{ State::Closed };
 		uint8_t m_next_flags	{ 0 };
 
+		size_t m_last_sent_window_size { 0 };
+
 		Thread* m_thread { nullptr };
 
+		// TODO: actually support these
+		bool m_keep_alive { false };
+		bool m_no_delay { false };
+
+		bool m_should_send_zero_window { false };
+		bool m_should_send_window_update { false };
+
 		uint64_t m_time_wait_start_ms { 0 };
 
 		ThreadBlocker m_thread_blocker;

kernel/include/kernel/Networking/UDPSocket.h

@@ -25,26 +25,28 @@ namespace Kernel
 	public:
 		static BAN::ErrorOr<BAN::RefPtr<UDPSocket>> create(NetworkLayer&, const Socket::Info&);
 
-		virtual NetworkProtocol protocol() const override { return NetworkProtocol::UDP; }
+		NetworkProtocol protocol() const override { return NetworkProtocol::UDP; }
 
-		virtual size_t protocol_header_size() const override { return sizeof(UDPHeader); }
+		size_t protocol_header_size() const override { return sizeof(UDPHeader); }
-		virtual void add_protocol_header(BAN::ByteSpan packet, uint16_t dst_port, PseudoHeader) override;
+		void get_protocol_header(BAN::ByteSpan header, BAN::ConstByteSpan payload, uint16_t dst_port, PseudoHeader) override;
 
 	protected:
-		virtual void receive_packet(BAN::ConstByteSpan, const sockaddr* sender, socklen_t sender_len) override;
+		void receive_packet(BAN::ConstByteSpan, const sockaddr* sender, socklen_t sender_len) override;
 
-		virtual BAN::ErrorOr<void> connect_impl(const sockaddr*, socklen_t) override;
+		BAN::ErrorOr<void> connect_impl(const sockaddr*, socklen_t) override;
-		virtual BAN::ErrorOr<void> bind_impl(const sockaddr* address, socklen_t address_len) override;
+		BAN::ErrorOr<void> bind_impl(const sockaddr* address, socklen_t address_len) override;
-		virtual BAN::ErrorOr<size_t> recvmsg_impl(msghdr& message, int flags) override;
+		BAN::ErrorOr<size_t> recvmsg_impl(msghdr& message, int flags) override;
-		virtual BAN::ErrorOr<size_t> sendmsg_impl(const msghdr& message, int flags) override;
+		BAN::ErrorOr<size_t> sendmsg_impl(const msghdr& message, int flags) override;
-		virtual BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*) override { return BAN::Error::from_errno(ENOTCONN); }
+		BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*) override { return BAN::Error::from_errno(ENOTCONN); }
+		BAN::ErrorOr<void> getsockopt_impl(int, int, void*, socklen_t*) override;
+		BAN::ErrorOr<void> setsockopt_impl(int, int, const void*, socklen_t) override;
 
-		virtual BAN::ErrorOr<long> ioctl_impl(int, void*) override;
+		BAN::ErrorOr<long> ioctl_impl(int, void*) override;
 
-		virtual bool can_read_impl() const override { return !m_packets.empty(); }
+		bool can_read_impl() const override { return !m_packets.empty(); }
-		virtual bool can_write_impl() const override { return true; }
+		bool can_write_impl() const override { return true; }
-		virtual bool has_error_impl() const override { return false; }
+		bool has_error_impl() const override { return false; }
-		virtual bool has_hungup_impl() const override { return false; }
+		bool has_hungup_impl() const override { return false; }
 
 	private:
 		UDPSocket(NetworkLayer&, const Socket::Info&);

kernel/include/kernel/Networking/UNIX/Socket.h

@@ -6,7 +6,6 @@
 #include <kernel/FS/Socket.h>
 #include <kernel/FS/TmpFS/Inode.h>
 #include <kernel/FS/VirtualFileSystem.h>
-#include <kernel/Lock/SpinLock.h>
 #include <kernel/OpenFileDescriptorSet.h>
 
 namespace Kernel
@@ -32,6 +31,8 @@ namespace Kernel
 		virtual BAN::ErrorOr<size_t> recvmsg_impl(msghdr& message, int flags) override;
 		virtual BAN::ErrorOr<size_t> sendmsg_impl(const msghdr& message, int flags) override;
 		virtual BAN::ErrorOr<void> getpeername_impl(sockaddr*, socklen_t*) override;
+		virtual BAN::ErrorOr<void> getsockopt_impl(int, int, void*, socklen_t*) override;
+		virtual BAN::ErrorOr<void> setsockopt_impl(int, int, const void*, socklen_t) override;
 
 		virtual bool can_read_impl() const override;
 		virtual bool can_write_impl() const override;
@@ -69,9 +70,10 @@ namespace Kernel
 			size_t size;
 			BAN::Vector<FDWrapper> fds;
 			BAN::Optional<struct ucred> ucred;
+			BAN::WeakPtr<UnixDomainSocket> sender;
 		};
 
-		BAN::ErrorOr<void> add_packet(const msghdr&, PacketInfo&&);
+		BAN::ErrorOr<size_t> add_packet(const msghdr&, PacketInfo&&, bool dont_block);
 
 	private:
 		const Socket::Type		m_socket_type;
@@ -81,10 +83,14 @@ namespace Kernel
 
 		BAN::CircularQueue<PacketInfo, 512>	m_packet_infos;
 		size_t								m_packet_size_total { 0 };
+		size_t								m_packet_buffer_tail { 0 };
 		BAN::UniqPtr<VirtualRange>			m_packet_buffer;
-		Mutex								m_packet_lock;
+		mutable Mutex						m_packet_lock;
 		ThreadBlocker						m_packet_thread_blocker;
 
+		BAN::Atomic<size_t> m_sndbuf { 0 };
+		BAN::Atomic<size_t> m_bytes_sent { 0 };
+
 		friend class BAN::RefPtr<UnixDomainSocket>;
 	};
 

kernel/include/kernel/OpenFileDescriptorSet.h

@@ -44,6 +44,10 @@ namespace Kernel
 		void close_all();
 		void close_cloexec();
 
+		bool is_cloexec(int fd);
+		void add_cloexec(int fd);
+		void remove_cloexec(int fd);
+
 		BAN::ErrorOr<void> flock(int fd, int op);
 
 		BAN::ErrorOr<size_t> read(int fd, BAN::ByteSpan);
@@ -84,27 +88,6 @@ namespace Kernel
 			friend class BAN::RefPtr<OpenFileDescription>;
 		};
 
-		struct OpenFile
-		{
-			OpenFile() = default;
-			OpenFile(BAN::RefPtr<OpenFileDescription> description, int descriptor_flags)
-				: description(BAN::move(description))
-				, descriptor_flags(descriptor_flags)
-			{ }
-
-			BAN::RefPtr<Inode> inode() const { ASSERT(description); return description->file.inode; }
-			BAN::StringView    path()  const { ASSERT(description); return description->file.canonical_path.sv(); }
-
-			int& status_flags() { ASSERT(description); return description->status_flags; }
-			const int& status_flags() const { ASSERT(description); return description->status_flags; }
-
-			off_t& offset() { ASSERT(description); return description->offset; }
-			const off_t& offset() const { ASSERT(description); return description->offset; }
-
-			BAN::RefPtr<OpenFileDescription> description;
-			int descriptor_flags { 0 };
-		};
-
 		BAN::ErrorOr<void> validate_fd(int) const;
 		BAN::ErrorOr<int> get_free_fd() const;
 		BAN::ErrorOr<void> get_free_fd_pair(int fds[2]) const;
@@ -139,7 +122,8 @@ namespace Kernel
 		const Credentials& m_credentials;
 		mutable Mutex m_mutex;
 
-		BAN::Array<OpenFile, OPEN_MAX> m_open_files;
+		BAN::Array<BAN::RefPtr<OpenFileDescription>, OPEN_MAX> m_open_files;
+		BAN::Array<uint32_t, (OPEN_MAX + 31) / 32> m_cloexec_files {};
 	};
 
 }

kernel/include/kernel/PCI.h

@@ -187,7 +187,7 @@ namespace Kernel::PCI
 		void initialize_impl();
 
 	private:
-		static constexpr uint8_t m_msi_count = IRQ_SYSCALL - IRQ_MSI_BASE;
+		static constexpr uint8_t m_msi_count = IRQ_MSI_END - IRQ_MSI_BASE;
 		using PCIBus = BAN::Array<BAN::Array<Device, 8>, 32>;
 		BAN::Array<PCIBus, 256>  m_buses;
 		BAN::Array<paddr_t, 256> m_bus_pcie_paddr;

kernel/include/kernel/Process.h

@@ -9,6 +9,7 @@
 #include <kernel/ELF.h>
 #include <kernel/FS/Inode.h>
 #include <kernel/Lock/Mutex.h>
+#include <kernel/Lock/RWLock.h>
 #include <kernel/Memory/Heap.h>
 #include <kernel/Memory/MemoryRegion.h>
 #include <kernel/Memory/SharedMemoryObject.h>
@@ -146,6 +147,8 @@ namespace Kernel
 		BAN::ErrorOr<long> sys_epoll_ctl(int epfd, int op, int fd, struct epoll_event* event);
 		BAN::ErrorOr<long> sys_epoll_pwait2(int epfd, struct epoll_event* events, int maxevents, const struct timespec* timeout, const sigset_t* sigmask);
 
+		BAN::ErrorOr<long> sys_eventfd(unsigned int initval_hi, int flags);
+
 		BAN::ErrorOr<long> sys_pipe(int fildes[2]);
 		BAN::ErrorOr<long> sys_dup2(int fildes, int fildes2);
 
@@ -218,8 +221,6 @@ namespace Kernel
 		BAN::ErrorOr<long> sys_tcgetpgrp(int fd);
 		BAN::ErrorOr<long> sys_tcsetpgrp(int fd, pid_t pgid);
 
-		BAN::ErrorOr<long> sys_termid(char*);
-
 		BAN::ErrorOr<long> sys_clock_gettime(clockid_t, timespec*);
 
 		BAN::ErrorOr<long> sys_load_keymap(const char* path);
@@ -228,6 +229,8 @@ namespace Kernel
 
 		static Process& current() { return Thread::current().process(); }
 
+		vaddr_t shared_page_vaddr() const { return m_shared_page_vaddr; }
+
 		PageTable& page_table() { return m_page_table ? *m_page_table : PageTable::kernel(); }
 
 		size_t proc_meminfo(off_t offset, BAN::ByteSpan) const;
@@ -272,17 +275,20 @@ namespace Kernel
 		};
 
 		// Adds new region to the sorted array of mapped regions
+		// You must hold writer end of m_mapped_region_lock when calling this.
 		BAN::ErrorOr<void> add_mapped_region(BAN::UniqPtr<MemoryRegion>&&);
 		// If address is contained by a region, returns the index of that.
 		// Otherwise returns the address of the first region after this address.
+		// You must hold reader end of m_mapped_region_lock when calling this.
 		size_t find_mapped_region(vaddr_t) const;
 
 		BAN::ErrorOr<VirtualFileSystem::File> find_file(int fd, const char* path, int flags) const;
 		BAN::ErrorOr<FileParent> find_parent_file(int fd, const char* path, int flags) const;
 		BAN::ErrorOr<VirtualFileSystem::File> find_relative_parent(int fd, const char* path) const;
 
-		BAN::ErrorOr<void> validate_string_access(const char*);
+		BAN::ErrorOr<void> read_from_user(const void* user_addr, void* out, size_t size);
-		BAN::ErrorOr<void> validate_pointer_access(const void*, size_t, bool needs_write);
+		BAN::ErrorOr<void> read_string_from_user(const char* user_addr, char* out, size_t max_size);
+		BAN::ErrorOr<void> write_to_user(void* user_addr, const void* in, size_t size);
 		BAN::ErrorOr<MemoryRegion*> validate_and_pin_pointer_access(const void*, size_t, bool needs_write);
 
 		uint64_t signal_pending_mask() const
@@ -327,6 +333,7 @@ namespace Kernel
 
 		OpenFileDescriptorSet m_open_file_descriptors;
 
+		mutable RWLock m_memory_region_lock;
 		BAN::Vector<BAN::UniqPtr<MemoryRegion>> m_mapped_regions;
 
 		pid_t m_sid;
@@ -342,6 +349,22 @@ namespace Kernel
 		VirtualFileSystem::File m_working_directory;
 		VirtualFileSystem::File m_root_file;
 
+		vaddr_t m_shared_page_vaddr { 0 };
+
+		struct futex_t
+		{
+			Mutex mutex;
+			ThreadBlocker blocker;
+			uint32_t waiters { 0 };
+			uint32_t to_wakeup { 0 };
+		};
+
+		static BAN::HashMap<paddr_t, BAN::UniqPtr<futex_t>> s_futexes;
+		static Mutex s_futex_lock;
+
+		BAN::HashMap<paddr_t, BAN::UniqPtr<futex_t>> m_futexes;
+		Mutex m_futex_lock;
+
 		BAN::Vector<Thread*> m_threads;
 
 		struct pthread_info_t
@@ -375,6 +398,7 @@ namespace Kernel
 		BAN::UniqPtr<PageTable> m_page_table;
 		BAN::RefPtr<TTY> m_controlling_terminal;
 
+		friend class OpenFileDescriptorSet;
 		friend class Thread;
 	};
 

kernel/include/kernel/Processor.h

@@ -3,10 +3,12 @@
 #include <BAN/Atomic.h>
 #include <BAN/ForwardList.h>
 
+#include <kernel/API/SharedPage.h>
 #include <kernel/Arch.h>
 #include <kernel/GDT.h>
 #include <kernel/IDT.h>
 #include <kernel/InterruptStack.h>
+#include <kernel/Memory/Types.h>
 #include <kernel/ProcessorID.h>
 #include <kernel/Scheduler.h>
 
@@ -33,6 +35,7 @@ namespace Kernel
 				FlushTLB,
 				NewThread,
 				UnblockThread,
+				UpdateTSC,
 				StackTrace,
 			};
 			SMPMessage* next { nullptr };
@@ -43,6 +46,7 @@ namespace Kernel
 				{
 					uintptr_t vaddr;
 					size_t page_count;
+					void* page_table;
 				} flush_tlb;
 				SchedulerQueue::Node* new_thread;
 				SchedulerQueue::Node* unblock_thread;
@@ -55,10 +59,12 @@ namespace Kernel
 		static Processor& initialize();
 
 		static ProcessorID current_id() { return read_gs_sized<ProcessorID>(offsetof(Processor, m_id)); }
+		static uint8_t current_index()  { return read_gs_sized<uint8_t>(offsetof(Processor, m_index)); }
 		static ProcessorID id_from_index(size_t index);
 
-		static uint8_t count()       { return s_processor_count; }
+		static uint8_t count()        { return s_processor_count; }
-		static bool is_smp_enabled() { return s_is_smp_enabled; }
+		static bool is_smp_enabled()  { return s_is_smp_enabled; }
+		static void set_smp_enabled() { s_is_smp_enabled = true; }
 		static void wait_until_processors_ready();
 
 		static void toggle_should_print_cpu_load() { s_should_print_cpu_load = !s_should_print_cpu_load; }
@@ -77,8 +83,11 @@ namespace Kernel
 
 		static InterruptState get_interrupt_state()
 		{
-			uintptr_t flags;
+#if ARCH(x86_64)
-			asm volatile("pushf; pop %0" : "=rm"(flags));
+			const auto flags = __builtin_ia32_readeflags_u64();
+#elif ARCH(i686)
+			const auto flags = __builtin_ia32_readeflags_u32();
+#endif
 			if (flags & (1 << 9))
 				return InterruptState::Enabled;
 			return InterruptState::Disabled;
@@ -97,6 +106,8 @@ namespace Kernel
 		uintptr_t stack_bottom() const	{ return reinterpret_cast<uintptr_t>(m_stack); }
 		uintptr_t stack_top() const		{ return stack_bottom() + s_stack_size; }
 
+		static void set_thread_syscall_stack(vaddr_t vaddr) { write_gs_sized<vaddr_t>(offsetof(Processor, m_thread_syscall_stack), vaddr); }
+
 		static GDT& gdt() { return *read_gs_sized<GDT*>(offsetof(Processor, m_gdt)); }
 		static IDT& idt() { return *read_gs_sized<IDT*>(offsetof(Processor, m_idt)); }
 
@@ -106,6 +117,16 @@ namespace Kernel
 		static void yield();
 		static Scheduler& scheduler() { return *read_gs_sized<Scheduler*>(offsetof(Processor, m_scheduler)); }
 
+		static void initialize_tsc(uint8_t shift, uint64_t mult, uint64_t realtime_seconds);
+		static void update_tsc();
+		static uint64_t ns_since_boot_tsc();
+
+		static Thread* get_current_sse_thread() { return read_gs_sized<Thread*>(offsetof(Processor, m_sse_thread)); };
+		static void set_current_sse_thread(Thread* thread) { write_gs_sized<Thread*>(offsetof(Processor, m_sse_thread), thread); };
+
+		static paddr_t shared_page_paddr() { return s_shared_page_paddr; }
+		static volatile API::SharedPage& shared_page() { return *reinterpret_cast<API::SharedPage*>(s_shared_page_vaddr); }
+
 		static void handle_ipi();
 
 		static void handle_smp_messages();
@@ -116,6 +137,21 @@ namespace Kernel
 		static void load_fsbase();
 		static void load_gsbase();
 
+		static void disable_sse()
+		{
+			uintptr_t dummy;
+#if ARCH(x86_64)
+			asm volatile("movq %%cr0, %0; orq $0x08, %0; movq %0, %%cr0" : "=r"(dummy));
+#elif ARCH(i686)
+			asm volatile("movl %%cr0, %0; orl $0x08, %0; movl %0, %%cr0" : "=r"(dummy));
+#endif
+		}
+
+		static void enable_sse()
+		{
+			asm volatile("clts");
+		}
+
 	private:
 		Processor() = default;
 		~Processor() { ASSERT_NOT_REACHED(); }
@@ -123,6 +159,14 @@ namespace Kernel
 		static ProcessorID read_processor_id();
 
 		static void initialize_smp();
+		static void initialize_shared_page();
+
+		static void dummy()
+		{
+#if ARCH(x86_64)
+			static_assert(offsetof(Processor, m_thread_syscall_stack) == 8, "This is hardcoded in Syscall.S");
+#endif
+		}
 
 		template<typename T>
 		static T read_gs_sized(uintptr_t offset) requires(sizeof(T) <= 8)
@@ -161,8 +205,15 @@ namespace Kernel
 		static BAN::Atomic<uint8_t> s_processor_count;
 		static BAN::Atomic<bool>    s_is_smp_enabled;
 		static BAN::Atomic<bool>    s_should_print_cpu_load;
+		static paddr_t              s_shared_page_paddr;
+		static vaddr_t              s_shared_page_vaddr;
 
 		ProcessorID m_id { 0 };
+		uint8_t m_index { 0 };
+
+		vaddr_t m_thread_syscall_stack;
+
+		Thread* m_sse_thread { nullptr };
 
 		static constexpr size_t s_stack_size { 4096 };
 		void* m_stack { nullptr };

kernel/include/kernel/Scheduler.h

@@ -57,7 +57,7 @@ namespace Kernel
 		static BAN::ErrorOr<Scheduler*> create();
 		BAN::ErrorOr<void> initialize();
 
-		void reschedule(InterruptStack*, InterruptRegisters*);
+		void reschedule(YieldRegisters*);
 		void reschedule_if_idle();
 
 		void timer_interrupt();
@@ -82,6 +82,8 @@ namespace Kernel
 		void update_most_loaded_node_queue(SchedulerQueue::Node*, SchedulerQueue* target_queue);
 		void remove_node_from_most_loaded(SchedulerQueue::Node*);
 
+		void wake_up_sleeping_threads();
+
 		void do_load_balancing();
 
 		class ProcessorID find_least_loaded_processor() const;
@@ -99,7 +101,7 @@ namespace Kernel
 		InterruptStack* m_interrupt_stack { nullptr };
 		InterruptRegisters* m_interrupt_registers { nullptr };
 
-		uint64_t m_last_reschedule_ns { 0 };
+		uint64_t m_next_reschedule_ns { 0 };
 		uint64_t m_last_load_balance_ns { 0 };
 
 		struct ThreadInfo

kernel/include/kernel/Storage/ATA/AHCI/Device.h

@@ -1,8 +1,9 @@
 #pragma once
 
-#include <kernel/ThreadBlocker.h>
+#include <kernel/Memory/DMARegion.h>
 #include <kernel/Storage/ATA/AHCI/Definitions.h>
 #include <kernel/Storage/ATA/ATADevice.h>
+#include <kernel/ThreadBlocker.h>
 
 namespace Kernel
 {
@@ -34,6 +35,8 @@ namespace Kernel
 		BAN::ErrorOr<void> block_until_command_completed(uint32_t command_slot);
 
 	private:
+		Mutex m_mutex;
+
 		BAN::RefPtr<AHCIController> m_controller;
 		volatile HBAPortMemorySpace* const m_port;
 

kernel/include/kernel/Storage/DiskCache.h

@@ -3,6 +3,7 @@
 #include <BAN/Array.h>
 #include <BAN/ByteSpan.h>
 #include <BAN/Vector.h>
+#include <kernel/Lock/RWLock.h>
 #include <kernel/Memory/Types.h>
 
 namespace Kernel
@@ -28,6 +29,8 @@ namespace Kernel
 	private:
 		BAN::ErrorOr<void> sync_cache_index(size_t index);
 
+		size_t find_sector_cache_index(uint64_t sector) const;
+
 	private:
 		struct PageCache
 		{
@@ -35,9 +38,13 @@ namespace Kernel
 			uint64_t first_sector { 0 };
 			uint8_t sector_mask { 0 };
 			uint8_t dirty_mask { 0 };
+			bool syncing { false };
 		};
 
 	private:
+		RWLock m_rw_lock;
+		Mutex m_sync_mutex;
+
 		const size_t m_sector_size;
 		StorageDevice& m_device;
 		BAN::Vector<PageCache> m_cache;

kernel/include/kernel/Storage/StorageDevice.h

@@ -51,7 +51,6 @@ namespace Kernel
 		virtual bool has_hungup_impl() const override { return false; }
 
 	private:
-		Mutex								m_mutex;
 		BAN::Optional<DiskCache>			m_disk_cache;
 		BAN::Vector<BAN::RefPtr<Partition>>	m_partitions;
 

kernel/include/kernel/Syscall.h

@@ -1,27 +0,0 @@
-#pragma once
-
-#include <kernel/Attributes.h>
-#include <kernel/IDT.h>
-#include <stdint.h>
-#include <sys/syscall.h>
-
-namespace Kernel
-{
-
-	ALWAYS_INLINE long syscall(int syscall, uintptr_t arg1 = 0, uintptr_t arg2 = 0, uintptr_t arg3 = 0, uintptr_t arg4 = 0, uintptr_t arg5 = 0)
-	{
-		long ret;
-		asm volatile("int %[irq]"
-			: "=a"(ret)
-			: [irq]"i"(static_cast<int>(IRQ_SYSCALL)) // WTF GCC 15
-			, "a"(syscall)
-			, "b"((uintptr_t)arg1)
-			, "c"((uintptr_t)arg2)
-			, "d"((uintptr_t)arg3)
-			, "S"((uintptr_t)arg4)
-			, "D"((uintptr_t)arg5)
-			: "memory");
-		return ret;
-	}
-
-}

kernel/include/kernel/Terminal/TTY.h

@@ -3,6 +3,7 @@
 #include <BAN/Array.h>
 #include <kernel/Device/Device.h>
 #include <kernel/Lock/SpinLock.h>
+#include <kernel/Memory/ByteRingBuffer.h>
 #include <kernel/Terminal/TerminalDriver.h>
 #include <kernel/ThreadBlocker.h>
 #include <LibInput/KeyEvent.h>
@@ -102,8 +103,7 @@ namespace Kernel
 
 		struct Buffer
 		{
-			BAN::Array<uint8_t, 1024> buffer;
+			BAN::UniqPtr<ByteRingBuffer> buffer;
-			size_t bytes { 0 };
 			bool flush { false };
 			ThreadBlocker thread_blocker;
 		};

kernel/include/kernel/Thread.h

@@ -16,6 +16,7 @@
 namespace Kernel
 {
 
+	class MemoryBackedRegion;
 	class Process;
 
 	class Thread
@@ -37,8 +38,12 @@ namespace Kernel
 		//        stack overflows on some machines with 8 page stack
 		static constexpr size_t kernel_stack_size    { PAGE_SIZE * 16 };
 
-		// TODO: userspace stack is hard limited to 32 MiB, maybe make this dynamic?
+		// TODO: userspace stack size is hard limited, maybe make this dynamic?
+#if ARCH(x86_64)
 		static constexpr size_t userspace_stack_size { 32 << 20 };
+#elif ARCH(i686)
+		static constexpr size_t userspace_stack_size { 4 << 20 };
+#endif
 
 	public:
 		static BAN::ErrorOr<Thread*> create_kernel(entry_t, void*);
@@ -55,12 +60,10 @@ namespace Kernel
 		// Returns true, if thread is going to trigger signal
 		bool is_interrupted_by_signal(bool skip_stop_and_cont = false) const;
 
-		// Returns true if pending signal can be added to thread
-		bool can_add_signal_to_execute() const;
-		bool will_execute_signal() const;
 		// Returns true if handled signal had SA_RESTART
-		bool handle_signal(int signal = 0, const siginfo_t& signal_info = {});
+		bool handle_signal_if_interrupted();
-		void add_signal(int signal, const siginfo_t& info);
+		bool handle_signal(int signal, const siginfo_t&);
+		void add_signal(int signal, const siginfo_t&);
 		void set_suspend_signal_mask(uint64_t sigmask);
 
 		static bool is_stopping_signal(int signal);
@@ -103,9 +106,7 @@ namespace Kernel
 		vaddr_t kernel_stack_top() const	{ return m_kernel_stack->vaddr() + m_kernel_stack->size(); }
 		VirtualRange& kernel_stack() { return *m_kernel_stack; }
 
-		vaddr_t userspace_stack_bottom() const	{ return is_userspace() ? m_userspace_stack->vaddr() : UINTPTR_MAX; }
+		MemoryBackedRegion& userspace_stack() { ASSERT(is_userspace() && m_userspace_stack); return *m_userspace_stack; }
-		vaddr_t userspace_stack_top() const		{ return is_userspace() ? m_userspace_stack->vaddr() + m_userspace_stack->size() : 0; }
-		VirtualRange& userspace_stack() { ASSERT(is_userspace()); return *m_userspace_stack; }
 
 		static Thread& current();
 		static pid_t current_tid();
@@ -122,16 +123,17 @@ namespace Kernel
 		void set_cpu_time_start();
 		void set_cpu_time_stop();
 
+		void update_processor_index_address();
+
 		void set_fsbase(vaddr_t base) { m_fsbase = base; }
 		vaddr_t get_fsbase() const { return m_fsbase; }
 		void set_gsbase(vaddr_t base) { m_gsbase = base; }
 		vaddr_t get_gsbase() const { return m_gsbase; }
 
-		size_t virtual_page_count() const { return (m_kernel_stack ? (m_kernel_stack->size() / PAGE_SIZE) : 0) + (m_userspace_stack ? (m_userspace_stack->size() / PAGE_SIZE) : 0); }
+		size_t virtual_page_count() const { return m_kernel_stack ? (m_kernel_stack->size() / PAGE_SIZE) : 0; }
 		size_t physical_page_count() const { return virtual_page_count(); }
 
-		InterruptStack& interrupt_stack() { return m_interrupt_stack; }
+		YieldRegisters& yield_registers() { return m_yield_registers; }
-		InterruptRegisters& interrupt_registers() { return m_interrupt_registers; }
 
 		void save_sse();
 		void load_sse();
@@ -153,26 +155,37 @@ namespace Kernel
 
 		bool currently_on_alternate_stack() const;
 
+		struct signal_handle_info_t
+		{
+			vaddr_t handler;
+			vaddr_t stack_top;
+			uint64_t restore_sigmask;
+			bool has_sa_restart;
+		};
+		signal_handle_info_t remove_signal_and_get_info(int signal);
+		void handle_signal_impl(int signal, const siginfo_t&, const signal_handle_info_t&);
+
 	private:
 		// NOTE: this is the first member to force it being last destructed
 		//       {kernel,userspace}_stack has to be destroyed before page table
 		BAN::UniqPtr<PageTable>    m_keep_alive_page_table;
 
 		BAN::UniqPtr<VirtualRange> m_kernel_stack;
-		BAN::UniqPtr<VirtualRange> m_userspace_stack;
+		MemoryBackedRegion*        m_userspace_stack      { nullptr };
 		const pid_t                m_tid                  { 0 };
 		State                      m_state                { State::NotStarted };
 		Process*                   m_process              { nullptr };
 		bool                       m_is_userspace         { false };
 		bool                       m_delete_process       { false };
 
+		bool                       m_has_custom_fsbase    { false };
 		vaddr_t                    m_fsbase               { 0 };
+		bool                       m_has_custom_gsbase    { false };
 		vaddr_t                    m_gsbase               { 0 };
 
 		SchedulerQueue::Node*      m_scheduler_node       { nullptr };
 
-		InterruptStack             m_interrupt_stack      { };
+		YieldRegisters             m_yield_registers      { };
-		InterruptRegisters         m_interrupt_registers  { };
 
 		siginfo_t                  m_signal_infos[_SIGMAX + 1] { };
 		uint64_t                   m_signal_pending_mask  { 0 };

kernel/include/kernel/Timer/Timer.h

@@ -35,6 +35,8 @@ namespace Kernel
 		static SystemTimer& get();
 		static bool is_initialized();
 
+		void initialize_tsc();
+
 		virtual uint64_t ms_since_boot() const override;
 		virtual uint64_t ns_since_boot() const override;
 		virtual timespec time_since_boot() const override;
@@ -47,6 +49,9 @@ namespace Kernel
 
 		void dont_invoke_scheduler() { m_timer->m_should_invoke_scheduler = false; }
 
+		void update_tsc() const;
+		uint64_t ns_since_boot_no_tsc() const;
+
 		timespec real_time() const;
 
 	private:
@@ -54,10 +59,14 @@ namespace Kernel
 
 		void initialize_timers(bool force_pic);
 
+		uint64_t get_tsc_frequency() const;
+
 	private:
 		uint64_t m_boot_time { 0 };
 		BAN::UniqPtr<RTC> m_rtc;
 		BAN::UniqPtr<Timer> m_timer;
+		bool m_has_invariant_tsc { false };
+		mutable uint32_t m_timer_ticks { 0 };
 	};
 
 }

kernel/include/kernel/USB/Device.h

@@ -75,6 +75,7 @@ namespace Kernel
 
 		BAN::ErrorOr<void> initialize();
 
+		const USBDeviceDescriptor& device_descriptor() const { return m_descriptor.descriptor; }
 		const BAN::Vector<ConfigurationDescriptor>& configurations() { return m_descriptor.configurations; }
 
 		virtual BAN::ErrorOr<uint8_t> initialize_device_on_hub_port(uint8_t port_id, USB::SpeedClass) = 0;

kernel/include/kernel/USB/HID/HIDDriver.h

@@ -55,6 +55,8 @@ namespace Kernel
 		{}
 		virtual ~USBHIDDevice() = default;
 
+		virtual BAN::ErrorOr<void> initialize() { return {}; }
+
 		virtual void start_report() = 0;
 		virtual void stop_report() = 0;
 

kernel/include/kernel/USB/HID/Joystick.h

@@ -0,0 +1,71 @@
+#pragma once
+
+#include <kernel/USB/HID/HIDDriver.h>
+
+#include <LibInput/Joystick.h>
+
+namespace Kernel
+{
+
+	class USBJoystick final : public USBHIDDevice
+	{
+		BAN_NON_COPYABLE(USBJoystick);
+		BAN_NON_MOVABLE(USBJoystick);
+
+		enum class Type
+		{
+			Unknown,
+			DualShock3,
+		};
+
+	public:
+		BAN::ErrorOr<void> initialize() override;
+
+		void start_report() override;
+		void stop_report() override;
+
+		void handle_array(uint16_t usage_page, uint16_t usage) override;
+		void handle_variable(uint16_t usage_page, uint16_t usage, int64_t state) override;
+		void handle_variable_absolute(uint16_t usage_page, uint16_t usage, int64_t state, int64_t min, int64_t max) override;
+
+		void update() override;
+
+	protected:
+		BAN::ErrorOr<size_t> read_impl(off_t, BAN::ByteSpan) override;
+		bool can_read_impl() const override { return true; }
+
+		BAN::ErrorOr<long> ioctl_impl(int request, void* arg) override;
+
+	private:
+		USBJoystick(USBHIDDriver&);
+		~USBJoystick() = default;
+
+	private:
+		void initialize_type();
+
+		BAN::ErrorOr<void> update_dualshock3_state(uint8_t led_bitmap, uint8_t rumble_strength);
+
+	private:
+		USBHIDDriver& m_driver;
+
+		Type m_type { Type::Unknown };
+
+		BAN::UniqPtr<DMARegion> m_send_buffer;
+
+		SpinLock m_state_lock;
+		InterruptState m_interrupt_state;
+		LibInput::JoystickState m_state;
+		size_t m_state_index { 0 };
+
+		BAN::Atomic<bool> m_has_got_report { false };
+
+		Mutex m_command_mutex;
+
+		BAN::Atomic<bool> m_has_initialized_leds { false };
+		uint8_t m_led_state       { 0b0001 };
+		uint8_t m_rumble_strength { 0x00 };
+
+		friend class BAN::RefPtr<USBJoystick>;
+	};
+
+}

kernel/kernel/ACPI/ACPI.cpp

@@ -531,11 +531,8 @@ acpi_release_global_lock:
 			return BAN::Error::from_errno(EFAULT);
 		}
 
-		if (!s5_node.as.package->elements[0].resolved || !s5_node.as.package->elements[1].resolved)
+		TRY(AML::resolve_package_element(s5_node.as.package->elements[0], true));
-		{
+		TRY(AML::resolve_package_element(s5_node.as.package->elements[1], true));
-			dwarnln("TODO: lazy evaluate package \\_S5 elements");
-			return BAN::Error::from_errno(ENOTSUP);
-		}
 
 		auto slp_typa_node = TRY(AML::convert_node(TRY(s5_node.as.package->elements[0].value.node->copy()), AML::ConvInteger, sizeof(uint64_t)));
 		auto slp_typb_node = TRY(AML::convert_node(TRY(s5_node.as.package->elements[1].value.node->copy()), AML::ConvInteger, sizeof(uint64_t)));

kernel/kernel/Audio/AC97/Controller.cpp

@@ -1,5 +1,6 @@
 #include <kernel/Audio/AC97/Controller.h>
 #include <kernel/Audio/AC97/Definitions.h>
+#include <kernel/FS/DevFS/FileSystem.h>
 
 namespace Kernel
 {
@@ -105,18 +106,20 @@ namespace Kernel
 		IOCE  = 1 << 4,
 	};
 
-	BAN::ErrorOr<BAN::RefPtr<AC97AudioController>> AC97AudioController::create(PCI::Device& pci_device)
+	BAN::ErrorOr<void> AC97AudioController::create(PCI::Device& pci_device)
 	{
 		auto* ac97_ptr = new AC97AudioController(pci_device);
 		if (ac97_ptr == nullptr)
 			return BAN::Error::from_errno(ENOMEM);
 		auto ac97 = BAN::RefPtr<AC97AudioController>::adopt(ac97_ptr);
 		TRY(ac97->initialize());
-		return ac97;
+		return {};
 	}
 
 	BAN::ErrorOr<void> AC97AudioController::initialize()
 	{
+		TRY(AudioController::initialize());
+
 		m_pci_device.enable_bus_mastering();
 
 		m_mixer = TRY(m_pci_device.allocate_bar_region(0));
@@ -134,8 +137,27 @@ namespace Kernel
 		// Reset mixer to default values
 		m_mixer->write16(AudioMixerRegister::Reset, 0);
 
-		// Master volume 100%, no mute
+		// Master volumes
-		m_mixer->write16(AudioMixerRegister::MasterVolume, 0x0000);
+		m_mixer->write16(AudioMixerRegister::MasterVolume, 0x2020);
+		if (m_mixer->read16(AudioMixerRegister::MasterVolume) == 0x2020)
+		{
+			m_volume_info = {
+				.min_mdB = -94500,
+				.max_mdB = 0,
+				.step_mdB = 1500,
+				.mdB = 0,
+			};
+		}
+		else
+		{
+			m_volume_info = {
+				.min_mdB = -46500,
+				.max_mdB = 0,
+				.step_mdB = 1500,
+				.mdB = 0,
+			};
+		}
+		m_mixer->write16(AudioMixerRegister::MasterVolume, get_volume_data());
 
 		// PCM output volume left/right +0 db, no mute
 		m_mixer->write16(AudioMixerRegister::PCMOutVolume, 0x0808);
@@ -147,6 +169,8 @@ namespace Kernel
 		// disable transfer, enable all interrupts
 		m_bus_master->write8(BusMasterRegister::PO_CR, IOCE | FEIE | LVBIE);
 
+		DevFileSystem::get().add_device(this);
+
 		return {};
 	}
 
@@ -182,6 +206,19 @@ namespace Kernel
 		return {};
 	}
 
+	uint32_t AC97AudioController::get_volume_data() const
+	{
+		const uint32_t steps = (-m_volume_info.mdB + m_volume_info.step_mdB / 2) / m_volume_info.step_mdB;
+		return (steps << 8) | steps;
+	}
+
+	BAN::ErrorOr<void> AC97AudioController::set_volume_mdB(int32_t mdB)
+	{
+		m_volume_info.mdB = BAN::Math::clamp(mdB, m_volume_info.min_mdB, m_volume_info.max_mdB);
+		m_mixer->write16(AudioMixerRegister::MasterVolume, get_volume_data());
+		return {};
+	}
+
 	void AC97AudioController::handle_new_data()
 	{
 		ASSERT(m_spinlock.current_processor_has_lock());
@@ -200,23 +237,22 @@ namespace Kernel
 			if (next_bld_head == m_bdl_tail)
 				break;
 
-			const size_t sample_data_tail = (m_sample_data_head + m_sample_data_capacity - m_sample_data_size) % m_sample_data_capacity;
+			const size_t sample_frames = BAN::Math::min(m_sample_data->size() / get_channels() / sizeof(uint16_t), m_samples_per_entry / get_channels());
-
+			if (sample_frames == 0)
-			const size_t max_memcpy = BAN::Math::min(m_sample_data_size, m_sample_data_capacity - sample_data_tail);
-			const size_t samples = BAN::Math::min(max_memcpy / 2, m_samples_per_entry);
-			if (samples == 0)
 				break;
 
+			const size_t copy_total_bytes = sample_frames * get_channels() * sizeof(uint16_t);
+
 			auto& entry = reinterpret_cast<AC97::BufferDescriptorListEntry*>(m_bdl_region->vaddr())[m_bdl_head];
-			entry.samples = samples;
+			entry.samples = sample_frames * get_channels();
 			entry.flags = (1 << 15);
 			memcpy(
 				reinterpret_cast<void*>(m_bdl_region->paddr_to_vaddr(entry.address)),
-				&m_sample_data[sample_data_tail],
+				m_sample_data->get_data().data(),
-				samples * 2
+				copy_total_bytes
 			);
 
-			m_sample_data_size -= samples * 2;
+			m_sample_data->pop(copy_total_bytes);
 
 			lvi = m_bdl_head;
 			m_bdl_head = next_bld_head;

kernel/kernel/Audio/Controller.cpp

@@ -1,5 +1,6 @@
 #include <kernel/Audio/AC97/Controller.h>
 #include <kernel/Audio/Controller.h>
+#include <kernel/Audio/HDAudio/Controller.h>
 #include <kernel/Device/DeviceNumbers.h>
 #include <kernel/FS/DevFS/FileSystem.h>
 #include <kernel/Lock/SpinLockAsMutex.h>
@@ -20,23 +21,26 @@ namespace Kernel
 		BAN::Formatter::print([&ptr](char c) { *ptr++ = c; }, "audio{}", minor(m_rdev));
 	}
 
-	BAN::ErrorOr<BAN::RefPtr<AudioController>> AudioController::create(PCI::Device& pci_device)
+	BAN::ErrorOr<void> AudioController::create(PCI::Device& pci_device)
 	{
 		switch (pci_device.subclass())
 		{
 			case 0x01:
 				// We should confirm that the card is actually AC97 but I'm trusting osdev wiki on this one
 				// > you can probably expect that every sound card with subclass 0x01 is sound card compatibile with AC97
-				if (auto ret = AC97AudioController::create(pci_device); !ret.is_error())
+				if (auto ret = AC97AudioController::create(pci_device); ret.is_error())
-				{
-					DevFileSystem::get().add_device(ret.value());
-					return BAN::RefPtr<AudioController>(ret.release_value());
-				}
-				else
 				{
 					dwarnln("Failed to initialize AC97: {}", ret.error());
 					return ret.release_error();
 				}
+				break;
+			case 0x03:
+				if (auto ret = HDAudioController::create(pci_device); ret.is_error())
+				{
+					dwarnln("Failed to initialize Intel HDA: {}", ret.error());
+					return ret.release_error();
+				}
+				break;
 			default:
 				dprintln("Unsupported Sound card (PCI {2H}:{2H}:{2H})",
 					pci_device.class_code(),
@@ -45,37 +49,32 @@ namespace Kernel
 				);
 				return BAN::Error::from_errno(ENOTSUP);
 		}
+
+		return {};
 	}
 
+	BAN::ErrorOr<void> AudioController::initialize()
+	{
+		m_sample_data = TRY(ByteRingBuffer::create(m_sample_data_capacity));
+		return {};
+	}
 
 	BAN::ErrorOr<size_t> AudioController::write_impl(off_t, BAN::ConstByteSpan buffer)
 	{
 		SpinLockGuard lock_guard(m_spinlock);
 
-		while (m_sample_data_size >= m_sample_data_capacity)
+		while (m_sample_data->full())
 		{
 			SpinLockGuardAsMutex smutex(lock_guard);
 			TRY(Thread::current().block_or_eintr_indefinite(m_sample_data_blocker, &smutex));
 		}
 
-		size_t nwritten = 0;
+		const size_t to_copy = BAN::Math::min(buffer.size(), m_sample_data->free());
-		while (nwritten < buffer.size())
+		m_sample_data->push(buffer.slice(0, to_copy));
-		{
-			if (m_sample_data_size >= m_sample_data_capacity)
-				break;
-
-			const size_t max_memcpy = BAN::Math::min(m_sample_data_capacity - m_sample_data_size, m_sample_data_capacity - m_sample_data_head);
-			const size_t to_copy = BAN::Math::min(buffer.size() - nwritten, max_memcpy);
-			memcpy(m_sample_data + m_sample_data_head, buffer.data() + nwritten, to_copy);
-
-			nwritten += to_copy;
-			m_sample_data_head = (m_sample_data_head + to_copy) % m_sample_data_capacity;
-			m_sample_data_size += to_copy;
-		}
 
 		handle_new_data();
 
-		return nwritten;
+		return to_copy;
 	}
 
 	BAN::ErrorOr<long> AudioController::ioctl_impl(int cmd, void* arg)
@@ -92,11 +91,26 @@ namespace Kernel
 			case SND_GET_BUFFERSZ:
 			{
 				SpinLockGuard _(m_spinlock);
-				*static_cast<uint32_t*>(arg) = m_sample_data_size;
+				*static_cast<uint32_t*>(arg) = m_sample_data->size();
 				if (cmd == SND_RESET_BUFFER)
-					m_sample_data_size = 0;
+					m_sample_data->pop(m_sample_data->size());
 				return 0;
 			}
+			case SND_GET_TOTAL_PINS:
+				*static_cast<uint32_t*>(arg) = get_total_pins();
+				return 0;
+			case SND_GET_PIN:
+				*static_cast<uint32_t*>(arg) = get_current_pin();
+				return 0;
+			case SND_SET_PIN:
+				TRY(set_current_pin(*static_cast<uint32_t*>(arg)));
+				return 0;
+			case SND_GET_VOLUME_INFO:
+				*static_cast<snd_volume_info*>(arg) = m_volume_info;
+				return 0;
+			case SND_SET_VOLUME_MDB:
+				TRY(set_volume_mdB(*static_cast<int32_t*>(arg)));
+				return 0;
 		}
 
 		return CharacterDevice::ioctl_impl(cmd, arg);

kernel/kernel/Audio/HDAudio/AudioFunctionGroup.cpp

@@ -0,0 +1,630 @@
+#include <kernel/Audio/HDAudio/AudioFunctionGroup.h>
+#include <kernel/Audio/HDAudio/Registers.h>
+#include <kernel/FS/DevFS/FileSystem.h>
+
+#include <BAN/Sort.h>
+
+namespace Kernel
+{
+
+	BAN::ErrorOr<BAN::RefPtr<HDAudioFunctionGroup>> HDAudioFunctionGroup::create(BAN::RefPtr<HDAudioController> controller, uint8_t cid, HDAudio::AFGNode&& afg_node)
+	{
+		auto* audio_group_ptr = new HDAudioFunctionGroup(controller, cid, BAN::move(afg_node));
+		if (audio_group_ptr == nullptr)
+			return BAN::Error::from_errno(ENOMEM);
+		auto audio_group = BAN::RefPtr<HDAudioFunctionGroup>::adopt(audio_group_ptr);
+		TRY(audio_group->initialize());
+		return audio_group;
+	}
+
+	HDAudioFunctionGroup::~HDAudioFunctionGroup()
+	{
+		if (m_stream_id != 0xFF)
+			m_controller->deallocate_stream_id(m_stream_id);
+		if (m_stream_index != 0xFF)
+			m_controller->deallocate_stream(m_stream_index);
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::initialize()
+	{
+		TRY(AudioController::initialize());
+
+		if constexpr(DEBUG_HDAUDIO)
+		{
+			const auto widget_to_string =
+				[](HDAudio::AFGWidget::Type type) -> const char*
+				{
+					using HDAudio::AFGWidget;
+					switch (type)
+					{
+						case AFGWidget::Type::OutputConverter: return "DAC";
+						case AFGWidget::Type::InputConverter:  return "ADC";
+						case AFGWidget::Type::Mixer:           return "Mixer";
+						case AFGWidget::Type::Selector:        return "Selector";
+						case AFGWidget::Type::PinComplex:      return "Pin";
+						case AFGWidget::Type::Power:           return "Power";
+						case AFGWidget::Type::VolumeKnob:      return "VolumeKnob";
+						case AFGWidget::Type::BeepGenerator:   return "BeepGenerator";
+					}
+					ASSERT_NOT_REACHED();
+				};
+
+			dprintln("AFG {}", m_afg_node.id);
+			for (auto widget : m_afg_node.widgets)
+			{
+				if (widget.type == HDAudio::AFGWidget::Type::PinComplex)
+				{
+					dprintln("  widget {}: {} ({}, {}, {}), {32b}",
+						widget.id,
+						widget_to_string(widget.type),
+						(int)widget.pin_complex.output,
+						(int)widget.pin_complex.input,
+						(int)widget.pin_complex.display,
+						widget.pin_complex.config
+					);
+				}
+				else
+				{
+					dprintln("  widget {}: {}",
+						widget.id,
+						widget_to_string(widget.type)
+					);
+				}
+
+				if (!widget.connections.empty())
+					dprintln("    connections {}", widget.connections);
+			}
+		}
+
+		TRY(initialize_stream());
+
+		if (auto ret = initialize_output(); ret.is_error())
+		{
+			// No usable pins, not really an error
+			if (ret.error().get_error_code() == ENODEV)
+				return {};
+			return ret.release_error();
+		}
+
+		DevFileSystem::get().add_device(this);
+
+		return {};
+	}
+
+	uint32_t HDAudioFunctionGroup::get_total_pins() const
+	{
+		return m_output_pins.size();
+	}
+
+	uint32_t HDAudioFunctionGroup::get_current_pin() const
+	{
+		const auto current_id = m_output_paths[m_output_path_index].front()->id;
+		for (size_t i = 0; i < m_output_pins.size(); i++)
+			if (m_output_pins[i]->id == current_id)
+				return i;
+		ASSERT_NOT_REACHED();
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::set_current_pin(uint32_t pin)
+	{
+		if (pin >= m_output_pins.size())
+			return BAN::Error::from_errno(EINVAL);
+
+		if (auto ret = disable_output_path(m_output_path_index); ret.is_error())
+			dwarnln("failed to disable old output path {}", ret.error());
+
+		const uint32_t pin_id = m_output_pins[pin]->id;
+		for (size_t i = 0; i < m_output_paths.size(); i++)
+		{
+			if (m_output_paths[i].front()->id != pin_id)
+				continue;
+
+			if (auto ret = enable_output_path(i); ret.is_error())
+			{
+				if (ret.error().get_error_code() == ENOTSUP)
+					continue;
+				dwarnln("path {} not supported", i);
+				return ret.release_error();
+			}
+
+			m_output_path_index = i;
+			return {};
+		}
+
+		dwarnln("failed to set output widget to {}", pin_id);
+
+		return BAN::Error::from_errno(ENOTSUP);
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::set_volume_mdB(int32_t mdB)
+	{
+		mdB = BAN::Math::clamp(mdB, m_volume_info.min_mdB, m_volume_info.max_mdB);
+
+		const auto& path = m_output_paths[m_output_path_index];
+		for (size_t i = 0; i < path.size(); i++)
+		{
+			if (!path[i]->output_amplifier.has_value())
+				continue;
+
+			const int32_t step_round = (mdB >= 0)
+				? +m_volume_info.step_mdB / 2
+				: -m_volume_info.step_mdB / 2;
+			const uint32_t step = (mdB + step_round) / m_volume_info.step_mdB + path[i]->output_amplifier->offset;
+			const uint32_t volume = 0b1'0'1'1'0000'0'0000000 | step;
+
+			TRY(m_controller->send_command({
+				.data = static_cast<uint8_t>(volume & 0xFF),
+				.command = static_cast<uint16_t>(0x300 | (volume >> 8)),
+				.node_index = path[i]->id,
+				.codec_address = m_cid,
+			}));
+
+			break;
+		}
+
+		m_volume_info.mdB = mdB;
+
+		return {};
+	}
+
+	size_t HDAudioFunctionGroup::bdl_offset() const
+	{
+		const size_t bdl_entry_bytes = m_bdl_entry_sample_frames * get_channels() * sizeof(uint16_t);
+		const size_t bdl_total_size = bdl_entry_bytes * m_bdl_entry_count;
+		if (auto rem = bdl_total_size % 128)
+			return bdl_total_size + (128 - rem);
+		return bdl_total_size;
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::initialize_stream()
+	{
+		const size_t bdl_entry_bytes = m_bdl_entry_sample_frames * get_channels() * sizeof(uint16_t);
+		const size_t bdl_list_size   = m_bdl_entry_count * sizeof(HDAudio::BDLEntry);
+
+		m_bdl_region = TRY(DMARegion::create(bdl_offset() + bdl_list_size));
+		if (!m_controller->is_64bit() && (m_bdl_region->paddr() >> 32))
+		{
+			dwarnln("no 64 bit support but allocated bdl has 64 bit address :(");
+			return BAN::Error::from_errno(ENOTSUP);
+		}
+
+		auto* bdl = reinterpret_cast<volatile HDAudio::BDLEntry*>(m_bdl_region->vaddr() + bdl_offset());
+		for (size_t i = 0; i < m_bdl_entry_count; i++)
+		{
+			bdl[i].address = m_bdl_region->paddr() + i * bdl_entry_bytes;
+			bdl[i].length = bdl_entry_bytes;
+			bdl[i].ioc = 1;
+		}
+
+		ASSERT(m_stream_id == 0xFF);
+		m_stream_id = TRY(m_controller->allocate_stream_id());
+
+		ASSERT(m_stream_index == 0xFF);
+		m_stream_index = TRY(m_controller->allocate_stream(HDAudio::StreamType::Output, this));
+
+		reset_stream();
+
+		return {};
+	}
+
+	void HDAudioFunctionGroup::reset_stream()
+	{
+		using Regs = HDAudio::Regs;
+
+		auto& bar = m_controller->bar0();
+		const auto base = 0x80 + m_stream_index * 0x20;
+
+		const size_t bdl_entry_bytes = m_bdl_entry_sample_frames * get_channels() * sizeof(uint16_t);
+
+		// stop stream
+		bar.write8(base + Regs::SDCTL, bar.read8(base + Regs::SDCTL) & 0xFD);
+
+		// reset stream
+		bar.write8(base + Regs::SDCTL, (bar.read8(base + Regs::SDCTL) & 0xFE) | 1);
+		while (!(bar.read8(base + Regs::SDCTL) & 1))
+			Processor::pause();
+		bar.write8(base + Regs::SDCTL, (bar.read8(base + Regs::SDCTL) & 0xFE));
+		while ((bar.read8(base + Regs::SDCTL) & 1))
+			Processor::pause();
+
+		// set bdl address, total size and lvi
+		const paddr_t bdl_paddr = m_bdl_region->paddr() + bdl_offset();
+		bar.write32(base + Regs::SDBDPL, bdl_paddr);
+		if (m_controller->is_64bit())
+			bar.write32(base + Regs::SDBDPU, bdl_paddr >> 32);
+		bar.write32(base + Regs::SDCBL, bdl_entry_bytes * m_bdl_entry_count);
+		bar.write16(base + Regs::SDLVI, (bar.read16(base + Regs::SDLVI) & 0xFF00) | (m_bdl_entry_count - 1));
+
+		// set stream format
+		bar.write16(base + Regs::SDFMT, get_format_data());
+
+		// set stream id, not bidirectional
+		bar.write8(base + Regs::SDCTL + 2, (bar.read8(base + Regs::SDCTL + 2) & 0x07) | (m_stream_id << 4));
+
+		m_bdl_head = 0;
+		m_bdl_tail = 0;
+		m_stream_running = false;
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::initialize_output()
+	{
+		for (const auto& widget : m_afg_node.widgets)
+		{
+			if (widget.type != HDAudio::AFGWidget::Type::PinComplex || !widget.pin_complex.output)
+				continue;
+
+			// no physical connection
+			if ((widget.pin_complex.config >> 30) == 0b01)
+				continue;
+
+			// needs a GPU
+			if (widget.pin_complex.display)
+				continue;
+
+			BAN::Vector<const HDAudio::AFGWidget*> path;
+			TRY(path.push_back(&widget));
+			TRY(recurse_output_paths(widget, path));
+
+			if (!m_output_paths.empty() && m_output_paths.back().front()->id == widget.id)
+				TRY(m_output_pins.push_back(&widget));
+		}
+
+		if (m_output_pins.empty())
+			return BAN::Error::from_errno(ENODEV);
+
+		// prefer short paths
+		BAN::sort::sort(m_output_paths.begin(), m_output_paths.end(),
+			[](const auto& a, const auto& b) {
+				if (a.front()->id != b.front()->id)
+					return a.front()->id < b.front()->id;
+				return a.size() < b.size();
+			}
+		);
+
+		dprintln_if(DEBUG_HDAUDIO, "found {} paths from output to DAC", m_output_paths.size());
+
+		// select first supported path
+		// FIXME: prefer associations
+		// FIXME: does this pin even have a device?
+		auto result = BAN::Error::from_errno(ENODEV);
+		for (size_t i = 0; i < m_output_paths.size(); i++)
+		{
+			if (auto ret = enable_output_path(i); ret.is_error())
+			{
+				if (ret.error().get_error_code() != ENOTSUP)
+					return ret.release_error();
+				dwarnln("path {} not supported", i);
+				result = BAN::Error::from_errno(ENOTSUP);
+				continue;
+			}
+
+			m_output_path_index = i;
+			break;
+		}
+
+		if (m_output_path_index >= m_output_paths.size())
+		{
+			dwarnln("could not find any usable output path");
+			return result;
+		}
+
+		dprintln_if(DEBUG_HDAUDIO, "routed output path");
+		for (const auto* widget : m_output_paths[m_output_path_index])
+			dprintln_if(DEBUG_HDAUDIO, "  {}", widget->id);
+
+		return {};
+	}
+
+	uint16_t HDAudioFunctionGroup::get_format_data() const
+	{
+		// TODO: don't hardcode this
+		// format: PCM, 48 kHz, 16 bit, 2 channels
+		return 0b0'0'000'000'0'001'0001;
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::enable_output_path(uint8_t index)
+	{
+		ASSERT(index < m_output_paths.size());
+		const auto& path = m_output_paths[index];
+
+		for (const auto* widget : path)
+		{
+			switch (widget->type)
+			{
+				using HDAudio::AFGWidget;
+				case AFGWidget::Type::OutputConverter:
+				case AFGWidget::Type::PinComplex:
+					break;
+				default:
+					dwarnln("FIXME: support enabling widget type {}", static_cast<int>(widget->type));
+					return BAN::Error::from_errno(ENOTSUP);
+			}
+		}
+
+		const auto format = get_format_data();
+
+		for (size_t i = 0; i < path.size(); i++)
+		{
+			// set power state D0
+			TRY(m_controller->send_command({
+				.data = 0x00,
+				.command = 0x705,
+				.node_index = path[i]->id,
+				.codec_address = m_cid,
+			}));
+
+			// set connection index
+			if (i + 1 < path.size() && path[i]->connections.size() > 1)
+			{
+				uint8_t index = 0;
+				for (; index < path[i]->connections.size(); index++)
+					if (path[i]->connections[index] == path[i + 1]->id)
+						break;
+				ASSERT(index < path[i]->connections.size());
+
+				TRY(m_controller->send_command({
+					.data = index,
+					.command = 0x701,
+					.node_index = path[i]->id,
+					.codec_address = m_cid,
+				}));
+			}
+
+			// set volume to 0 dB, no mute
+			if (path[i]->output_amplifier.has_value())
+			{
+				const uint32_t volume = 0b1'0'1'1'0000'0'0000000 | path[i]->output_amplifier->offset;
+				TRY(m_controller->send_command({
+					.data = static_cast<uint8_t>(volume & 0xFF),
+					.command = static_cast<uint16_t>(0x300 | (volume >> 8)),
+					.node_index = path[i]->id,
+					.codec_address = m_cid,
+				}));
+			}
+
+			switch (path[i]->type)
+			{
+				using HDAudio::AFGWidget;
+
+				case AFGWidget::Type::OutputConverter:
+					// set stream and channel 0
+					TRY(m_controller->send_command({
+						.data = static_cast<uint8_t>(m_stream_id << 4),
+						.command = 0x706,
+						.node_index = path[i]->id,
+						.codec_address = m_cid,
+					}));
+					// set format
+					TRY(m_controller->send_command({
+						.data = static_cast<uint8_t>(format & 0xFF),
+						.command = static_cast<uint16_t>(0x200 | (format >> 8)),
+						.node_index = path[i]->id,
+						.codec_address = m_cid,
+					}));
+					break;
+
+				case AFGWidget::Type::PinComplex:
+					// enable output and H-Phn
+					TRY(m_controller->send_command({
+						.data = 0x80 | 0x40,
+						.command = 0x707,
+						.node_index = path[i]->id,
+						.codec_address = m_cid,
+					}));
+					// enable EAPD
+					TRY(m_controller->send_command({
+						.data = 0x02,
+						.command = 0x70C,
+						.node_index = path[i]->id,
+						.codec_address = m_cid,
+					}));
+					break;
+
+				default:
+					ASSERT_NOT_REACHED();
+			}
+		}
+
+		// update volume info to this path
+		m_volume_info.min_mdB = 0;
+		m_volume_info.max_mdB = 0;
+		m_volume_info.step_mdB = 0;
+		for (size_t i = 0; i < path.size(); i++)
+		{
+			if (!path[i]->output_amplifier.has_value())
+				continue;
+			const auto& amp = path[i]->output_amplifier.value();
+
+			const int32_t step_mdB = amp.step_size * 250;
+			m_volume_info.step_mdB = step_mdB;
+			m_volume_info.min_mdB = -amp.offset * step_mdB;
+			m_volume_info.max_mdB = (amp.num_steps - amp.offset) * step_mdB;
+			m_volume_info.mdB = BAN::Math::clamp(m_volume_info.mdB, m_volume_info.min_mdB, m_volume_info.max_mdB);
+
+			const int32_t step_round = (m_volume_info.mdB >= 0)
+				? +step_mdB / 2
+				: -step_mdB / 2;
+			const uint32_t step = (m_volume_info.mdB + step_round) / step_mdB + amp.offset;
+			const uint32_t volume = 0b1'0'1'1'0000'0'0000000 | step;
+
+			TRY(m_controller->send_command({
+				.data = static_cast<uint8_t>(volume & 0xFF),
+				.command = static_cast<uint16_t>(0x300 | (volume >> 8)),
+				.node_index = path[i]->id,
+				.codec_address = m_cid,
+			}));
+
+			break;
+		}
+
+		if (m_volume_info.min_mdB == 0 && m_volume_info.max_mdB == 0)
+			m_volume_info.mdB = 0;
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::disable_output_path(uint8_t index)
+	{
+		ASSERT(index < m_output_paths.size());
+		const auto& path = m_output_paths[index];
+
+		for (size_t i = 0; i < path.size(); i++)
+		{
+			// set power state D3
+			TRY(m_controller->send_command({
+				.data = 0x03,
+				.command = 0x705,
+				.node_index = path[i]->id,
+				.codec_address = m_cid,
+			}));
+
+			switch (path[i]->type)
+			{
+				using HDAudio::AFGWidget;
+
+				case AFGWidget::Type::OutputConverter:
+					break;
+
+				case AFGWidget::Type::PinComplex:
+					// disable output and H-Phn
+					TRY(m_controller->send_command({
+						.data = 0x00,
+						.command = 0x707,
+						.node_index = path[i]->id,
+						.codec_address = m_cid,
+					}));
+					// disable EAPD
+					TRY(m_controller->send_command({
+						.data = 0x00,
+						.command = 0x70C,
+						.node_index = path[i]->id,
+						.codec_address = m_cid,
+					}));
+					break;
+
+				default:
+					ASSERT_NOT_REACHED();
+			}
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> HDAudioFunctionGroup::recurse_output_paths(const HDAudio::AFGWidget& widget, BAN::Vector<const HDAudio::AFGWidget*>& path)
+	{
+		// we've reached a DAC
+		if (widget.type == HDAudio::AFGWidget::Type::OutputConverter)
+		{
+			BAN::Vector<const HDAudio::AFGWidget*> path_copy;
+			TRY(path_copy.resize(path.size()));
+			for (size_t i = 0; i < path.size(); i++)
+				path_copy[i] = path[i];
+			TRY(m_output_paths.push_back(BAN::move(path_copy)));
+			return {};
+		}
+
+		// check all connections
+		for (const auto& connection : m_afg_node.widgets)
+		{
+			if (!widget.connections.contains(connection.id))
+				continue;
+
+			// cycle detection
+			for (const auto* w : path)
+				if (w == &connection)
+					goto already_visited;
+
+			TRY(path.push_back(&connection));
+			TRY(recurse_output_paths(connection, path));
+			path.pop_back();
+
+		already_visited:
+			continue;
+		}
+
+		return {};
+	}
+
+	void HDAudioFunctionGroup::handle_new_data()
+	{
+		queue_bdl_data();
+	}
+
+	void HDAudioFunctionGroup::queue_bdl_data()
+	{
+		ASSERT(m_spinlock.current_processor_has_lock());
+
+		const size_t bdl_entry_bytes = m_bdl_entry_sample_frames * get_channels() * sizeof(uint16_t);
+
+		while ((m_bdl_head + 1) % m_bdl_entry_count != m_bdl_tail)
+		{
+			const size_t sample_frames = BAN::Math::min(m_sample_data->size() / get_channels() / sizeof(uint16_t), m_bdl_entry_sample_frames);
+			if (sample_frames == 0)
+				break;
+
+			const size_t copy_total_bytes = sample_frames * get_channels() * sizeof(uint16_t);
+
+			memcpy(
+				reinterpret_cast<void*>(m_bdl_region->vaddr() + m_bdl_head * bdl_entry_bytes),
+				m_sample_data->get_data().data(),
+				copy_total_bytes
+			);
+
+			if (copy_total_bytes < bdl_entry_bytes)
+			{
+				memset(
+					reinterpret_cast<void*>(m_bdl_region->vaddr() + m_bdl_head * bdl_entry_bytes + copy_total_bytes),
+					0x00,
+					bdl_entry_bytes - copy_total_bytes
+				);
+			}
+
+			m_sample_data->pop(copy_total_bytes);
+			m_bdl_head = (m_bdl_head + 1) % m_bdl_entry_count;
+		}
+
+		if (m_bdl_head == m_bdl_tail || m_stream_running)
+			return;
+
+		// start the stream and enable IOC and descriptor error interrupts
+		auto& bar = m_controller->bar0();
+		const auto base = 0x80 + m_stream_index * 0x20;
+		bar.write8(base + HDAudio::Regs::SDCTL, bar.read8(base + HDAudio::Regs::SDCTL) | 0x16);
+
+		m_stream_running = true;
+	}
+
+	void HDAudioFunctionGroup::on_stream_interrupt(uint8_t stream_index)
+	{
+		using Regs = HDAudio::Regs;
+
+		ASSERT(stream_index == m_stream_index);
+
+		auto& bar = m_controller->bar0();
+		const uint16_t base = 0x80 + stream_index * 0x20;
+
+		const uint8_t sts = bar.read8(base + Regs::SDSTS);
+		bar.write8(base + Regs::SDSTS, sts & 0x3C);
+
+		if (sts & (1 << 4))
+			derrorln("descriptor error");
+
+		// ignore fifo errors as they are too common on real hw :D
+		//if (sts & (1 << 3))
+		//	derrorln("fifo error");
+
+		if (sts & (1 << 2))
+		{
+			SpinLockGuard _(m_spinlock);
+
+			ASSERT(m_stream_running);
+
+			m_bdl_tail = (m_bdl_tail + 1) % m_bdl_entry_count;
+			if (m_bdl_tail == m_bdl_head)
+				reset_stream();
+
+			queue_bdl_data();
+		}
+	}
+
+}

kernel/kernel/Audio/HDAudio/Controller.cpp

@@ -0,0 +1,514 @@
+#include <kernel/Audio/HDAudio/AudioFunctionGroup.h>
+#include <kernel/Audio/HDAudio/Controller.h>
+#include <kernel/Audio/HDAudio/Registers.h>
+#include <kernel/Lock/LockGuard.h>
+#include <kernel/Lock/SpinLockAsMutex.h>
+#include <kernel/MMIO.h>
+#include <kernel/Timer/Timer.h>
+
+namespace Kernel
+{
+
+	BAN::ErrorOr<void> HDAudioController::create(PCI::Device& pci_device)
+	{
+		auto intel_hda_ptr = new HDAudioController(pci_device);
+		if (intel_hda_ptr == nullptr)
+			return BAN::Error::from_errno(ENOMEM);
+		auto intel_hda = BAN::RefPtr<HDAudioController>::adopt(intel_hda_ptr);
+		TRY(intel_hda->initialize());
+		return {};
+	}
+
+	BAN::ErrorOr<void> HDAudioController::initialize()
+	{
+		using Regs = HDAudio::Regs;
+
+		m_pci_device.enable_bus_mastering();
+
+		m_bar0 = TRY(m_pci_device.allocate_bar_region(0));
+
+		dprintln("HD audio");
+		dprintln("  revision {}.{}",
+			m_bar0->read8(Regs::VMAJ),
+			m_bar0->read8(Regs::VMIN)
+		);
+
+		const uint16_t global_cap = m_bar0->read16(Regs::GCAP);
+		m_output_streams = (global_cap >> 12) & 0x0F;
+		m_input_streams  = (global_cap >>  8) & 0x0F;
+		m_bidir_streams  = (global_cap >>  3) & 0x1F;
+		m_is64bit        = (global_cap & 1);
+
+		if (m_output_streams + m_input_streams + m_bidir_streams > 30)
+		{
+			dwarnln("HD audio controller has {} streams, 30 is the maximum valid count",
+				m_output_streams + m_input_streams + m_bidir_streams
+			);
+			return BAN::Error::from_errno(EINVAL);
+		}
+
+		dprintln("  output streams: {}", m_output_streams);
+		dprintln("  input streams:  {}", m_input_streams);
+		dprintln("  bidir streams:  {}", m_bidir_streams);
+		dprintln("  64 bit support: {}", m_is64bit);
+
+		TRY(reset_controller());
+
+		if (auto ret = initialize_ring_buffers(); ret.is_error())
+		{
+			if (ret.error().get_error_code() != ETIMEDOUT)
+				return ret.release_error();
+			m_use_immediate_command = true;
+		}
+
+		TRY(m_pci_device.reserve_interrupts(1));
+		m_pci_device.enable_interrupt(0, *this);
+		m_bar0->write32(Regs::INTCTL, UINT32_MAX);
+
+		const uint16_t state_sts = m_bar0->read16(Regs::STATESTS);
+		for (uint8_t codec_id = 0; codec_id < 15; codec_id++)
+		{
+			if (!(state_sts & (1 << codec_id)))
+				continue;
+
+			auto codec_or_error = initialize_codec(codec_id);
+			if (codec_or_error.is_error())
+				continue;
+
+			auto codec = codec_or_error.release_value();
+			for (auto& node : codec.nodes)
+				if (auto ret = HDAudioFunctionGroup::create(this, codec.id, BAN::move(node)); ret.is_error())
+					dwarnln("Failed to initialize AFG: {}", ret.error());
+		}
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> HDAudioController::reset_controller()
+	{
+		using HDAudio::Regs;
+
+		const auto timeout_ms = SystemTimer::get().ms_since_boot() + 100;
+
+		// transition into reset state
+		if (const uint32_t gcap = m_bar0->read32(Regs::GCTL); gcap & 1)
+		{
+			m_bar0->write32(Regs::GCTL, gcap & 0xFFFFFEFC);
+			while (m_bar0->read32(Regs::GCTL) & 1)
+			{
+				if (SystemTimer::get().ms_since_boot() > timeout_ms)
+					return BAN::Error::from_errno(ETIMEDOUT);
+				Processor::pause();
+			}
+		}
+
+		m_bar0->write32(Regs::GCTL, (m_bar0->read32(Regs::GCTL) & 0xFFFFFEFC) | 1);
+		while (!(m_bar0->read32(Regs::GCTL) & 1))
+		{
+			if (SystemTimer::get().ms_since_boot() > timeout_ms)
+				return BAN::Error::from_errno(ETIMEDOUT);
+			Processor::pause();
+		}
+
+		// 4.3 The software must wait at least 521 us (25 frames) after reading CRST as a 1
+		// before assuming that codecs have all made status change requests and have been
+		// registered by the controller
+		SystemTimer::get().sleep_ms(1);
+
+		return {};
+	}
+
+	BAN::ErrorOr<void> HDAudioController::initialize_ring_buffers()
+	{
+		using Regs = HDAudio::Regs;
+
+		// CORB: at most 1024 bytes (256 * uint32_t)
+		// RIRB: at most 2048 bytes (256 * uint32_t * 2)
+		m_ring_buffer_region = TRY(DMARegion::create(3 * 256 * sizeof(uint32_t)));
+
+		struct SizeInfo
+		{
+			uint16_t size;
+			uint8_t value;
+		};
+
+		const auto get_size_info =
+			[](uint8_t byte) -> BAN::ErrorOr<SizeInfo>
+			{
+				if (byte & 0x40)
+					return SizeInfo { 256, 2 };
+				if (byte & 0x20)
+					return SizeInfo {  16, 1 };
+				if (byte & 0x10)
+					return SizeInfo {   2, 0 };
+				return BAN::Error::from_errno(EINVAL);
+			};
+
+		const auto corb_size = TRY(get_size_info(m_bar0->read8(Regs::CORBSIZE)));
+		const auto rirb_size = TRY(get_size_info(m_bar0->read8(Regs::RIRBSIZE)));
+
+		m_corb = {
+			.vaddr = m_ring_buffer_region->vaddr(),
+			.index = 1,
+			.size = corb_size.size,
+		};
+
+		m_rirb = {
+			.vaddr = m_ring_buffer_region->vaddr() + 1024,
+			.index = 1,
+			.size = rirb_size.size,
+		};
+
+		const paddr_t corb_paddr = m_ring_buffer_region->paddr();
+		const paddr_t rirb_paddr = m_ring_buffer_region->paddr() + 1024;
+		if (!m_is64bit && ((corb_paddr >> 32) || (rirb_paddr >> 32)))
+		{
+			dwarnln("no 64 bit support but allocated ring buffers have 64 bit addresses :(");
+			return BAN::Error::from_errno(ENOTSUP);
+		}
+
+		// disable corb and rirb
+		m_bar0->write8(Regs::CORBCTL,    (m_bar0->read8(Regs::CORBCTL) & 0xFC));
+		m_bar0->write8(Regs::RIRBCTL,    (m_bar0->read8(Regs::RIRBCTL) & 0xF8));
+
+		// set base address
+		m_bar0->write32(Regs::CORBLBASE,  corb_paddr | (m_bar0->read32(Regs::CORBLBASE) & 0x0000007F));
+		if (m_is64bit)
+			m_bar0->write32(Regs::CORBUBASE,  corb_paddr >> 32);
+		// set number of entries
+		m_bar0->write8(Regs::CORBSIZE,   (m_bar0->read8(Regs::CORBSIZE) & 0xFC) | corb_size.value);
+		// zero write pointer
+		m_bar0->write16(Regs::CORBWP,    (m_bar0->read16(Regs::CORBWP) & 0xFF00));
+		// reset read pointer
+		const uint64_t corb_timeout_ms = SystemTimer::get().ms_since_boot() + 100;
+		m_bar0->write16(Regs::CORBRP,    (m_bar0->read16(Regs::CORBRP) & 0x7FFF) | 0x8000);
+		while (!(m_bar0->read16(Regs::CORBRP) & 0x8000))
+		{
+			if (SystemTimer::get().ms_since_boot() > corb_timeout_ms)
+				return BAN::Error::from_errno(ETIMEDOUT);
+			Processor::pause();
+		}
+		m_bar0->write16(Regs::CORBRP,    (m_bar0->read16(Regs::CORBRP) & 0x7FFF));
+		while ((m_bar0->read16(Regs::CORBRP) & 0x8000))
+		{
+			if (SystemTimer::get().ms_since_boot() > corb_timeout_ms)
+				return BAN::Error::from_errno(ETIMEDOUT);
+			Processor::pause();
+		}
+
+		// set base address
+		m_bar0->write32(Regs::RIRBLBASE,  rirb_paddr | (m_bar0->read32(Regs::RIRBLBASE) & 0x0000007F));
+		if (m_is64bit)
+			m_bar0->write32(Regs::RIRBUBASE,  rirb_paddr >> 32);
+		// set number of entries
+		m_bar0->write8(Regs::RIRBSIZE,   (m_bar0->read8(Regs::RIRBSIZE) & 0xFC) | rirb_size.value);
+		// reset write pointer
+		m_bar0->write16(Regs::RIRBWP,    (m_bar0->read16(Regs::RIRBWP)  & 0x7FFF) | 0x8000);
+		// send interrupt on every packet
+		m_bar0->write16(Regs::RINTCNT,   (m_bar0->read16(Regs::RINTCNT) & 0xFF00) | 0x01);
+
+		// enable corb and rirb
+		m_bar0->write8(Regs::CORBCTL,    (m_bar0->read8(Regs::CORBCTL) & 0xFC) | 3);
+		m_bar0->write8(Regs::RIRBCTL,    (m_bar0->read8(Regs::RIRBCTL) & 0xF8) | 7);
+
+		return {};
+	}
+
+	BAN::ErrorOr<HDAudio::Codec> HDAudioController::initialize_codec(uint8_t codec)
+	{
+		const auto resp = TRY(send_command({
+			.data = 0x04,
+			.command = 0xF00,
+			.node_index = 0,
+			.codec_address = codec,
+		}));
+		const uint8_t start = (resp >> 16) & 0xFF;
+		const uint8_t count = (resp >>  0) & 0xFF;
+		if (count == 0)
+			return BAN::Error::from_errno(ENODEV);
+
+		HDAudio::Codec result {};
+		result.id = codec;
+		TRY(result.nodes.reserve(count));
+
+		for (size_t i = 0; i < count; i++)
+			if (auto node_or_error = initialize_node(codec, start + i); !node_or_error.is_error())
+				MUST(result.nodes.emplace_back(node_or_error.release_value()));
+
+		return result;
+	}
+
+	BAN::ErrorOr<HDAudio::AFGNode> HDAudioController::initialize_node(uint8_t codec, uint8_t node)
+	{
+		{
+			const auto resp = TRY(send_command({
+				.data = 0x05,
+				.command = 0xF00,
+				.node_index = node,
+				.codec_address = codec,
+			}));
+			const uint8_t type = (resp >> 0) & 0xFF;
+			if (type != 0x01)
+				return BAN::Error::from_errno(ENODEV);
+		}
+
+		const auto resp = TRY(send_command({
+			.data = 0x04,
+			.command = 0xF00,
+			.node_index = node,
+			.codec_address = codec,
+		}));
+		const uint8_t start = (resp >> 16) & 0xFF;
+		const uint8_t count = (resp >>  0) & 0xFF;
+
+		HDAudio::AFGNode result {};
+		result.id = node;
+		TRY(result.widgets.reserve(count));
+
+		for (size_t i = 0; i < count; i++)
+			if (auto widget_or_error = initialize_widget(codec, start + i); !widget_or_error.is_error())
+				MUST(result.widgets.emplace_back(widget_or_error.release_value()));
+
+		return result;
+	}
+
+	BAN::ErrorOr<HDAudio::AFGWidget> HDAudioController::initialize_widget(uint8_t codec, uint8_t widget)
+	{
+		const auto send_command_or_zero =
+			[codec, widget, this](uint16_t cmd, uint8_t data) -> uint32_t
+			{
+				const auto command = HDAudio::CORBEntry {
+					.data = data,
+					.command = cmd,
+					.node_index = widget,
+					.codec_address = codec,
+				};
+				if (auto res = send_command(command); !res.is_error())
+					return res.release_value();
+				return 0;
+			};
+
+		using HDAudio::AFGWidget;
+		const AFGWidget::Type type_list[] {
+			AFGWidget::Type::OutputConverter,
+			AFGWidget::Type::InputConverter,
+			AFGWidget::Type::Mixer,
+			AFGWidget::Type::Selector,
+			AFGWidget::Type::PinComplex,
+			AFGWidget::Type::Power,
+			AFGWidget::Type::VolumeKnob,
+			AFGWidget::Type::BeepGenerator,
+		};
+
+		const uint8_t type = (send_command_or_zero(0xF00, 0x09) >> 20) & 0x0F;
+		if (type > sizeof(type_list) / sizeof(*type_list))
+			return BAN::Error::from_errno(ENOTSUP);
+
+		AFGWidget result {};
+		result.type = type_list[type];
+		result.id = widget;
+
+		if (result.type == AFGWidget::Type::PinComplex)
+		{
+			const uint32_t cap = send_command_or_zero(0xF00, 0x0C);
+			result.pin_complex = {
+				.input   = !!(cap & (1 << 5)),
+				.output  = !!(cap & (1 << 4)),
+				.display = !!(cap & ((1 << 7) | (1 << 24))),
+				.config  = send_command_or_zero(0xF1C, 0x00),
+			};
+		}
+
+		if (const uint32_t out_amp_cap = send_command_or_zero(0xF00, 0x12))
+		{
+			const uint8_t offset    = (out_amp_cap >>  0) & 0x7F;
+			const uint8_t num_steps = (out_amp_cap >>  8) & 0x7F;
+			const uint8_t step_size = (out_amp_cap >> 16) & 0x7F;
+			const bool    mute      = (out_amp_cap >> 31);
+			result.output_amplifier = HDAudio::AFGWidget::Amplifier {
+				.offset = offset,
+				.num_steps = num_steps,
+				.step_size = step_size,
+				.mute = mute,
+			};
+		}
+
+		const uint8_t connection_info = send_command_or_zero(0xF00, 0x0E);
+		const uint8_t conn_width = (connection_info & 0x80) ? 2 : 1;
+		const uint8_t conn_count = connection_info & 0x3F;
+		const uint16_t conn_mask = (1 << (8 * conn_width)) - 1;
+
+		TRY(result.connections.resize(conn_count, 0));
+		for (size_t i = 0; i < conn_count; i += 4 / conn_width)
+		{
+			const uint32_t conn = send_command_or_zero(0xF02, i);
+			for (size_t j = 0; j < sizeof(conn) / conn_width && i + j < conn_count; j++)
+				result.connections[i + j] = (conn >> (8 * conn_width * j)) & conn_mask;
+		}
+
+		return result;
+	}
+
+	BAN::ErrorOr<uint32_t> HDAudioController::send_command(HDAudio::CORBEntry command)
+	{
+		using Regs = HDAudio::Regs;
+
+		// TODO: allow concurrent commands with CORB/RIRB
+		LockGuard _(m_command_mutex);
+
+		if (!m_use_immediate_command)
+		{
+			SpinLockGuard sguard(m_rb_lock);
+
+			MMIO::write32(m_corb.vaddr + m_corb.index * sizeof(uint32_t), command.raw);
+			m_bar0->write16(Regs::CORBWP, (m_bar0->read16(Regs::CORBWP) & 0xFF00) | m_corb.index);
+			m_corb.index = (m_corb.index + 1) % m_corb.size;
+
+			const uint64_t waketime_ms = SystemTimer::get().ms_since_boot() + 10;
+			while ((m_bar0->read16(Regs::RIRBWP) & 0xFF) != m_rirb.index)
+			{
+				if (SystemTimer::get().ms_since_boot() > waketime_ms)
+					return BAN::Error::from_errno(ETIMEDOUT);
+				SpinLockGuardAsMutex smutex(sguard);
+				m_rb_blocker.block_with_timeout_ms(10, &smutex);
+			}
+
+			const size_t offset = 2 * m_rirb.index * sizeof(uint32_t);
+			m_rirb.index = (m_rirb.index + 1) % m_rirb.size;
+			return MMIO::read32(m_rirb.vaddr + offset);
+		}
+		else
+		{
+			uint64_t waketime_ms = SystemTimer::get().ms_since_boot() + 10;
+			while (m_bar0->read16(Regs::ICIS) & 1)
+			{
+				if (SystemTimer::get().ms_since_boot() > waketime_ms)
+					break;
+				Processor::pause();
+			}
+
+			// clear ICB if it did not clear "in reasonable timeout period"
+			// and make sure IRV is cleared
+			if (m_bar0->read16(Regs::ICIS) & 3)
+				m_bar0->write16(Regs::ICIS, (m_bar0->read16(Regs::ICIS) & 0x00FC) | 2);
+
+			m_bar0->write32(Regs::ICOI, command.raw);
+
+			m_bar0->write16(Regs::ICIS, (m_bar0->read16(Regs::ICIS) & 0x00FC) | 1);
+
+			waketime_ms = SystemTimer::get().ms_since_boot() + 10;
+			while (!(m_bar0->read16(Regs::ICIS) & 2))
+			{
+				if (SystemTimer::get().ms_since_boot() > waketime_ms)
+					return BAN::Error::from_errno(ETIMEDOUT);
+				Processor::pause();
+			}
+
+			return m_bar0->read32(Regs::ICII);
+		}
+	}
+
+	void HDAudioController::handle_irq()
+	{
+		using Regs = HDAudio::Regs;
+
+		const uint32_t intsts = m_bar0->read32(Regs::INTSTS);
+		if (!(intsts & (1u << 31)))
+			return;
+
+		if (intsts & (1 << 30))
+		{
+			if (const uint8_t rirbsts = m_bar0->read8(Regs::RIRBSTS) & ((1 << 2) | (1 << 0)))
+			{
+				if (rirbsts & (1 << 2))
+					dwarnln("RIRB response overrun");
+				if (rirbsts & (1 << 0))
+				{
+					SpinLockGuard _(m_rb_lock);
+					m_rb_blocker.unblock();
+				}
+				m_bar0->write8(Regs::RIRBSTS, rirbsts);
+			}
+
+			if (const uint8_t corbsts = m_bar0->read8(Regs::CORBSTS) & (1 << 0))
+			{
+				dwarnln("CORB memory error");
+				m_bar0->write8(Regs::CORBSTS, corbsts);
+			}
+		}
+
+		for (size_t i = 0; i < 30; i++)
+		{
+			if (!(intsts & (1 << i)))
+				continue;
+			if (m_allocated_streams[i] == nullptr)
+				dwarnln("interrupt from an unallocated stream??");
+			else
+				static_cast<HDAudioFunctionGroup*>(m_allocated_streams[i])->on_stream_interrupt(i);
+		}
+	}
+
+	uint8_t HDAudioController::get_stream_index(HDAudio::StreamType type, uint8_t index) const
+	{
+		switch (type)
+		{
+			case HDAudio::StreamType::Bidirectional:
+				index += m_output_streams;
+				[[fallthrough]];
+			case HDAudio::StreamType::Output:
+				index += m_input_streams;
+				[[fallthrough]];
+			case HDAudio::StreamType::Input:
+				break;
+		}
+		return index;
+	}
+
+	BAN::ErrorOr<uint8_t> HDAudioController::allocate_stream_id()
+	{
+		for (uint8_t id = 1; id < 16; id++)
+		{
+			if (m_allocated_stream_ids & (1 << id))
+				continue;
+			m_allocated_stream_ids |= 1 << id;
+			return id;
+		}
+
+		return BAN::Error::from_errno(EAGAIN);
+	}
+
+	void HDAudioController::deallocate_stream_id(uint8_t id)
+	{
+		ASSERT(m_allocated_stream_ids & (1 << id));
+		m_allocated_stream_ids &= ~(1 << id);
+	}
+
+	BAN::ErrorOr<uint8_t> HDAudioController::allocate_stream(HDAudio::StreamType type, void* afg)
+	{
+		const uint8_t stream_count_lookup[] {
+			[(int)HDAudio::StreamType::Input]         = m_input_streams,
+			[(int)HDAudio::StreamType::Output]        = m_output_streams,
+			[(int)HDAudio::StreamType::Bidirectional] = m_bidir_streams,
+		};
+
+		const uint8_t stream_count = stream_count_lookup[static_cast<int>(type)];
+		for (uint8_t i = 0; i < stream_count; i++)
+		{
+			const uint8_t index = get_stream_index(type, i);
+			if (m_allocated_streams[index])
+				continue;
+			m_allocated_streams[index] = afg;
+			return index;
+		}
+
+		return BAN::Error::from_errno(EAGAIN);
+	}
+
+	void HDAudioController::deallocate_stream(uint8_t index)
+	{
+		ASSERT(m_allocated_streams[index]);
+		m_allocated_streams[index] = nullptr;
+		// TODO: maybe make sure the stream is stopped/reset (?)
+	}
+
+}

kernel/kernel/CPUID.cpp

@@ -75,6 +75,16 @@ namespace CPUID
 		return buffer[3] & (1 << 26);
 	}
 
+	bool has_invariant_tsc()
+	{
+		uint32_t buffer[4] {};
+		get_cpuid(0x80000000, buffer);
+		if (buffer[0] < 0x80000007)
+			return false;
+		get_cpuid(0x80000007, buffer);
+		return buffer[3] & (1 << 8);
+	}
+
 	const char* feature_string_ecx(uint32_t feat)
 	{
 		switch (feat)

kernel/kernel/Debug.cpp

@@ -29,28 +29,33 @@ namespace Debug
 	static uint8_t s_debug_ansi_state { 0 };
 
 	void dump_stack_trace()
+	{
+		dump_stack_trace(0, reinterpret_cast<uintptr_t>(__builtin_frame_address(0)));
+	}
+
+	void dump_stack_trace(uintptr_t ip, uintptr_t bp)
 	{
 		using namespace Kernel;
 
 		struct stackframe
 		{
 			stackframe* bp;
-			uintptr_t ip;
+			void* ip;
 		};
 
 		SpinLockGuard _(s_debug_lock);
 
-		stackframe* frame = (stackframe*)__builtin_frame_address(0);
+		const stackframe* frame = reinterpret_cast<const stackframe*>(bp);
-		if (!frame)
+
-		{
+		void* first_ip = frame->ip;
-			dprintln("Could not get frame address");
+		void* last_ip = 0;
-			return;
-		}
-		uintptr_t first_ip = frame->ip;
-		uintptr_t last_ip = 0;
 		bool first = true;
 
 		BAN::Formatter::print(Debug::putchar, "\e[36mStack trace:\r\n");
+
+		if (ip != 0)
+			BAN::Formatter::print(Debug::putchar, "    {}\r\n", reinterpret_cast<void*>(ip));
+
 		while (frame)
 		{
 			if (!PageTable::is_valid_pointer((vaddr_t)frame))
@@ -330,6 +335,8 @@ namespace Debug
 
 	void print_prefix(const char* file, int line)
 	{
+		if (file[0] == '.' && file[1] == '/')
+			file += 2;
 		auto ms_since_boot = Kernel::SystemTimer::is_initialized() ? Kernel::SystemTimer::get().ms_since_boot() : 0;
 		BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {}:{}: ", ms_since_boot / 1000, ms_since_boot % 1000, file, line);
 	}

kernel/kernel/Device/DebugDevice.cpp

@@ -21,10 +21,11 @@ namespace Kernel
 	{
 		auto ms_since_boot = SystemTimer::get().ms_since_boot();
 		SpinLockGuard _(Debug::s_debug_lock);
-		BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {} {}: ",
+		BAN::Formatter::print(Debug::putchar, "[{5}.{3}] {}:{} {}: ",
 			ms_since_boot / 1000,
 			ms_since_boot % 1000,
 			Kernel::Process::current().pid(),
+			Thread::current().tid(),
 			Kernel::Process::current().name()
 		);
 		for (size_t i = 0; i < buffer.size(); i++)

kernel/kernel/Device/FramebufferDevice.cpp

@@ -6,6 +6,7 @@
 #include <kernel/Terminal/FramebufferTerminal.h>
 
 #include <sys/framebuffer.h>
+#include <sys/ioctl.h>
 #include <sys/mman.h>
 #include <sys/sysmacros.h>
 
@@ -133,6 +134,26 @@ namespace Kernel
 		return bytes_to_copy;
 	}
 
+	BAN::ErrorOr<long> FramebufferDevice::ioctl_impl(int cmd, void* arg)
+	{
+		switch (cmd)
+		{
+			case FB_MSYNC_RECTANGLE:
+			{
+				auto& rectangle = *static_cast<fb_msync_region*>(arg);
+				sync_pixels_rectangle(
+					rectangle.min_x,
+					rectangle.min_y,
+					rectangle.max_x - rectangle.min_x,
+					rectangle.max_y - rectangle.min_y
+				);
+				return 0;
+			}
+		}
+
+		return CharacterDevice::ioctl(cmd, arg);
+	}
+
 	uint32_t FramebufferDevice::get_pixel(uint32_t x, uint32_t y) const
 	{
 		ASSERT(x < m_width && y < m_height);