From 356935bd4f312822d661a2c10245eafae609b0f5 Mon Sep 17 00:00:00 2001
From: Bananymous <oskari.alaranta@bananymous.com>
Date: Tue, 20 May 2025 21:13:31 +0300
Subject: [PATCH] LibC: Fix strtou{,l,ll} for negative values

Also set endp if base is invalid
---
 userspace/libraries/LibC/stdlib.cpp | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/userspace/libraries/LibC/stdlib.cpp b/userspace/libraries/LibC/stdlib.cpp
index 33219c63c1..3549db4e2b 100644
--- a/userspace/libraries/LibC/stdlib.cpp
+++ b/userspace/libraries/LibC/stdlib.cpp
@@ -69,10 +69,8 @@ static constexpr int get_base_digit(char c, int base)
 }
 
 template<BAN::integral T>
-static constexpr bool will_digit_append_overflow(bool negative, T current, int digit, int base)
+static constexpr bool will_digit_append_overflow(T current, int digit, int base)
 {
-	if (BAN::is_unsigned_v<T> && negative && digit)
-		return true;
 	if (BAN::Math::will_multiplication_overflow<T>(current, base))
 		return true;
 	if (BAN::Math::will_addition_overflow<T>(current * base, current < 0 ? -digit : digit))
@@ -86,6 +84,8 @@ static T strtoT(const char* str, char** endp, int base, int& error)
 	// validate base
 	if (base != 0 && (base < 2 || base > 36))
 	{
+		if (endp)
+			*endp = const_cast<char*>(str);
 		error = EINVAL;
 		return 0;
 	}
@@ -134,11 +134,18 @@ static T strtoT(const char* str, char** endp, int base, int& error)
 			break;
 		str++;
 
-		overflow = will_digit_append_overflow(negative, result, digit, base);
+		overflow = will_digit_append_overflow(result, digit, base);
 		if (!overflow)
-			result = result * base + (negative ? -digit : digit);
+		{
+			if (negative && !BAN::is_unsigned_v<T>)
+				digit = -digit;
+			result = result * base + digit;
+		}
 	}
 
+	if (negative && BAN::is_unsigned_v<T>)
+		result = -result;
+
 	// save endp if asked
 	if (endp)
 	{