Kernel: Improve open() POSIX compatability

Also rename Process::sys_creat -> create_file since it is not
actually a syscall and only used by open as a healper.

kernel/include/kernel/Process.h

@@ -89,12 +89,12 @@ namespace Kernel
 		BAN::ErrorOr<long> sys_getegid() const { return m_credentials.egid(); }
 		BAN::ErrorOr<long> sys_getpgid(pid_t);
 
+		BAN::ErrorOr<void> create_file(BAN::StringView name, mode_t mode);
 		BAN::ErrorOr<long> sys_open(BAN::StringView, int, mode_t = 0);
 		BAN::ErrorOr<long> sys_openat(int, BAN::StringView, int, mode_t = 0);
 		BAN::ErrorOr<long> sys_close(int fd);
 		BAN::ErrorOr<long> sys_read(int fd, void* buffer, size_t count);
 		BAN::ErrorOr<long> sys_write(int fd, const void* buffer, size_t count);
-		BAN::ErrorOr<long> sys_creat(BAN::StringView name, mode_t);
 
 		BAN::ErrorOr<long> sys_pipe(int fildes[2]);
 		BAN::ErrorOr<long> sys_dup(int fildes);

kernel/kernel/OpenFileDescriptorSet.cpp

@@ -55,7 +55,7 @@ namespace Kernel
 
 	BAN::ErrorOr<int> OpenFileDescriptorSet::open(BAN::StringView absolute_path, int flags)
 	{
-		if (flags & ~(O_RDONLY | O_WRONLY | O_NOFOLLOW | O_SEARCH | O_APPEND | O_TRUNC | O_CLOEXEC | O_TTY_INIT))
+		if (flags & ~(O_RDONLY | O_WRONLY | O_NOFOLLOW | O_SEARCH | O_APPEND | O_TRUNC | O_CLOEXEC | O_TTY_INIT | O_DIRECTORY))
 			return BAN::Error::from_errno(ENOTSUP);
 
 		int access_mask = O_EXEC | O_RDONLY | O_WRONLY | O_SEARCH;
@@ -64,7 +64,10 @@ namespace Kernel
 
 		auto file = TRY(VirtualFileSystem::get().file_from_absolute_path(m_credentials, absolute_path, flags));
 
-		if (flags & O_TRUNC)
+		if ((flags & O_DIRECTORY) && !file.inode->mode().ifdir())
+			return BAN::Error::from_errno(ENOTDIR);
+
+		if ((flags & O_TRUNC) && (flags & O_WRONLY) && file.inode->mode().ifreg())
 			TRY(file.inode->truncate(0));
 
 		int fd = TRY(get_free_fd());

kernel/kernel/Process.cpp

@@ -598,6 +598,26 @@ namespace Kernel
 		m_has_called_exec = true;
 	}
 
+	BAN::ErrorOr<void> Process::create_file(BAN::StringView path, mode_t mode)
+	{
+		LockGuard _(m_lock);
+
+		auto absolute_path = TRY(absolute_path_of(path));
+
+		size_t index;
+		for (index = absolute_path.size(); index > 0; index--)
+			if (absolute_path[index - 1] == '/')
+				break;
+
+		auto directory = absolute_path.sv().substring(0, index);
+		auto file_name = absolute_path.sv().substring(index);
+
+		auto parent_inode = TRY(VirtualFileSystem::get().file_from_absolute_path(m_credentials, directory, O_WRONLY)).inode;
+		TRY(parent_inode->create_file(file_name, S_IFREG | (mode & 0777), m_credentials.euid(), m_credentials.egid()));
+
+		return {};
+	}
+
 	BAN::ErrorOr<long> Process::sys_open(BAN::StringView path, int flags, mode_t mode)
 	{
 		LockGuard _(m_lock);
@@ -605,11 +625,13 @@ namespace Kernel
 
 		if (flags & O_CREAT)
 		{
+			if (flags & O_DIRECTORY)
+				return BAN::Error::from_errno(ENOTSUP);
 			auto file_or_error = VirtualFileSystem::get().file_from_absolute_path(m_credentials, absolute_path, O_WRONLY);
 			if (file_or_error.is_error())
 			{
 				if (file_or_error.error().get_error_code() == ENOENT)
-					TRY(sys_creat(path, mode));
+					TRY(create_file(path, mode));
 				else
 					return file_or_error.release_error();
 			}
@@ -697,29 +719,6 @@ namespace Kernel
 		return TRY(m_open_file_descriptors.tell(fd));
 	}
 
-	BAN::ErrorOr<long> Process::sys_creat(BAN::StringView path, mode_t mode)
-	{
-		if ((mode & 0777) != mode)
-			return BAN::Error::from_errno(EINVAL);
-
-		LockGuard _(m_lock);
-
-		auto absolute_path = TRY(absolute_path_of(path));
-
-		size_t index;
-		for (index = absolute_path.size(); index > 0; index--)
-			if (absolute_path[index - 1] == '/')
-				break;
-
-		auto directory = absolute_path.sv().substring(0, index);
-		auto file_name = absolute_path.sv().substring(index);
-
-		auto parent_file = TRY(VirtualFileSystem::get().file_from_absolute_path(m_credentials, directory, O_WRONLY));
-		TRY(parent_file.inode->create_file(file_name, S_IFREG | mode, m_credentials.euid(), m_credentials.egid()));
-
-		return 0;
-	}
-
 	BAN::ErrorOr<void> Process::mount(BAN::StringView source, BAN::StringView target)
 	{
 		BAN::String absolute_source, absolute_target;