Kernel: Allow booting with multiple processors on i686

Also remove unnecessary ds clearing in x86_64.

kernel/arch/i686/PageTable.cpp

@@ -117,16 +117,7 @@ namespace Kernel
 		ASSERT(s_global_pdpte == 0);
 		s_global_pdpte = V2P(allocate_zeroed_page_aligned_page());
 
-		ASSERT(m_highest_paging_struct == 0);
-		m_highest_paging_struct = V2P(kmalloc(32, 32, true));
-		ASSERT(m_highest_paging_struct);
-
-		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
-		pdpt[0] = 0;
-		pdpt[1] = 0;
-		pdpt[2] = 0;
-		pdpt[3] = s_global_pdpte;
-		static_assert(KERNEL_OFFSET == 0xC0000000);
+		map_kernel_memory();
 
 		prepare_fast_page();
 
@@ -171,8 +162,7 @@ namespace Kernel
 		constexpr uint64_t pte   = (fast_page() >> 12) & 0x1FF;
 
 		uint64_t* pdpt = reinterpret_cast<uint64_t*>(P2V(m_highest_paging_struct));
-		ASSERT(!(pdpt[pdpte] & Flags::Present));
-		pdpt[pdpte] = V2P(allocate_zeroed_page_aligned_page()) | Flags::Present;
+		ASSERT(pdpt[pdpte] & Flags::Present);
 
 		uint64_t* pd = reinterpret_cast<uint64_t*>(P2V(pdpt[pdpte]) & PAGE_ADDR_MASK);
 		ASSERT(!(pd[pde] & Flags::Present));
@@ -247,7 +237,7 @@ namespace Kernel
 		pdpt[0] = 0;
 		pdpt[1] = 0;
 		pdpt[2] = 0;
-		pdpt[3] = s_global_pdpte;
+		pdpt[3] = s_global_pdpte | Flags::Present;
 		static_assert(KERNEL_OFFSET == 0xC0000000);
 	}
 

kernel/arch/i686/Syscall.S

@@ -1,19 +1,31 @@
 .global sys_fork_trampoline
 sys_fork_trampoline:
-	ud2
-	subl $4, %esp
-	pushl %ebx
 	pushl %ebp
+	pushl %ebx
+	pushl %esi
+	pushl %edi
+
 	call read_ip
 	testl %eax, %eax
-	je .done
+	jz .reload_stack
+
+	movl %esp, %ebx
+
 	subl $8, %esp
 	pushl %eax
-	pushl %esp
+	pushl %ebx
 	call sys_fork
 	addl $16, %esp
-.done:
-	popl %ebp
+
+ .done:
+	popl %edi
+	popl %esi
 	popl %ebx
-	addl $4, %esp
+	popl %ebp
 	ret
+
+ .reload_stack:
+	call get_thread_start_sp
+	movl %eax, %esp
+	xorl %eax, %eax
+	jmp .done

kernel/arch/i686/Thread.S

@@ -7,7 +7,7 @@ read_ip:
 # void start_kernel_thread()
 .global start_kernel_thread
 start_kernel_thread:
-	call get_start_kernel_thread_sp
+	call get_thread_start_sp
 	movl %eax, %esp
 
 	# STACK LAYOUT
@@ -32,3 +32,37 @@ start_kernel_thread:
 	pushl %edi
 	call *%esi
 	addl $16, %esp
+
+
+.global start_userspace_thread
+start_userspace_thread:
+	call get_thread_start_sp
+	movl %eax, %esp
+
+	# STACK LAYOUT
+	#   entry
+	#   argc
+	#   argv
+	#   envp
+	#   userspace stack
+
+	call get_userspace_thread_stack_top
+
+	movw $(0x20 | 3), %bx
+	movw %bx, %ds
+	movw %bx, %es
+	movw %bx, %fs
+	movw %bx, %gs
+	xorw %bx, %bx
+
+	popl %edx
+	popl %esi
+	popl %edi
+	popl %ecx
+
+	pushl $(0x20 | 3)
+	pushl %eax
+	pushl $0x202
+	pushl $(0x18 | 3)
+	pushl %ecx
+	iret

kernel/arch/i686/boot.S

@@ -231,7 +231,6 @@ system_halt:
 .code16
 .global ap_trampoline
 ap_trampoline:
-	ud2
 	jmp 1f
 .align 8
 ap_stack_ptr:
@@ -239,10 +238,7 @@ ap_stack_ptr:
 1:
 	cli; cld
 	ljmpl $0x00, $ap_cs_clear
-
 ap_cs_clear:
-	xorw %ax, %ax
-	movw %ax, %ds
 
 	# load ap gdt and enter protected mode
 	lgdt ap_gdtr

kernel/arch/i686/interrupts.S

@@ -1,78 +1,30 @@
-isr_stub:
-	pusha
+.macro push_userspace
+	pushw %gs
+	pushw %fs
+	pushw %es
+	pushw %ds
+	pushal
+.endm
 
-	movl %cr0, %eax; pushl %eax
-	movl %cr2, %eax; pushl %eax
-	movl %cr3, %eax; pushl %eax
-	movl %cr4, %eax; pushl %eax
+.macro load_kernel_segments
+	movw $0x10, %ax
+	movw %ax, %ds
+	movw %ax, %es
+	movw %ax, %fs
 
-	movl %esp,     %eax // register ptr
-	leal 56(%esp), %ebx // interrupt stack ptr
-	movl 52(%esp), %ecx // error code
-	movl 48(%esp), %edx // isr number
+	movw $0x28, %ax
+	movw %ax, %gs
+.endm
 
-	subl $12, %esp
-	pushl %eax
-	pushl %ebx
-	pushl %ecx
-	pushl %edx
-	call cpp_isr_handler
-	addl $44, %esp
-
-	popa
-	addl $8, %esp
-	iret
-
-irq_stub:
-	pusha
-
-	movl 32(%esp), %eax # interrupt number
-
-	subl $8, %esp
-	pushl %eax
-	call cpp_irq_handler
-	addl $12, %esp
-
-	popa
-	addl $8, %esp
-	iret
-
-.global asm_reschedule_handler
-asm_reschedule_handler:
-	pusha
-
-	movl %esp,     %eax # interrupt registers ptr
-	leal 32(%esp), %ebx # interrupt stack ptr
-
-	subl $12, %esp
-	pushl %eax
-	pushl %ebx
-	call cpp_reschedule_handler
-	addl $20, %esp
-
-	popa
-	iret
-
-// arguments in EAX, EBX, ECX, EDX, ESI, EDI
-.global syscall_asm
-syscall_asm:
-	ud2
-	pusha
-
-	pushl %esp
-	addl $36, (%esp)
-
-	pushl %edi
-	pushl %esi
-	pushl %edx
-	pushl %ecx
-	pushl %ebx
-	pushl %eax
-
-	call cpp_syscall_handler
-
-	addl $60, %esp
+.macro pop_userspace
+	popal
+	popw %ds
+	popw %es
+	popw %fs
+	popw %gs
+.endm
 
+.macro pop_userspace_skip_eax
 	popl %edi
 	popl %esi
 	popl %ebp
@@ -82,6 +34,93 @@ syscall_asm:
 	popl %ecx
 	addl $4, %esp
 
+	popw %ds
+	popw %es
+	popw %fs
+	popw %gs
+.endm
+
+isr_stub:
+	push_userspace
+	load_kernel_segments
+
+	movl %cr0, %eax; pushl %eax
+	movl %cr2, %eax; pushl %eax
+	movl %cr3, %eax; pushl %eax
+	movl %cr4, %eax; pushl %eax
+
+	movl %esp,     %eax // register ptr
+	leal 64(%esp), %ebx // interrupt stack ptr
+	movl 60(%esp), %ecx // error code
+	movl 56(%esp), %edx // isr number
+
+	subl $12, %esp
+	pushl %eax
+	pushl %ebx
+	pushl %ecx
+	pushl %edx
+	call cpp_isr_handler
+	addl $44, %esp
+
+	pop_userspace
+	addl $8, %esp
+	iret
+
+irq_stub:
+	push_userspace
+	load_kernel_segments
+
+	movl 40(%esp), %eax # interrupt number
+
+	subl $12, %esp
+	pushl %eax
+	call cpp_irq_handler
+	addl $16, %esp
+
+	pop_userspace
+	addl $8, %esp
+	iret
+
+.global asm_reschedule_handler
+asm_reschedule_handler:
+	push_userspace
+	load_kernel_segments
+
+	movl %esp,     %eax # interrupt registers ptr
+	leal 40(%esp), %ebx # interrupt stack ptr
+
+	subl $12, %esp
+	pushl %eax
+	pushl %ebx
+	call cpp_reschedule_handler
+	addl $20, %esp
+
+	pop_userspace
+	iret
+
+// arguments in EAX, EBX, ECX, EDX, ESI, EDI
+.global syscall_asm
+syscall_asm:
+	push_userspace
+
+	subl $8, %esp
+
+	pushl %esp
+	addl $48, (%esp)
+
+	pushl %edi
+	pushl %esi
+	pushl %edx
+	pushl %ecx
+	pushl %ebx
+	pushl %eax
+
+	load_kernel_segments
+
+	call cpp_syscall_handler
+	addl $36, %esp
+
+	pop_userspace_skip_eax
 	iret
 
 .macro isr n

kernel/arch/x86_64/Syscall.S

@@ -6,13 +6,16 @@ sys_fork_trampoline:
 	pushq %r13
 	pushq %r14
 	pushq %r15
+
 	call read_ip
 	testq %rax, %rax
-	je .done
+	je .reload_stack
+
 	movq %rax, %rsi
 	movq %rsp, %rdi
 	call sys_fork
-.done:
+
+ .done:
 	popq %r15
 	popq %r14
 	popq %r13
@@ -20,3 +23,9 @@ sys_fork_trampoline:
 	popq %rbp
 	popq %rbx
 	ret
+
+ .reload_stack:
+	call get_thread_start_sp
+	movq %rax, %rsp
+	xorq %rax, %rax
+	jmp .done

kernel/arch/x86_64/Thread.S

@@ -7,7 +7,7 @@ read_ip:
 # void start_kernel_thread()
 .global start_kernel_thread
 start_kernel_thread:
-	call get_start_kernel_thread_sp
+	call get_thread_start_sp
 	movq %rax, %rsp
 
 	# STACK LAYOUT
@@ -24,3 +24,29 @@ start_kernel_thread:
 	movq 24(%rsp), %rdi
 	movq 16(%rsp), %rsi
 	call *%rsi
+
+.global start_userspace_thread
+start_userspace_thread:
+	call get_thread_start_sp
+	movq %rax, %rsp
+
+	# STACK LAYOUT
+	#   entry
+	#   argc
+	#   argv
+	#   envp
+	#   userspace stack
+
+	call get_userspace_thread_stack_top
+
+	popq %rdx
+	popq %rsi
+	popq %rdi
+	popq %rcx
+
+	pushq $(0x20 | 3)
+	pushq %rax
+	pushq $0x202
+	pushq $(0x18 | 3)
+	pushq %rcx
+	iretq

kernel/arch/x86_64/boot.S

@@ -249,10 +249,7 @@ ap_stack_ptr:
 1:
 	cli; cld
 	ljmpl $0x00, $ap_cs_clear
-
 ap_cs_clear:
-	xorw %ax, %ax
-	movw %ax, %ds
 
 	# load ap gdt and enter protected mode
 	lgdt ap_gdtr

kernel/include/kernel/GDT.h

@@ -124,6 +124,7 @@ namespace Kernel
 			m_tss.rsp0 = sp;
 #elif ARCH(i686)
 			m_tss.esp0 = sp;
+			m_tss.ss0 = 0x10;
 #endif
 		}
 

kernel/kernel/Scheduler.cpp

@@ -29,11 +29,6 @@ namespace Kernel
 		return *s_instance;
 	}
 
-	extern "C" uintptr_t get_start_kernel_thread_sp()
-	{
-		return Scheduler::get().current_thread().kernel_stack_top() - 4 * sizeof(uintptr_t);
-	}
-
 	void Scheduler::start()
 	{
 		ASSERT(Processor::get_interrupt_state() == InterruptState::Disabled);
@@ -120,11 +115,7 @@ namespace Kernel
 		if (thread->state() == Thread::State::NotStarted)
 			thread->m_state = Thread::State::Executing;
 
-		ASSERT(thread->interrupt_stack().ip);
-		ASSERT(thread->interrupt_stack().sp);
-
 		Processor::gdt().set_tss_stack(thread->kernel_stack_top());
-
 		Processor::get_interrupt_stack() = thread->interrupt_stack();
 		Processor::get_interrupt_registers() = thread->interrupt_registers();
 	}
@@ -154,7 +145,9 @@ namespace Kernel
 			"movq %[load_sp], %%rsp;"
 			"int %[ipi];"
 			"movq %%rcx, %%rsp;"
-			:: [load_sp]"r"(Processor::current_stack_top()),
+			// NOTE: This is offset by 2 pointers since interrupt without PL change
+			//       does not push SP and SS. This allows accessing "whole" interrupt stack.
+			:: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)),
 			   [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI)
 			:  "memory", "rcx"
 		);
@@ -164,7 +157,9 @@ namespace Kernel
 			"movl %[load_sp], %%esp;"
 			"int %[ipi];"
 			"movl %%ecx, %%esp;"
-			:: [load_sp]"r"(Processor::current_stack_top()),
+			// NOTE: This is offset by 2 pointers since interrupt without PL change
+			//       does not push SP and SS. This allows accessing "whole" interrupt stack.
+			:: [load_sp]"r"(Processor::current_stack_top() - 2 * sizeof(uintptr_t)),
 			   [ipi]"i"(IRQ_VECTOR_BASE + IRQ_IPI)
 			:  "memory", "ecx"
 		);

kernel/kernel/Syscall.cpp

@@ -28,9 +28,9 @@ namespace Kernel
 #undef O
 	};
 
-	extern "C" long cpp_syscall_handler(int syscall, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5, InterruptStack& interrupt_stack)
+	extern "C" long cpp_syscall_handler(int syscall, uintptr_t arg1, uintptr_t arg2, uintptr_t arg3, uintptr_t arg4, uintptr_t arg5, InterruptStack* interrupt_stack)
 	{
-		ASSERT((interrupt_stack.cs & 0b11) == 0b11);
+		ASSERT(GDT::is_user_segment(interrupt_stack->cs));
 
 		asm volatile("sti");
 

kernel/kernel/Thread.cpp

@@ -13,6 +13,7 @@ namespace Kernel
 {
 
 	extern "C" [[noreturn]] void start_kernel_thread();
+	extern "C" [[noreturn]] void start_userspace_thread();
 
 	extern "C" void signal_trampoline();
 
@@ -23,6 +24,16 @@ namespace Kernel
 		*(uintptr_t*)rsp = (uintptr_t)value;
 	}
 
+	extern "C" uintptr_t get_thread_start_sp()
+	{
+		return Thread::current().interrupt_stack().sp;
+	}
+
+	extern "C" uintptr_t get_userspace_thread_stack_top()
+	{
+		return Thread::current().userspace_stack_top() - 4 * sizeof(uintptr_t);
+	}
+
 	static pid_t s_next_tid = 1;
 
 	BAN::ErrorOr<Thread*> Thread::create_kernel(entry_t entry, void* data, Process* process)
@@ -181,6 +192,12 @@ namespace Kernel
 		thread->m_interrupt_stack.sp = sp;
 		thread->m_interrupt_stack.ss = 0x10;
 
+#if ARCH(x86_64)
+		thread->m_interrupt_registers.rax = 0;
+#elif ARCH(i686)
+		thread->m_interrupt_registers.eax = 0;
+#endif
+
 		thread_deleter.disable();
 
 		return thread;
@@ -197,19 +214,19 @@ namespace Kernel
 		ASSERT(userspace_info.entry);
 
 		// Initialize stack for returning
-		PageTable::with_fast_page(process().page_table().physical_address_of(userspace_stack_top() - PAGE_SIZE), [&] {
+		PageTable::with_fast_page(process().page_table().physical_address_of(kernel_stack_top() - PAGE_SIZE), [&] {
 			uintptr_t sp = PageTable::fast_page() + PAGE_SIZE;
-			write_to_stack(sp, nullptr);
+			write_to_stack(sp, userspace_info.entry);
 			write_to_stack(sp, userspace_info.argc);
 			write_to_stack(sp, userspace_info.argv);
 			write_to_stack(sp, userspace_info.envp);
 		});
 
-		m_interrupt_stack.ip = userspace_info.entry;
-		m_interrupt_stack.cs = 0x18 | 3;
-		m_interrupt_stack.flags = 0x202;
-		m_interrupt_stack.sp = userspace_stack_top() - 4 * sizeof(uintptr_t);
-		m_interrupt_stack.ss = 0x20 | 3;
+		m_interrupt_stack.ip = reinterpret_cast<vaddr_t>(start_userspace_thread);;
+		m_interrupt_stack.cs = 0x08;
+		m_interrupt_stack.flags = 0x002;
+		m_interrupt_stack.sp = kernel_stack_top() - 4 * sizeof(uintptr_t);
+		m_interrupt_stack.ss = 0x10;
 
 		memset(&m_interrupt_registers, 0, sizeof(InterruptRegisters));
 	}

libc/arch/i686/crt0.S

@@ -2,31 +2,32 @@
 
 .global _start
 _start:
-	# zero out stack frame
 	pushl $0
-	pushl $0
-	movl %esp, %ebp
-
-	# FIXME: handle stack alignment
-	ud2
-
-	# push argc, argv, environ for call to main
-	pushl %edx
-	pushl %esi
 	pushl %edi
-
-	# initialize libc
+	pushl %esi
 	pushl %edx
-	call _init_libc
-	addl $4, %esp
 
-	# call global constructos
+	# STACK LAYOUT
+	#   null
+	#   argc
+	#   argv
+	#   envp
+
+	xorl %ebp, %ebp
+
+	# init libc (envp already as argument)
+	call _init_libc
+
+	# call global constructors
 	call _init
 
-	# call main, arguments are already on stack
+	# call main
+	movl 0(%esp), %eax
+	xchgl %eax, 8(%esp)
+	movl %eax, (%esp)
 	call main
 
-	# cleanly exit the process
+	subl $12, %esp
 	pushl %eax
 	call exit
 

libc/arch/x86_64/crt0.S

@@ -2,6 +2,11 @@
 
 .global _start
 _start:
+	pushq $0
+	pushq %rdi
+	pushq %rsi
+	pushq %rdx
+
 	# STACK LAYOUT
 	#   null
 	#   argc