11use aero_syscall:: SyscallError ;
22use raw_cpuid:: CpuId ;
33
4- use crate :: arch:: gdt:: GdtEntryType ;
4+ use crate :: arch:: gdt:: { GdtEntryType , Tss , USER_CS , USER_SS } ;
55use crate :: mem:: paging:: VirtAddr ;
66use crate :: userland:: scheduler:: { self , ExitStatus } ;
77use crate :: utils:: sync:: IrqGuard ;
88
99use super :: interrupts:: InterruptErrorStack ;
1010use super :: io;
1111
12- extern "C" {
13- fn x86_64_syscall_handler ( ) ;
14- fn x86_64_sysenter_handler ( ) ;
15- }
12+ use core:: mem:: offset_of;
1613
1714const ARCH_SET_GS : usize = 0x1001 ;
1815const ARCH_SET_FS : usize = 0x1002 ;
1916const ARCH_GET_FS : usize = 0x1003 ;
2017const ARCH_GET_GS : usize = 0x1004 ;
2118
19+ core:: arch:: global_asm!( include_str!( "./registers.S" ) ) ;
20+
21+ /// 64-bit SYSCALL instruction entry point.
22+ ///
23+ /// The instruction supports to to 6 arguments in registers.
24+ ///
25+ /// Registers state on entry:
26+ /// * `RAX` - system call number
27+ /// * `RCX` - return address
28+ /// * `R11` - saved flags (note: R11 is callee-clobbered register in C ABI)
29+ /// * `RDI` - argument 1
30+ /// * `RSI` - argument 2
31+ /// * `RDX` - argument 3
32+ /// * `R10` - argument 4 (needs to be moved to RCX to conform to C ABI)
33+ /// * `R8` - argument 5
34+ /// * `R9` - argument 6
35+ ///
36+ /// (note: `R12`..`R15`, `RBP`, `RBX` are callee-preserved in C ABI)
37+ ///
38+ /// The instruction saves the `RIP` to `RCX`, clears `RFLAGS.RF` then saves `RFLAGS` to `R11`.
39+ /// Followed by, it loads the new `SS`, `CS`, and `RIP` from previously programmed MSRs.
40+ ///
41+ /// The instruction also does not save anything on the stack and does *not* change the `RSP`.
42+ #[ naked]
43+ unsafe extern "C" fn x86_64_syscall_handler ( ) {
44+ asm ! (
45+ // make the GS base point to the kernel TLS
46+ "swapgs" ,
47+ // save the user stack pointer
48+ "mov qword ptr gs:{tss_temp_ustack_off}, rsp" ,
49+ // restore kernel stack
50+ "mov rsp, qword ptr gs:{tss_rsp0_off}" ,
51+ "push {userland_ss}" ,
52+ // push userspace stack ptr
53+ "push qword ptr gs:{tss_temp_ustack_off}" ,
54+ "push r11" ,
55+ "push {userland_cs}" ,
56+ "push rcx" ,
57+
58+ "push rax" ,
59+ "push_scratch" ,
60+ "push_preserved" ,
61+
62+ // push a fake error code to match with the layout of `InterruptErrorStack`
63+ "push 0" ,
64+
65+ "mov rdi, rsp" ,
66+
67+ "cld" ,
68+ "call {x86_64_do_syscall}" ,
69+ "cli" ,
70+
71+ // pop the fake error code
72+ "add rsp, 8" ,
73+
74+ "pop_preserved" ,
75+ "pop_scratch" ,
76+
77+ // cook the sysret frame
78+ "pop rcx" ,
79+ "add rsp, 8" ,
80+ "pop r11" ,
81+ "pop rsp" ,
82+
83+ // restore user GS
84+ "swapgs" ,
85+ "sysretq" ,
86+
87+ // constants:
88+ userland_cs = const USER_CS . bits( ) ,
89+ userland_ss = const USER_SS . bits( ) ,
90+ // XXX: add 8 bytes to skip the x86_64 cpu local self ptr
91+ tss_temp_ustack_off = const offset_of!( Tss , reserved2) + core:: mem:: size_of:: <usize >( ) ,
92+ tss_rsp0_off = const offset_of!( Tss , rsp) + core:: mem:: size_of:: <usize >( ) ,
93+ x86_64_do_syscall = sym x86_64_do_syscall,
94+ options( noreturn)
95+ )
96+ }
97+
98+ /// 64-bit SYSENTER instruction entry point.
99+ ///
100+ /// The SYSENTER mechanism performs a fast transition to the kernel.
101+ ///
102+ /// The new `CS` is loaded from the `IA32_SYSENTER_CS` MSR, and the new instruction and stack
103+ /// pointers are loaded from `IA32_SYSENTER_EIP` and `IA32_SYSENTER_ESP`, respectively. `RFLAGS.IF`
104+ /// is cleared, but other flags are unchanged.
105+ ///
106+ /// As the instruction does not save *any* state, the user is required to provide the return `RIP`
107+ /// and `RSP` in the `RCX` and `R11` registers, respectively. These addresses must be canonical.
108+ ///
109+ /// The instruction expects the call number and arguments in the same registers as for SYSCALL.
110+ #[ naked]
111+ unsafe extern "C" fn x86_64_sysenter_handler ( ) {
112+ asm ! (
113+ "swapgs" ,
114+ // Build the interrupt frame expected by the kernel.
115+ "push {userland_ss}" ,
116+ "push r11" ,
117+ "pushfq" ,
118+ "push {userland_cs}" ,
119+ "push rcx" ,
120+ // Mask the same flags as for SYSCALL.
121+ // XXX: Up to this point the code can be single-stepped if the user sets TF.
122+ "pushfq" ,
123+ "and dword ptr [rsp], 0x300" ,
124+ "popfq" ,
125+ "push rax" ,
126+ "push_scratch" ,
127+ "push_preserved" ,
128+ "push 0" ,
129+ // Store the stack pointer (interrupt frame ptr) in `RBP` for safe keeping, and align the
130+ // stack as specified by the SysV calling convention.
131+ "mov rbp, rsp" ,
132+ "and rsp, ~0xf" ,
133+ "mov rdi, rbp" ,
134+ "call {x86_64_check_sysenter}" ,
135+ "mov rdi, rbp" ,
136+ "call {x86_64_do_syscall}" ,
137+ // Reload the stack pointer, skipping the error code.
138+ "lea rsp, [rbp + 8]" ,
139+ "pop_preserved" ,
140+ "pop_scratch" ,
141+ // Pop the `IRET` frame into the registers expected by `SYSEXIT`.
142+ "pop rdx" , // return `RIP` in `RDX`
143+ "add rsp, 8" ,
144+ "popfq" , // restore saved `RFLAGS`
145+ "pop rcx" , // return `RSP` in `RCX`
146+ // SAFETY: The above call to `x86_64_check_sysenter` is guarantees that we execute
147+ // `sysexit` with canonical addresses in RCX and RDX. Otherwise we would fault in the
148+ // kernel having already swapped back to the user's GS.
149+ "swapgs" ,
150+ // SYSEXIT does *not* restore `IF` to re-enable interrupts.
151+ // This is done here, rather then when restoring `RFLAGS` above, since `STI` will keep
152+ "sti" ,
153+ // interrupts inhibited until after the *following* instruction executes.
154+ "sysexitq" ,
155+ // constants:
156+ userland_cs = const USER_CS . bits( ) ,
157+ userland_ss = const USER_SS . bits( ) ,
158+ x86_64_check_sysenter = sym x86_64_check_sysenter,
159+ x86_64_do_syscall = sym x86_64_do_syscall,
160+ options( noreturn)
161+ )
162+ }
163+
22164fn arch_prctl ( command : usize , address : usize ) -> Result < usize , SyscallError > {
23165 match command {
24166 ARCH_SET_FS => unsafe {
@@ -63,7 +205,6 @@ fn arch_prctl(command: usize, address: usize) -> Result<usize, SyscallError> {
63205///
64206/// We cannot execute `sysexit` on return with non-canonical return addresses, or we
65207/// will take a fault in the kernel with the user's GS base already swapped back.
66- #[ no_mangle]
67208pub ( super ) extern "sysv64" fn x86_64_check_sysenter ( stack : & mut InterruptErrorStack ) {
68209 let rip = stack. stack . iret . rip ;
69210 let rsp = stack. stack . iret . rsp ;
@@ -77,7 +218,6 @@ pub(super) extern "sysv64" fn x86_64_check_sysenter(stack: &mut InterruptErrorSt
77218 }
78219}
79220
80- #[ no_mangle]
81221pub ( super ) extern "C" fn x86_64_do_syscall ( stack : & mut InterruptErrorStack ) {
82222 let stack = & mut stack. stack ;
83223
@@ -156,6 +296,7 @@ pub(super) fn init() {
156296 . map_or ( false , |i| i. has_sysenter_sysexit ( ) ) ;
157297
158298 if has_sysenter {
299+ log:: info!( "enabling support for sysenter" ) ;
159300 unsafe {
160301 io:: wrmsr (
161302 io:: IA32_SYSENTER_CS ,
0 commit comments