general/occlum
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix two bugs by updating SDK's stack ranges on user/kernel switch

Browse Source 
Before this commit, there are two strange bugs:
1. No backtraces are displayed on panic by Rust; and,
2. Thread local storage in Rust sometimes causes panics.

It turns out that the the root cause of the two bugs are the same: Occlum's
patch to Intel SGX SDK that informs SDK about the stack range of the currnet
LibOS user-level thread. The problem about this patch is that it modifies some
fundamental data structures and Rust SGX SDK does not know the modification.
This causes Rust SGX SDK to panic in certain conditions.

To resolve the conflict for good, this commit gets rid of the patch to Intel
SGX SDK by updating SDK's stack ranges upon user/kernel switch.
This commit is contained in:
zongmin.gu 2020-03-06 06:34:36 +00:00 committed by Tate, Hongliang Tian
parent 6f986855e9
commit a1e003ebdb
4 changed files with 59 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/libos/include/task.h
View File

@ -14,6 +14,8 @@ extern "C" {
// See Struct Task in process.rs
struct Task {
uint64_t kernel_rsp;
uint64_t kernel_stack_base;
uint64_t kernel_stack_limit;
uint64_t kernel_fs;
uint64_t user_rsp;
uint64_t user_stack_base;
@ -36,15 +38,21 @@ void do_exit_task(void);
#else /* __ASSEMBLY__ */
/* See /<path-to-linux-sgx>/common/inc/internal/thread_data.h */
#define TD_STACK_BASE (8 * 2)
#define TD_STACK_LIMIT (8 * 3)
#define TD_STACKGUARD_OFFSET (8 * 5)
/* Override the field for stack guard */
#define TD_TASK_OFFSET TD_STACKGUARD_OFFSET
#define TASK_KERNEL_RSP (8 * 0)
#define TASK_KERNEL_FS (8 * 1)
#define TASK_USER_RSP (8 * 2)
#define TASK_USER_FS (8 * 5)
#define TASK_USER_ENTRY_ADDR (8 * 6)
#define TASK_KERNEL_STACK_BASE (8 * 1)
#define TASK_KERNEL_STACK_LIMIT (8 * 2)
#define TASK_KERNEL_FS (8 * 3)
#define TASK_USER_RSP (8 * 4)
#define TASK_USER_STACK_BASE (8 * 5)
#define TASK_USER_STACK_LIMIT (8 * 6)
#define TASK_USER_FS (8 * 7)
#define TASK_USER_ENTRY_ADDR (8 * 8)
/* arch_prctl syscall number and parameter */
#define ARCH_PRCTL 0x9E

35
src/libos/src/process/task.c
View File

@ -1,5 +1,18 @@
#include "task.h"
/* See /<path-to-linux-sgx>/common/inc/internal/thread_data.h */
typedef struct _thread_data_t
{
uint64_t reserved1[2];
uint64_t stack_base_addr;
uint64_t stack_limit_addr;
uint64_t reserved2[15];
uint64_t stack_commit_addr;
} thread_data_t;
extern thread_data_t *get_thread_data(void);
extern void __run_task(struct Task* task);
extern uint64_t __get_stack_guard(void);
@ -26,12 +39,19 @@ static uint64_t get_syscall_stack(struct Task* this_task) {
int do_run_task(struct Task* task) {
jmp_buf libos_state = {0};
thread_data_t* td = get_thread_data();
task->saved_state = &libos_state;
task->kernel_rsp = get_syscall_stack(task);
task->kernel_stack_base = td->stack_base_addr;
task->kernel_stack_limit = td->stack_limit_addr;
//Do do not support stack expansion, need a new design on SGX2 platform.
//Set the stack_commit_addr to 0, as the result no stack expansion happens at any situations
__atomic_store_n(&td->stack_commit_addr, 0, __ATOMIC_RELAXED);
td->stack_base_addr = task->user_stack_base;
td->stack_limit_addr = task->user_stack_limit;
td->stack_commit_addr = task->user_stack_limit;
if (sgx_enable_user_stack(task->user_stack_base, task->user_stack_limit)) {
return -1;
}
SET_CURRENT_TASK(task);
int second = setjmp(libos_state);
@ -41,12 +61,19 @@ int do_run_task(struct Task* task) {
// Jump from do_exit_task
RESET_CURRENT_TASK();
sgx_disable_user_stack();
return 0;
}
void do_exit_task(void) {
struct Task* task = __get_current_task();
jmp_buf* jb = task->saved_state;
thread_data_t* td = get_thread_data();
//Do do not support stack expansion, need a new design on SGX2 platform.
//Set the stack_commit_addr to 0, as the result no stack expansion happens at any situations
__atomic_store_n(&td->stack_commit_addr, 0, __ATOMIC_RELAXED);
td->stack_base_addr = task->kernel_stack_base;
td->stack_limit_addr = task->kernel_stack_limit;
td->stack_commit_addr = task->kernel_stack_limit;
longjmp(*jb, 1);
}

2
src/libos/src/process/task.rs
View File

@ -7,6 +7,8 @@ use super::*;
#[repr(C)]
pub struct Task {
kernel_rsp: usize,
kernel_stack_base: usize,
kernel_stack_limit: usize,
kernel_fs: usize,
user_rsp: usize,
user_stack_base: usize,

14
src/libos/src/syscall/syscall_entry_x86-64.S
View File

@ -46,6 +46,13 @@ __occlum_syscall:
wrfsbase %r11
#endif
// Use kernel stack base and limit
movq TASK_KERNEL_STACK_BASE(%r12), %r11
movq %r11, %gs:TD_STACK_BASE
movq TASK_KERNEL_STACK_LIMIT(%r12), %r11
movq %r11, %gs:TD_STACK_LIMIT
// Make %rsp 16-byte aligned before call
sub $0x8, %rsp
// Pass arg5
@ -71,6 +78,13 @@ __occlum_syscall:
wrfsbase %r11
#endif
// Use user stack base and limit
movq TASK_USER_STACK_BASE(%r12), %r11
movq %r11, %gs:TD_STACK_BASE
movq TASK_USER_STACK_LIMIT(%r12), %r11
movq %r11, %gs:TD_STACK_LIMIT
// Switch to the user stack
movq %rbp, %rsp
// Restore callee-saved registers