From a84803e951d7ab01bfbcc1e3950384b8fd3da16f Mon Sep 17 00:00:00 2001 From: "Tate, Hongliang Tian" Date: Wed, 18 Dec 2019 08:27:54 +0000 Subject: [PATCH] Refactor Occlum PAL as a shared library By providing Occlum PAL as a shared library, it is now possible to embed and use Occlum in an user-controled process (instead of an Occlum-controlled one). The APIs of Occlum PAL can be found in `src/pal/include/occlum_pal_api.h`. The Occlum PAL library, namely `libocclum-pal.so`, can be found in `.occlum/build/lib`. To use the library, check out the source code of `occlum-run` (under `src/run`), which can be seen as a sample code for using the Occlum PAL library. --- Makefile | 2 + src/Enclave.edl | 76 +++- src/Makefile | 2 + src/libos/Cargo.toml | 2 +- src/libos/Makefile | 55 ++- src/libos/src/entry.rs | 40 +- .../attestation/sgx_attestation_agent.rs | 14 +- src/libos/src/fs/mod.rs | 2 +- src/libos/src/lib.rs | 2 +- src/libos/src/net/socket_file/recv.rs | 4 +- src/libos/src/net/socket_file/send.rs | 4 +- src/libos/src/process/mod.rs | 2 +- src/libos/src/process/sched.rs | 16 +- src/libos/src/process/spawn/mod.rs | 58 ++- src/libos/src/process/task.rs | 34 +- src/libos/src/process/thread.rs | 2 +- src/libos/src/time/mod.rs | 16 +- src/pal/Makefile | 29 +- src/pal/atomic.h | 16 - src/pal/futex.c | 32 -- src/pal/futex.h | 9 - src/pal/include/occlum_pal_api.h | 48 ++ src/pal/pal.c | 423 ------------------ src/pal/pal.lds | 8 + src/pal/{ => src/ocalls}/attestation.c | 11 +- src/pal/src/ocalls/fs.c | 6 + src/pal/src/ocalls/net.c | 63 +++ src/pal/src/ocalls/ocalls.h | 5 + src/pal/src/ocalls/sched.c | 24 + src/pal/src/ocalls/spawn.c | 45 ++ src/pal/src/ocalls/time.c | 21 + src/pal/src/pal_api.c | 93 ++++ src/pal/src/pal_enclave.c | 138 ++++++ src/pal/src/pal_enclave.h | 12 + src/pal/src/pal_error.c | 90 ++++ src/pal/src/pal_error.h | 9 + src/pal/src/pal_log.h | 15 + src/pal/src/pal_syscall.h | 10 + src/pal/task.c | 91 ---- src/pal/task.h | 8 - src/run/Makefile | 30 ++ src/run/main.c | 42 ++ tools/occlum | 16 +- tools/occlum-build-enclave | 10 +- 44 files changed, 914 insertions(+), 721 deletions(-) delete mode 100644 src/pal/atomic.h delete mode 100644 src/pal/futex.c delete mode 100644 src/pal/futex.h create mode 100644 src/pal/include/occlum_pal_api.h delete mode 100644 src/pal/pal.c create mode 100644 src/pal/pal.lds rename src/pal/{ => src/ocalls}/attestation.c (83%) create mode 100644 src/pal/src/ocalls/fs.c create mode 100644 src/pal/src/ocalls/net.c create mode 100644 src/pal/src/ocalls/ocalls.h create mode 100644 src/pal/src/ocalls/sched.c create mode 100644 src/pal/src/ocalls/spawn.c create mode 100644 src/pal/src/ocalls/time.c create mode 100644 src/pal/src/pal_api.c create mode 100644 src/pal/src/pal_enclave.c create mode 100644 src/pal/src/pal_enclave.h create mode 100644 src/pal/src/pal_error.c create mode 100644 src/pal/src/pal_error.h create mode 100644 src/pal/src/pal_log.h create mode 100644 src/pal/src/pal_syscall.h delete mode 100644 src/pal/task.c delete mode 100644 src/pal/task.h create mode 100644 src/run/Makefile create mode 100644 src/run/main.c diff --git a/Makefile b/Makefile index b0188dcd..b81ff8fb 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,8 @@ install: install -t $(OCCLUM_PREFIX)/src/libos/ -m 444 src/libos/Makefile src/libos/Enclave.lds install -d $(OCCLUM_PREFIX)/src/libos/src/builtin/ install -t $(OCCLUM_PREFIX)/src/libos/src/builtin/ -m 444 src/libos/src/builtin/* + install -d $(OCCLUM_PREFIX)/include/ + install -t $(OCCLUM_PREFIX)/include/ -m 444 src/pal/include/* install -d $(OCCLUM_PREFIX)/etc/template/ install -t $(OCCLUM_PREFIX)/etc/template/ -m 444 etc/template/* diff --git a/src/Enclave.edl b/src/Enclave.edl index c1dae23b..73a67ca1 100644 --- a/src/Enclave.edl +++ b/src/Enclave.edl @@ -9,28 +9,61 @@ enclave { include "sgx_quote.h" trusted { - /* define ECALLs here. */ - public int libos_boot([in, string] const char* executable_path, [user_check] const char** argv); - public int libos_run(int host_tid); - /* This is only for debug usage */ - public int dummy_ecall(void); + /* + * Create a new LibOS process to do the task specified by the given + * arguments. + * + * @retval On success, return the thread ID of the + * newly-created process (pid == tid for a new process). On error, + * return -1. The user can check errno for the concrete error type. + */ + public int occlum_ecall_new_process( + [in, string] const char* executable_path, + [user_check] const char** argv); + + /* + * Execute the LibOS thread specified by the TID. + * + * This API is synchronous: it returns until the LibOS thread exits. + * + * @retval On success, return the exit status of the thread. On error, + * return -1. The user can check errno for the concrete error type. + */ + public int occlum_ecall_exec_thread(int libos_tid, int host_tid); + + /* + * A do-nothing ECall for debug purpose only. + */ + public void occlum_ecall_nop(void); }; untrusted { - void ocall_print_string([in, string] const char* msg); - int ocall_run_new_task(void); - void ocall_gettimeofday([out] long* sec, [out] long* us); - void ocall_clock_gettime(int clockid, [out] long* sec, [out] long* ns); - void ocall_nanosleep(long sec, long nsec); - void ocall_sync(void); - void ocall_sched_yield(void); - int ocall_sched_getaffinity([out] int *error, int pid, size_t cpusize, [out, size=cpusize] unsigned char* buf); - int ocall_sched_setaffinity([out] int *error, int pid, size_t cpusize, [in, size=cpusize] const unsigned char* buf); + /* + * Execute the LibOS thread specified by a TID in a new host OS thread. + * + * This API is asynchronous: it immediately returns after successfully + * creating a new host OS thread that will enter the enclave and execute the + * the LibOS thread (using occlum_ecall_exec_process). + * + * @retval On success, return 0. On error, return -1. + */ + int occlum_ocall_exec_thread_async(int libos_tid); - sgx_status_t ocall_sgx_init_quote( - [out] sgx_target_info_t *target_info, - [out] sgx_epid_group_id_t *epid_group_id); - sgx_status_t ocall_sgx_get_quote( + void occlum_ocall_gettimeofday([out] long* sec, [out] long* us); + void occlum_ocall_clock_gettime(int clockid, [out] long* sec, [out] long* ns); + + void occlum_ocall_nanosleep(long sec, long nsec); + + void occlum_ocall_sync(void); + + void occlum_ocall_sched_yield(void); + int occlum_ocall_sched_getaffinity([out] int *error, int pid, size_t cpusize, [out, size=cpusize] unsigned char* buf); + int occlum_ocall_sched_setaffinity([out] int *error, int pid, size_t cpusize, [in, size=cpusize] const unsigned char* buf); + + sgx_status_t occlum_ocall_sgx_init_quote( + [out] sgx_target_info_t* target_info, + [out] sgx_epid_group_id_t* epid_group_id); + sgx_status_t occlum_ocall_sgx_get_quote( [in, size=sigrl_len] uint8_t* sigrl, uint32_t sigrl_len, [in] sgx_report_t* report, @@ -38,10 +71,10 @@ enclave { [in] sgx_spid_t* spid, [in] sgx_quote_nonce_t* nonce, [out] sgx_report_t* qe_report, - [out, size=quote_buf_len] uint8_t* quote_buf, + [out, size=quote_buf_len] sgx_quote_t* quote_buf, uint32_t quote_buf_len); - int64_t ocall_sendmsg( + int64_t occlum_ocall_sendmsg( int sockfd, [in, size=msg_namelen] const void* msg_name, socklen_t msg_namelen, @@ -51,8 +84,7 @@ enclave { size_t msg_controllen, int flags ) propagate_errno; - - int64_t ocall_recvmsg( + int64_t occlum_ocall_recvmsg( int sockfd, [out, size=msg_namelen] void *msg_name, socklen_t msg_namelen, diff --git a/src/Makefile b/src/Makefile index 61124563..dfea9892 100644 --- a/src/Makefile +++ b/src/Makefile @@ -3,7 +3,9 @@ all: @$(MAKE) --no-print-directory -C libos @$(MAKE) --no-print-directory -C pal + @$(MAKE) --no-print-directory -C run clean: @$(MAKE) --no-print-directory -C libos clean @$(MAKE) --no-print-directory -C pal clean + @$(MAKE) --no-print-directory -C run clean diff --git a/src/libos/Cargo.toml b/src/libos/Cargo.toml index 6a4bb087..40ec8ccf 100644 --- a/src/libos/Cargo.toml +++ b/src/libos/Cargo.toml @@ -3,7 +3,7 @@ name = "Occlum" version = "0.8.0" [lib] -name = "occlum_rs" +name = "occlum_libos_core_rs" crate-type = ["staticlib"] [dependencies] diff --git a/src/libos/Makefile b/src/libos/Makefile index b4755cc0..f2514dfe 100644 --- a/src/libos/Makefile +++ b/src/libos/Makefile @@ -1,5 +1,38 @@ include ../sgxenv.mk +# An overview of the build process +# +# The target library: +# libocclum-libos.so +# +# The intermediate libraries: +# libocclum-libos-core.a +# libocclum_libos_core_rs.a +# +# Diagram: +# +# +-------------------+ +# |LibOS (Enclave)<--------+ libocclum-libos.so +# | +---------------| | +# | |LibOS Core <----------+ libocclum-libos-core.a +# | | +----------| | | +# | | |Rust <-------------+ libocclum_libos_core_rs.a +# | | +----------+ | | (Rust forbids the use of hypens in library names) +# | | |C | | | +# | | +----------+ | | +# | | |Assembly | | | +# | | +----------+ | | +# | +---------------+ | +# | |LibOS Builtins<-------+ Configurable by Occlum.json +# | +---------------+ | +# | | +# | +---------------+ | +# | |Rust SGX SDK <-----+ +# | +---------------| | +--+ Dependencies +# | |Intel SGX SDK<-----+ +# | +---------------| | +# +-------------------+ + # Build LibOS in debug or release mode LIBOS_RELEASE ?= 0 @@ -19,13 +52,9 @@ LIBOS_LOG ?= error ONLY_REBUILD_BUILTIN ?= 0 -# The final and intermediate libraries -# -# The relationships between the three Occlum libraries -# $(LIBOS_SO) > $(LIBOS_CORE_A) > $(LIBOS_RS_A) -LIBOS_SO := $(BUILD_DIR)/lib/libocclum.so -LIBOS_CORE_A := $(BUILD_DIR)/lib/libocclum_core.a -LIBOS_RS_A := $(BUILD_DIR)/lib/libocclum_rs.a +LIBOS_SO := $(BUILD_DIR)/lib/libocclum-libos.so +LIBOS_CORE_A := $(BUILD_DIR)/lib/libocclum-libos-core.a +LIBOS_CORE_RS_A := $(BUILD_DIR)/lib/libocclum_libos_core_rs.a # The dependent libraries LIBCOMPILER_RT_PATCH_A := $(BUILD_DIR)/lib/libcompiler-rt-patch.a @@ -62,7 +91,7 @@ C_FLAGS += -DOCCLUM_BUILTIN_VM_USER_SPACE_SIZE='($(OCCLUM_BUILTIN_VM_USER_SPACE_ endif _Other_Link_Flags := -L$(RUST_SGX_SDK_DIR)/compiler-rt/ -L$(BUILD_DIR)/lib -_Other_Enclave_Libs := -lcompiler-rt-patch -locclum_core -lsgx_tprotected_fs +_Other_Enclave_Libs := -lcompiler-rt-patch -locclum-libos-core -lsgx_tprotected_fs LINK_FLAGS := $(SGX_LFLAGS_T) .PHONY: all clean @@ -83,17 +112,17 @@ $(LIBOS_SO): $(BUILTIN_C_OBJS) @echo "LINK => $@" endif -$(LIBOS_CORE_A): $(LIBOS_RS_A) $(C_OBJS) $(CXX_OBJS) $(S_OBJS) $(EDL_C_OBJS) - @cp $(LIBOS_RS_A) $(LIBOS_CORE_A) +$(LIBOS_CORE_A): $(LIBOS_CORE_RS_A) $(C_OBJS) $(CXX_OBJS) $(S_OBJS) $(EDL_C_OBJS) + @cp $(LIBOS_CORE_RS_A) $(LIBOS_CORE_A) @ar r $@ $(C_OBJS) $(CXX_OBJS) $(S_OBJS) $(EDL_C_OBJS) @echo "GEN => $@" ifeq ($(LIBOS_RELEASE), 0) -$(LIBOS_RS_A): $(RUST_SRCS) +$(LIBOS_CORE_RS_A): $(RUST_SRCS) @RUSTC_BOOTSTRAP=1 cargo build --target-dir=$(RUST_TARGET_DIR) -Z unstable-options --out-dir=$(RUST_OUT_DIR) @echo "CARGO (debug) => $@" else -$(LIBOS_RS_A): $(RUST_SRCS) +$(LIBOS_CORE_RS_A): $(RUST_SRCS) @RUSTC_BOOTSTRAP=1 cargo build --release --target-dir=$(RUST_TARGET_DIR) -Z unstable-options --out-dir=$(RUST_OUT_DIR) @echo "CARGO (release) => $@" endif @@ -127,6 +156,6 @@ clean-builtin: @-$(RM) -f $(BUILTIN_C_OBJS) clean: clean-builtin - @-$(RM) -f $(LIBOS_SO) $(LIBOS_CORE_A) $(LIBOS_RS_A) \ + @-$(RM) -f $(LIBOS_SO) $(LIBOS_CORE_A) $(LIBOS_CORE_RS_A) \ $(LIBCOMPILER_RT_PATCH_A) $(EDL_C_OBJS) $(EDL_C_SRCS) $(C_OBJS) $(CXX_OBJS) $(S_OBJS) @-$(RM) -rf $(RUST_TARGET_DIR) diff --git a/src/libos/src/entry.rs b/src/libos/src/entry.rs index b2c72840..97affd2b 100644 --- a/src/libos/src/entry.rs +++ b/src/libos/src/entry.rs @@ -7,7 +7,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Once; use util::mem_util::from_untrusted::*; -const ENCLAVE_PATH: &'static str = ".occlum/build/lib/libocclum.signed.so"; +const ENCLAVE_PATH: &'static str = ".occlum/build/lib/libocclum-libos.signed.so"; lazy_static! { static ref INIT_ONCE: Once = Once::new(); @@ -15,7 +15,10 @@ lazy_static! { } #[no_mangle] -pub extern "C" fn libos_boot(path_buf: *const c_char, argv: *const *const c_char) -> i32 { +pub extern "C" fn occlum_ecall_new_process( + path_buf: *const c_char, + argv: *const *const c_char, +) -> i32 { INIT_ONCE.call_once(|| { // Init the log infrastructure first so that log messages will be printed afterwards util::log::init(); @@ -37,8 +40,8 @@ pub extern "C" fn libos_boot(path_buf: *const c_char, argv: *const *const c_char // register exception handlers (support cpuid & rdtsc for now) let _ = backtrace::enable_backtrace(ENCLAVE_PATH, PrintFormat::Short); panic::catch_unwind(|| { - backtrace::__rust_begin_short_backtrace(|| match do_boot(&path, &args) { - Ok(()) => 0, + backtrace::__rust_begin_short_backtrace(|| match do_new_process(&path, &args) { + Ok(pid_t) => pid_t as i32, Err(e) => { error!("failed to boot up LibOS: {}", e.backtrace()); EXIT_STATUS_INTERNAL_ERROR @@ -49,18 +52,20 @@ pub extern "C" fn libos_boot(path_buf: *const c_char, argv: *const *const c_char } #[no_mangle] -pub extern "C" fn libos_run(host_tid: i32) -> i32 { +pub extern "C" fn occlum_ecall_exec_thread(libos_pid: i32, host_tid: i32) -> i32 { if ALLOW_RUN.load(Ordering::SeqCst) == false { return EXIT_STATUS_INTERNAL_ERROR; } let _ = backtrace::enable_backtrace(ENCLAVE_PATH, PrintFormat::Short); panic::catch_unwind(|| { - backtrace::__rust_begin_short_backtrace(|| match do_run(host_tid as pid_t) { - Ok(exit_status) => exit_status, - Err(e) => { - error!("failed to execute a process: {}", e.backtrace()); - EXIT_STATUS_INTERNAL_ERROR + backtrace::__rust_begin_short_backtrace(|| { + match do_exec_thread(libos_pid as pid_t, host_tid as pid_t) { + Ok(exit_status) => exit_status, + Err(e) => { + error!("failed to execute a process: {}", e.backtrace()); + EXIT_STATUS_INTERNAL_ERROR + } } }) }) @@ -68,9 +73,7 @@ pub extern "C" fn libos_run(host_tid: i32) -> i32 { } #[no_mangle] -pub extern "C" fn dummy_ecall() -> i32 { - 0 -} +pub extern "C" fn occlum_ecall_nop() {} // Use -128 as a special value to indicate internal error from libos, not from // user programs. The LibOS ensures that an user program can only return a @@ -103,19 +106,20 @@ fn parse_arguments( Ok((path_buf, args)) } -fn do_boot(program_path: &PathBuf, argv: &Vec) -> Result<()> { +fn do_new_process(program_path: &PathBuf, argv: &Vec) -> Result { validate_program_path(program_path)?; let envp = &config::LIBOS_CONFIG.env; let file_actions = Vec::new(); let parent = &process::IDLE_PROCESS; let program_path_str = program_path.to_str().unwrap(); - process::do_spawn(&program_path_str, argv, envp, &file_actions, parent)?; - Ok(()) + let new_tid = + process::do_spawn_without_exec(&program_path_str, argv, envp, &file_actions, parent)?; + Ok(new_tid) } -fn do_run(host_tid: pid_t) -> Result { - let exit_status = process::run_task(host_tid)?; +fn do_exec_thread(libos_tid: pid_t, host_tid: pid_t) -> Result { + let exit_status = process::run_task(libos_tid, host_tid)?; // sync file system // TODO: only sync when all processes exit diff --git a/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs b/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs index 4077f7a3..a2dd06ea 100644 --- a/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs +++ b/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs @@ -56,7 +56,7 @@ impl InnerAgent { fn init_fields() -> Result<(sgx_target_info_t, sgx_epid_group_id_t)> { extern "C" { - pub fn ocall_sgx_init_quote( + pub fn occlum_ocall_sgx_init_quote( retval: *mut sgx_status_t, target_info: *mut sgx_target_info_t, epid_group_id: *mut sgx_epid_group_id_t, @@ -67,7 +67,7 @@ impl InnerAgent { let mut epid_group_id = Default::default(); unsafe { let mut retval = Default::default(); - let status = ocall_sgx_init_quote( + let status = occlum_ocall_sgx_init_quote( &mut retval as *mut sgx_status_t, &mut target_info as *mut sgx_target_info_t, &mut epid_group_id as *mut sgx_epid_group_id_t, @@ -75,7 +75,7 @@ impl InnerAgent { assert!(status == sgx_status_t::SGX_SUCCESS); if (retval != sgx_status_t::SGX_SUCCESS) { - return_errno!(EINVAL, "ocall_sgx_init_quote failed"); + return_errno!(EINVAL, "occlum_ocall_sgx_init_quote failed"); } } @@ -95,7 +95,7 @@ impl InnerAgent { nonce: &sgx_quote_nonce_t, ) -> Result { extern "C" { - pub fn ocall_sgx_get_quote( + pub fn occlum_ocall_sgx_get_quote( retval: *mut sgx_status_t, // Output sigrl: *const u8, // Input (optional) sigrl_len: u32, // Input (optional) @@ -137,7 +137,7 @@ impl InnerAgent { // Do OCall unsafe { let mut retval = Default::default(); - let status = ocall_sgx_get_quote( + let status = occlum_ocall_sgx_get_quote( &mut retval as *mut sgx_status_t, sigrl_ptr, sigrl_size, @@ -154,9 +154,9 @@ impl InnerAgent { if retval != sgx_status_t::SGX_SUCCESS { match retval { sgx_status_t::SGX_ERROR_BUSY => { - return_errno!(EBUSY, "ocall_sgx_get_quote is temporarily busy") + return_errno!(EBUSY, "occlum_ocall_sgx_get_quote is temporarily busy") } - _ => return_errno!(EINVAL, "ocall_sgx_get_quote failed"), + _ => return_errno!(EINVAL, "occlum_ocall_sgx_get_quote failed"), } } } diff --git a/src/libos/src/fs/mod.rs b/src/libos/src/fs/mod.rs index 27be5bfe..c776eb47 100644 --- a/src/libos/src/fs/mod.rs +++ b/src/libos/src/fs/mod.rs @@ -436,7 +436,7 @@ pub fn do_sendfile( } extern "C" { - fn ocall_sync() -> sgx_status_t; + fn occlum_ocall_sync() -> sgx_status_t; } impl Process { diff --git a/src/libos/src/lib.rs b/src/libos/src/lib.rs index c2a953be..031aa7ef 100644 --- a/src/libos/src/lib.rs +++ b/src/libos/src/lib.rs @@ -1,5 +1,5 @@ #![allow(unused)] -#![crate_name = "occlum_rs"] +#![crate_name = "occlum_libos_core_rs"] #![crate_type = "staticlib"] #![cfg_attr(not(target_env = "sgx"), no_std)] #![cfg_attr(target_env = "sgx", feature(rustc_private))] diff --git a/src/libos/src/net/socket_file/recv.rs b/src/libos/src/net/socket_file/recv.rs index 23d7e6b9..064f7f2b 100644 --- a/src/libos/src/net/socket_file/recv.rs +++ b/src/libos/src/net/socket_file/recv.rs @@ -75,7 +75,7 @@ impl SocketFile { // Do OCall let retval = try_libc!({ let mut retval = 0_isize; - let status = ocall_recvmsg( + let status = occlum_ocall_recvmsg( &mut retval as *mut isize, host_fd, msg_name, @@ -121,7 +121,7 @@ impl SocketFile { } extern "C" { - fn ocall_recvmsg( + fn occlum_ocall_recvmsg( ret: *mut ssize_t, fd: c_int, msg_name: *mut c_void, diff --git a/src/libos/src/net/socket_file/send.rs b/src/libos/src/net/socket_file/send.rs index a47c4a05..dfd3acf7 100644 --- a/src/libos/src/net/socket_file/send.rs +++ b/src/libos/src/net/socket_file/send.rs @@ -49,7 +49,7 @@ impl SocketFile { let flags = flags.to_u32() as i32; // Do OCall - let status = ocall_sendmsg( + let status = occlum_ocall_sendmsg( &mut retval as *mut isize, host_fd, msg_name, @@ -70,7 +70,7 @@ impl SocketFile { } extern "C" { - fn ocall_sendmsg( + fn occlum_ocall_sendmsg( ret: *mut ssize_t, fd: c_int, msg_name: *const c_void, diff --git a/src/libos/src/process/mod.rs b/src/libos/src/process/mod.rs index 572340c8..953f1ca8 100644 --- a/src/libos/src/process/mod.rs +++ b/src/libos/src/process/mod.rs @@ -6,7 +6,7 @@ pub use self::futex::{ pub use self::process::{Status, IDLE_PROCESS}; pub use self::process_table::get; pub use self::sched::{do_sched_getaffinity, do_sched_setaffinity, do_sched_yield, CpuSet}; -pub use self::spawn::{do_spawn, ElfFile, FileAction, ProgramHeaderExt}; +pub use self::spawn::{do_spawn, do_spawn_without_exec, ElfFile, FileAction, ProgramHeaderExt}; pub use self::task::{current_pid, get_current, run_task}; pub use self::thread::{do_clone, do_set_tid_address, CloneFlags, ThreadGroup}; pub use self::wait::{WaitQueue, Waiter}; diff --git a/src/libos/src/process/sched.rs b/src/libos/src/process/sched.rs index dbbf8986..5125b999 100644 --- a/src/libos/src/process/sched.rs +++ b/src/libos/src/process/sched.rs @@ -1,21 +1,21 @@ use super::*; extern "C" { - fn ocall_sched_getaffinity( + fn occlum_ocall_sched_getaffinity( ret: *mut i32, errno: *mut i32, pid: i32, cpusetsize: size_t, mask: *mut c_uchar, ) -> sgx_status_t; - fn ocall_sched_setaffinity( + fn occlum_ocall_sched_setaffinity( ret: *mut i32, errno: *mut i32, pid: i32, cpusetsize: size_t, mask: *const c_uchar, ) -> sgx_status_t; - fn ocall_sched_yield() -> sgx_status_t; + fn occlum_ocall_sched_yield() -> sgx_status_t; } pub struct CpuSet { @@ -96,11 +96,11 @@ pub fn do_sched_getaffinity(pid: pid_t, cpu_set: &mut CpuSet) -> Result { let mut ret = 0; let mut error = 0; unsafe { - ocall_sched_getaffinity(&mut ret, &mut error, host_tid as i32, cpusize, buf); + occlum_ocall_sched_getaffinity(&mut ret, &mut error, host_tid as i32, cpusize, buf); } if (ret < 0) { let errno = Errno::from(error as u32); - return_errno!(errno, "ocall_sched_getaffinity failed"); + return_errno!(errno, "occlum_ocall_sched_getaffinity failed"); } Ok(ret) } @@ -115,18 +115,18 @@ pub fn do_sched_setaffinity(pid: pid_t, cpu_set: &CpuSet) -> Result { let mut ret = 0; let mut error = 0; unsafe { - ocall_sched_setaffinity(&mut ret, &mut error, host_tid as i32, cpusize, buf); + occlum_ocall_sched_setaffinity(&mut ret, &mut error, host_tid as i32, cpusize, buf); } if (ret < 0) { let errno = Errno::from(error as u32); - return_errno!(errno, "ocall_sched_setaffinity failed"); + return_errno!(errno, "occlum_ocall_sched_setaffinity failed"); } Ok(ret) } pub fn do_sched_yield() { unsafe { - let status = ocall_sched_yield(); + let status = occlum_ocall_sched_yield(); assert!(status == sgx_status_t::SGX_SUCCESS); } } diff --git a/src/libos/src/process/spawn/mod.rs b/src/libos/src/process/spawn/mod.rs index 999ac1a0..a7d045de 100644 --- a/src/libos/src/process/spawn/mod.rs +++ b/src/libos/src/process/spawn/mod.rs @@ -17,27 +17,37 @@ mod elf_file; mod init_stack; mod init_vm; -#[derive(Debug)] -pub enum FileAction { - /// open(path, oflag, mode) had been called, and the returned file - /// descriptor, if not `fd`, had been changed to `fd`. - Open { - path: String, - mode: u32, - oflag: u32, - fd: FileDesc, - }, - Dup2(FileDesc, FileDesc), - Close(FileDesc), -} - pub fn do_spawn( elf_path: &str, argv: &[CString], envp: &[CString], file_actions: &[FileAction], parent_ref: &ProcessRef, -) -> Result { +) -> Result { + let (new_tid, new_process_ref) = new_process(elf_path, argv, envp, file_actions, parent_ref)?; + task::enqueue_and_exec_task(new_tid, new_process_ref); + Ok(new_tid) +} + +pub fn do_spawn_without_exec( + elf_path: &str, + argv: &[CString], + envp: &[CString], + file_actions: &[FileAction], + parent_ref: &ProcessRef, +) -> Result { + let (new_tid, new_process_ref) = new_process(elf_path, argv, envp, file_actions, parent_ref)?; + task::enqueue_task(new_tid, new_process_ref); + Ok(new_tid) +} + +fn new_process( + elf_path: &str, + argv: &[CString], + envp: &[CString], + file_actions: &[FileAction], + parent_ref: &ProcessRef, +) -> Result<(pid_t, ProcessRef)> { let elf_buf = load_elf_to_vec(elf_path, parent_ref) .cause_err(|e| errno!(e.errno(), "cannot load the executable"))?; let ldso_path = "/lib/ld-musl-x86_64.so.1"; @@ -86,8 +96,22 @@ pub fn do_spawn( }; parent_adopts_new_child(&parent_ref, &new_process_ref); process_table::put(new_pid, new_process_ref.clone()); - task::enqueue_task(new_process_ref); - Ok(new_pid) + let new_tid = new_pid; + Ok((new_tid, new_process_ref)) +} + +#[derive(Debug)] +pub enum FileAction { + /// open(path, oflag, mode) had been called, and the returned file + /// descriptor, if not `fd`, had been changed to `fd`. + Open { + path: String, + mode: u32, + oflag: u32, + fd: FileDesc, + }, + Dup2(FileDesc, FileDesc), + Close(FileDesc), } fn load_elf_to_vec(elf_path: &str, parent_ref: &ProcessRef) -> Result> { diff --git a/src/libos/src/process/task.rs b/src/libos/src/process/task.rs index 16834deb..f6b3379a 100644 --- a/src/libos/src/process/task.rs +++ b/src/libos/src/process/task.rs @@ -53,27 +53,39 @@ impl Task { } lazy_static! { - static ref NEW_PROCESS_QUEUE: SgxMutex> = - { SgxMutex::new(VecDeque::new()) }; + static ref NEW_PROCESS_TABLE: SgxMutex> = + { SgxMutex::new(HashMap::new()) }; } -pub fn enqueue_task(new_process: ProcessRef) { - NEW_PROCESS_QUEUE.lock().unwrap().push_back(new_process); +pub fn enqueue_task(new_tid: pid_t, new_process: ProcessRef) { + let existing_task = NEW_PROCESS_TABLE + .lock() + .unwrap() + .insert(new_tid, new_process); + // There should NOT have any pending process with the same ID + assert!(existing_task.is_none()); +} + +pub fn enqueue_and_exec_task(new_tid: pid_t, new_process: ProcessRef) { + enqueue_task(new_tid, new_process); let mut ret = 0; - let ocall_status = unsafe { ocall_run_new_task(&mut ret) }; + let ocall_status = unsafe { occlum_ocall_exec_thread_async(&mut ret, new_tid) }; if ocall_status != sgx_status_t::SGX_SUCCESS || ret != 0 { panic!("Failed to start the process"); } } -fn dequeue_task() -> Option { - NEW_PROCESS_QUEUE.lock().unwrap().pop_front() +fn dequeue_task(libos_tid: pid_t) -> Result { + NEW_PROCESS_TABLE + .lock() + .unwrap() + .remove(&libos_tid) + .ok_or_else(|| errno!(EAGAIN, "the given TID does not match any pending process")) } -pub fn run_task(host_tid: pid_t) -> Result { - let new_process: ProcessRef = - dequeue_task().ok_or_else(|| errno!(EAGAIN, "no new processes to run"))?; +pub fn run_task(libos_tid: pid_t, host_tid: pid_t) -> Result { + let new_process: ProcessRef = dequeue_task(libos_tid)?; set_current(&new_process); let (pid, task) = { @@ -148,6 +160,6 @@ fn reset_current() { } extern "C" { - fn ocall_run_new_task(ret: *mut i32) -> sgx_status_t; + fn occlum_ocall_exec_thread_async(ret: *mut i32, libos_tid: pid_t) -> sgx_status_t; fn do_run_task(task: *mut Task) -> i32; } diff --git a/src/libos/src/process/thread.rs b/src/libos/src/process/thread.rs index e4eaf120..42ad1295 100644 --- a/src/libos/src/process/thread.rs +++ b/src/libos/src/process/thread.rs @@ -106,7 +106,7 @@ pub fn do_clone( } } - task::enqueue_task(new_thread_ref); + task::enqueue_and_exec_task(new_thread_pid, new_thread_ref); Ok(new_thread_pid) } diff --git a/src/libos/src/time/mod.rs b/src/libos/src/time/mod.rs index 901ad7db..3ec36e20 100644 --- a/src/libos/src/time/mod.rs +++ b/src/libos/src/time/mod.rs @@ -26,13 +26,13 @@ impl timeval_t { pub fn do_gettimeofday() -> timeval_t { let mut tv: timeval_t = Default::default(); unsafe { - ocall_gettimeofday(&mut tv.sec as *mut time_t, &mut tv.usec as *mut suseconds_t); + occlum_ocall_gettimeofday(&mut tv.sec as *mut time_t, &mut tv.usec as *mut suseconds_t); } tv } extern "C" { - fn ocall_gettimeofday(sec: *mut time_t, usec: *mut suseconds_t) -> sgx_status_t; + fn occlum_ocall_gettimeofday(sec: *mut time_t, usec: *mut suseconds_t) -> sgx_status_t; } #[repr(C)] @@ -88,13 +88,17 @@ impl ClockID { pub fn do_clock_gettime(clockid: ClockID) -> Result { extern "C" { - fn ocall_clock_gettime(clockid: clockid_t, sec: *mut time_t, ns: *mut i64) -> sgx_status_t; + fn occlum_ocall_clock_gettime( + clockid: clockid_t, + sec: *mut time_t, + ns: *mut i64, + ) -> sgx_status_t; } let mut sec = 0; let mut nsec = 0; unsafe { - ocall_clock_gettime( + occlum_ocall_clock_gettime( clockid as clockid_t, &mut sec as *mut time_t, &mut nsec as *mut i64, @@ -105,10 +109,10 @@ pub fn do_clock_gettime(clockid: ClockID) -> Result { pub fn do_nanosleep(req: ×pec_t) -> Result<()> { extern "C" { - fn ocall_nanosleep(sec: time_t, nsec: i64) -> sgx_status_t; + fn occlum_ocall_nanosleep(sec: time_t, nsec: i64) -> sgx_status_t; } unsafe { - ocall_nanosleep(req.sec, req.nsec); + occlum_ocall_nanosleep(req.sec, req.nsec); } Ok(()) } diff --git a/src/pal/Makefile b/src/pal/Makefile index 77119979..2c0d411b 100644 --- a/src/pal/Makefile +++ b/src/pal/Makefile @@ -1,41 +1,42 @@ include ../sgxenv.mk -BIN := $(BUILD_DIR)/bin/occlum-pal +LIBOCCLUM_PAL_SO := $(BUILD_DIR)/lib/libocclum-pal.so # A dependency on Rust SGX SDK LIBSGX_USTDC_A := $(BUILD_DIR)/lib/libsgx_ustdc.a -EDL_C_SRCS := $(addprefix $(BUILD_DIR)/src/pal/,Enclave_u.c Enclave_u.h) -EDL_C_OBJS := $(addprefix $(BUILD_DIR)/src/pal/,Enclave_u.o) -C_SRCS := $(sort $(wildcard *.c */*.c)) +EDL_C_SRCS := $(addprefix $(BUILD_DIR)/src/pal/src/,Enclave_u.c Enclave_u.h) +EDL_C_OBJS := $(addprefix $(BUILD_DIR)/src/pal/src/,Enclave_u.o) +C_SRCS := $(sort $(wildcard src/*.c src/*/*.c)) C_OBJS := $(addprefix $(BUILD_DIR)/src/pal/,$(C_SRCS:.c=.o)) -CXX_SRCS := $(sort $(wildcard *.cpp */*.cpp)) +CXX_SRCS := $(sort $(wildcard src/*.cpp src/*/*.cpp)) CXX_OBJS := $(addprefix $(BUILD_DIR)/src/pal/,$(CXX_SRCS:.cpp=.o)) -C_COMMON_FLAGS := -I$(BUILD_DIR)/src/pal -Wno-unused-result +C_COMMON_FLAGS := -I$(BUILD_DIR)/src/pal/src -Iinclude C_FLAGS := $(C_COMMON_FLAGS) $(SGX_CFLAGS_U) CXX_FLAGS := $(C_COMMON_FLAGS) $(SGX_CXXFLAGS_U) -LINK_FLAGS := $(SGX_LFLAGS_U) -L$(RUST_SGX_SDK_DIR)/sgx_ustdc/ -lsgx_ustdc -lsgx_uprotected_fs +LINK_FLAGS := $(SGX_LFLAGS_U) -shared -L$(RUST_SGX_SDK_DIR)/sgx_ustdc/ -lsgx_ustdc -lsgx_uprotected_fs +LINK_FLAGS += -Wl,--version-script=pal.lds -ALL_BUILD_SUBDIRS := $(sort $(patsubst %/,%,$(dir $(BIN) $(EDL_C_OBJS) $(C_OBJS) $(CXX_OBJS)))) +ALL_BUILD_SUBDIRS := $(sort $(patsubst %/,%,$(dir $(LIBOCCLUM_PAL_SO) $(EDL_C_OBJS) $(C_OBJS) $(CXX_OBJS)))) .PHONY: all clean -all: $(ALL_BUILD_SUBDIRS) $(BIN) +all: $(ALL_BUILD_SUBDIRS) $(LIBOCCLUM_PAL_SO) $(ALL_BUILD_SUBDIRS): @mkdir -p $@ -$(BIN): $(LIBSGX_USTDC_A) $(EDL_C_OBJS) $(C_OBJS) $(CXX_OBJS) +$(LIBOCCLUM_PAL_SO): $(LIBSGX_USTDC_A) $(EDL_C_OBJS) $(C_OBJS) $(CXX_OBJS) @$(CXX) $^ -o $@ $(LINK_FLAGS) @echo "LINK => $@" -$(BUILD_DIR)/src/pal/Enclave_u.o: $(BUILD_DIR)/src/pal/Enclave_u.c +$(BUILD_DIR)/src/pal/src/Enclave_u.o: $(BUILD_DIR)/src/pal/src/Enclave_u.c @$(CC) $(C_FLAGS) -c $< -o $@ @echo "CC <= $@" -$(BUILD_DIR)/src/pal/Enclave_u.c: $(SGX_EDGER8R) ../Enclave.edl - @cd $(BUILD_DIR)/src/pal && $(SGX_EDGER8R) --untrusted $(CUR_DIR)/../Enclave.edl --search-path $(SGX_SDK)/include --search-path $(RUST_SGX_SDK_DIR)/edl/ +$(BUILD_DIR)/src/pal/src/Enclave_u.c: $(SGX_EDGER8R) ../Enclave.edl + @cd $(BUILD_DIR)/src/pal/src && $(SGX_EDGER8R) --untrusted $(CUR_DIR)/../Enclave.edl --search-path $(SGX_SDK)/include --search-path $(RUST_SGX_SDK_DIR)/edl/ @echo "GEN <= $@" $(BUILD_DIR)/src/pal/%.o: %.c @@ -52,4 +53,4 @@ $(LIBSGX_USTDC_A): @echo "GEN <= $@" clean: - @-$(RM) -f $(BIN) $(LIBSGX_USTDC_A) $(C_OBJS) $(CXX_OBJS) $(EDL_C_OBJS) $(EDL_C_SRCS) + @-$(RM) -f $(LIBOCCLUM_PAL_SO) $(LIBSGX_USTDC_A) $(C_OBJS) $(CXX_OBJS) $(EDL_C_OBJS) $(EDL_C_SRCS) diff --git a/src/pal/atomic.h b/src/pal/atomic.h deleted file mode 100644 index abd96973..00000000 --- a/src/pal/atomic.h +++ /dev/null @@ -1,16 +0,0 @@ -#ifndef __ATOMIC_H_ -#define __ATOMIC_H_ - -static inline int a_load(volatile int* n) { - return *(volatile int*)n; -} - -static inline void a_store(volatile int* n, int x) { - *n = x; -} - -static inline int a_fetch_and_add(volatile int* n, int a) { - return __sync_fetch_and_add(n, a); -} - -#endif /* __ATOMIC_H_ */ diff --git a/src/pal/futex.c b/src/pal/futex.c deleted file mode 100644 index 7c74577b..00000000 --- a/src/pal/futex.c +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include -#include -#include - -static inline long __syscall6(long n, long a1, long a2, long a3, long a4, long a5, long a6) -{ - unsigned long ret; - register long r10 __asm__("r10") = a4; - register long r8 __asm__("r8") = a5; - register long r9 __asm__("r9") = a6; - __asm__ __volatile__ ("syscall" : "=a"(ret) : "a"(n), "D"(a1), "S"(a2), - "d"(a3), "r"(r10), "r"(r8), "r"(r9) : "rcx", "r11", "memory"); - return ret; -} - -#define syscall(num, a1, a2, a3, a4, a5, a6) \ - __syscall6((num), (long)(a1), (long)(a2), (long)(a3), (long)(a4), (long)(a5), (long)(a6)) - -static inline int futex(volatile void *addr1, int op, int val1, struct timespec *timeout, - void *addr2, int val3) { - return (int) syscall(SYS_futex, addr1, op, val1, timeout, addr2, val3); -} - -int futex_wait(volatile int* uaddr, int val) { - return futex(uaddr, FUTEX_WAIT, val, NULL, NULL, 0); -} - -int futex_wakeup(volatile int* uaddr) { - return futex(uaddr, FUTEX_WAKE, INT_MAX, NULL, NULL, 0); -} diff --git a/src/pal/futex.h b/src/pal/futex.h deleted file mode 100644 index da87e00c..00000000 --- a/src/pal/futex.h +++ /dev/null @@ -1,9 +0,0 @@ -#ifndef __FUTEX_H_ -#define __FUTEX_H_ - -#include - -int futex_wait(volatile int* uaddr, int val); -int futex_wakeup(volatile int* uaddr); - -#endif /* __ATOMIC_H_ */ diff --git a/src/pal/include/occlum_pal_api.h b/src/pal/include/occlum_pal_api.h new file mode 100644 index 00000000..db176f41 --- /dev/null +++ b/src/pal/include/occlum_pal_api.h @@ -0,0 +1,48 @@ +#ifndef __OCCLUM_PAL_API_H__ +#define __OCCLUM_PAL_API_H__ + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * @brief Initialize an Occlum enclave + * + * @param instance_dir Specifies the path of an Occlum instance directory. + * Usually, this directory is initialized by executing + * "occlum init" command, which creates a hidden + * directory named ".occlum/". This ".occlum/" is an + * Occlum instance directory. The name of the directory is + * not necesarrily ".occlum"; it can be renamed to an + * arbitrary name. + * + * @retval If 0, then success; otherwise, check errno for the exact error type. + */ +int occlum_pal_init(const char* instance_dir); + +/* + * @brief Execute a command inside the Occlum enclave + * + * @param cmd_path The path of the command to be executed + * @param cmd_args The arguments to the command. The array must be NULL + * terminated. + * @param exit_status Output. The exit status of the command. Note that the + * exit status is returned if and only if the function + * succeeds. + * + * @retval If 0, then success; otherwise, check errno for the exact error type. + */ +int occlum_pal_exec(const char* cmd_path, const char** cmd_args, int* exit_status); + +/* + * @brief Destroy teh Occlum enclave + * + * @retval if 0, then success; otherwise, check errno for the exact error type. + */ +int occlum_pal_destroy(void); + +#ifdef __cplusplus +} +#endif + +#endif /* __OCCLUM_PAL_API_H__ */ diff --git a/src/pal/pal.c b/src/pal/pal.c deleted file mode 100644 index c2a76cf8..00000000 --- a/src/pal/pal.c +++ /dev/null @@ -1,423 +0,0 @@ -#include "Enclave_u.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include "task.h" - -#define MAX_PATH FILENAME_MAX -#define TOKEN_FILENAME "enclave.token" -#define ENCLAVE_FILENAME "libocclum.signed.so" - -// ========================================================================== -// Enclave Initialization -// ========================================================================== - -static sgx_enclave_id_t global_eid = 0; - -typedef struct _sgx_errlist_t { - sgx_status_t err; - const char *msg; - const char *sug; /* Suggestion */ -} sgx_errlist_t; - -/* Error code returned by sgx_create_enclave */ -static sgx_errlist_t sgx_errlist[] = { - { - SGX_ERROR_UNEXPECTED, - "Unexpected error occurred.", - NULL - }, - { - SGX_ERROR_INVALID_PARAMETER, - "Invalid parameter.", - NULL - }, - { - SGX_ERROR_OUT_OF_MEMORY, - "Out of memory.", - NULL - }, - { - SGX_ERROR_ENCLAVE_LOST, - "Power transition occurred.", - "Please refer to the sample \"PowerTransition\" for details." - }, - { - SGX_ERROR_INVALID_ENCLAVE, - "Invalid enclave image.", - NULL - }, - { - SGX_ERROR_INVALID_ENCLAVE_ID, - "Invalid enclave identification.", - NULL - }, - { - SGX_ERROR_INVALID_SIGNATURE, - "Invalid enclave signature.", - NULL - }, - { - SGX_ERROR_OUT_OF_EPC, - "Out of EPC memory.", - NULL - }, - { - SGX_ERROR_NO_DEVICE, - "Invalid SGX device.", - "Please make sure SGX module is enabled in the BIOS, and install SGX driver afterwards." - }, - { - SGX_ERROR_MEMORY_MAP_CONFLICT, - "Memory map conflicted.", - NULL - }, - { - SGX_ERROR_INVALID_METADATA, - "Invalid enclave metadata.", - NULL - }, - { - SGX_ERROR_DEVICE_BUSY, - "SGX device was busy.", - NULL - }, - { - SGX_ERROR_INVALID_VERSION, - "Enclave version was invalid.", - NULL - }, - { - SGX_ERROR_INVALID_ATTRIBUTE, - "Enclave was not authorized.", - NULL - }, - { - SGX_ERROR_ENCLAVE_FILE_ACCESS, - "Can't open enclave file.", - NULL - }, - { - SGX_ERROR_SERVICE_INVALID_PRIVILEGE, - "Enclave has no privilege to get run in the release mode.", - "Please rebuild the Occlum enclave with a legal signing key " - "(e.g., occlum build --sign-key ), " - "to get a legal signing key, please contact Intel." - }, -}; - -/* Check error conditions for loading enclave */ -static void print_error_message(sgx_status_t ret) { - size_t idx = 0; - size_t ttl = sizeof sgx_errlist/sizeof sgx_errlist[0]; - - for (idx = 0; idx < ttl; idx++) { - if(ret == sgx_errlist[idx].err) { - printf("Error: %s\n", sgx_errlist[idx].msg); - if(NULL != sgx_errlist[idx].sug) - printf("Info: %s\n", sgx_errlist[idx].sug); - break; - } - } - - if (idx == ttl) - printf("Error: Unexpected error occurred.\n"); -} - -static const char* get_enclave_absolute_path() { - static char enclave_path[MAX_PATH] = {0}; - // Get the absolute path of the executable - readlink("/proc/self/exe", enclave_path, sizeof(enclave_path)); - // Get the absolute path of the containing directory - dirname(enclave_path); - // Get the absolute path of the enclave - strncat(enclave_path, "/../lib/", sizeof(enclave_path)); - strncat(enclave_path, ENCLAVE_FILENAME, sizeof(enclave_path)); - return (const char*)enclave_path; -} - -/* Get enclave debug flag according to env "OCCLUM_RELEASE_ENCLAVE" */ -static int get_enclave_debug_flag() { - const char* release_enclave_val = getenv("OCCLUM_RELEASE_ENCLAVE"); - if (release_enclave_val) { - if (!strcmp(release_enclave_val, "1") || - !strcasecmp(release_enclave_val, "y") || - !strcasecmp(release_enclave_val, "yes") || - !strcasecmp(release_enclave_val, "true")) { - return 0; - } - } - return 1; -} - -/* Initialize the enclave: - * Step 1: try to retrieve the launch token saved by last transaction - * Step 2: call sgx_create_enclave to initialize an enclave instance - * Step 3: save the launch token if it is updated - */ -static int initialize_enclave() -{ - char token_path[MAX_PATH] = {'\0'}; - sgx_launch_token_t token = {0}; - sgx_status_t ret = SGX_ERROR_UNEXPECTED; - int updated = 0; - - /* Step 1: try to retrieve the launch token saved by last transaction - * if there is no token, then create a new one. - */ - /* try to get the token saved in $HOME */ - const char *home_dir = getpwuid(getuid())->pw_dir; - - if (home_dir != NULL && - (strlen(home_dir)+strlen("/")+sizeof(TOKEN_FILENAME)+1) <= MAX_PATH) { - /* compose the token path */ - strncpy(token_path, home_dir, strlen(home_dir)); - strncat(token_path, "/", strlen("/")); - strncat(token_path, TOKEN_FILENAME, sizeof(TOKEN_FILENAME)+1); - } else { - /* if token path is too long or $HOME is NULL */ - strncpy(token_path, TOKEN_FILENAME, sizeof(TOKEN_FILENAME)); - } - - FILE *fp = fopen(token_path, "rb"); - if (fp == NULL && (fp = fopen(token_path, "wb")) == NULL) { - printf("Warning: Failed to create/open the launch token file \"%s\".\n", token_path); - } - - if (fp != NULL) { - /* read the token from saved file */ - size_t read_num = fread(token, 1, sizeof(sgx_launch_token_t), fp); - if (read_num != 0 && read_num != sizeof(sgx_launch_token_t)) { - /* if token is invalid, clear the buffer */ - memset(&token, 0x0, sizeof(sgx_launch_token_t)); - printf("Warning: Invalid launch token read from \"%s\".\n", token_path); - } - } - /* Step 2: call sgx_create_enclave to initialize an enclave instance */ - /* Debug Support: set 2nd parameter to 1 */ - const char* enclave_path = get_enclave_absolute_path(); - int sgx_debug_flag = get_enclave_debug_flag(); - ret = sgx_create_enclave(enclave_path, sgx_debug_flag, &token, &updated, &global_eid, NULL); - if (ret != SGX_SUCCESS) { - print_error_message(ret); - if (fp != NULL) fclose(fp); - return -1; - } - - /* Step 3: save the launch token if it is updated */ - if (updated == 0 || fp == NULL) { - /* if the token is not updated, or file handler is invalid, do not perform saving */ - if (fp != NULL) fclose(fp); - return 0; - } - - /* reopen the file with write capablity */ - fp = freopen(token_path, "wb", fp); - if (fp == NULL) return 0; - size_t write_num = fwrite(token, 1, sizeof(sgx_launch_token_t), fp); - if (write_num != sizeof(sgx_launch_token_t)) - printf("Warning: Failed to save launch token to \"%s\".\n", token_path); - fclose(fp); - return 0; -} - -// ========================================================================== -// OCalls -// ========================================================================== - -void ocall_print_string(const char* msg) { - printf("%s", msg); -} - -int ocall_run_new_task(void) { - int ret = run_new_task(global_eid); - return ret; -} - -void ocall_gettimeofday(long* seconds, long* microseconds) { - struct timeval tv; - gettimeofday(&tv, NULL); - *seconds = tv.tv_sec; - *microseconds = tv.tv_usec; -} - -void ocall_clock_gettime(int clockid, time_t* sec, long* ns) { - struct timespec ts; - clock_gettime(clockid, &ts); - *sec = ts.tv_sec; - *ns = ts.tv_nsec; -} - -void ocall_nanosleep(time_t sec, long nsec) { - struct timespec tv = { .tv_sec = sec, .tv_nsec = nsec }; - nanosleep(&tv, NULL); -} - -int ocall_sched_getaffinity(int* error, int pid, size_t cpusize, unsigned char* buf) { - int ret = syscall(__NR_sched_getaffinity, pid, cpusize, buf); - if (error) { - *error = (ret == -1) ? errno : 0; - } - return ret; -} - -int ocall_sched_setaffinity(int* error, int pid, size_t cpusize, const unsigned char* buf) { - int ret = syscall(__NR_sched_setaffinity, pid, cpusize, buf); - if (error) { - *error = (ret == -1) ? errno : 0; - } - return ret; -} - -/* In the Linux implementation, sched_yield() always succeeds */ -void ocall_sched_yield(void) { - sched_yield(); -} - -void ocall_sync(void) { - sync(); -} - -ssize_t ocall_sendmsg(int sockfd, - const void *msg_name, - socklen_t msg_namelen, - const void *buf, - size_t buf_len, - const void *msg_control, - size_t msg_controllen, - int flags) -{ - struct iovec msg_iov = { .iov_base = (void*)buf, .iov_len = buf_len }; - struct iovec* p_msg_iov = buf != NULL ? &msg_iov : NULL; - size_t msg_iovlen = buf != NULL ? 1 : 0; - - struct msghdr msg = { - (void*) msg_name, - msg_namelen, - p_msg_iov, - msg_iovlen, - (void*) msg_control, - msg_controllen, - 0, - }; - return sendmsg(sockfd, &msg, flags); -} - -ssize_t ocall_recvmsg(int sockfd, - void *msg_name, - socklen_t msg_namelen, - socklen_t* msg_namelen_recv, - void *buf, - size_t buf_len, - void *msg_control, - size_t msg_controllen, - size_t* msg_controllen_recv, - int* msg_flags_recv, - int flags) -{ - struct iovec msg_iov = { .iov_base = buf, .iov_len = buf_len }; - struct iovec* p_msg_iov = buf != NULL ? &msg_iov : NULL; - size_t msg_iovlen = buf != NULL ? 1 : 0; - - struct msghdr msg = { - msg_name, - msg_namelen, - p_msg_iov, - msg_iovlen, - msg_control, - msg_controllen, - 0, - }; - ssize_t ret = recvmsg(sockfd, &msg, flags); - if (ret < 0) return ret; - - *msg_namelen_recv = msg.msg_namelen; - *msg_controllen_recv = msg.msg_controllen; - *msg_flags_recv = msg.msg_flags; - return ret; -} - -// ========================================================================== -// Main -// ========================================================================== - -/* Application entry */ -int SGX_CDECL main(int argc, const char *argv[]) -{ - struct timeval startup, libosready, appdie; - - gettimeofday(&startup, NULL); - sgx_status_t sgx_ret = SGX_SUCCESS; - int status = 0; - - if (argc < 2) { - printf("ERROR: at least one argument must be provided\n\n"); - printf("Usage: pal ...\n"); - return EXIT_FAILURE; - } - const char* executable_path = argv[1]; - - /* Initialize the enclave */ - if (initialize_enclave() < 0){ - printf("Enter a character before exit ...\n"); - getchar(); - return EXIT_FAILURE; - } - - // First ecall do a lot initializations. - // Count it as startup time. - dummy_ecall(global_eid, &status); - - gettimeofday(&libosready, NULL); - - sgx_ret = libos_boot(global_eid, &status, executable_path, &argv[2]); - if (sgx_ret != SGX_SUCCESS) { - print_error_message(sgx_ret); - return EXIT_FAILURE; - } - if (status != 0) { - return EXIT_FAILURE; - } - - // TODO: exit all tasks gracefully, instead of killing all remaining - // tasks automatically after the main task exits and the process - // terminates. - //status = wait_all_tasks(); - status = wait_main_task(); - - gettimeofday(&appdie, NULL); - - uint64_t libos_startup_time, app_runtime; - libos_startup_time = (libosready.tv_sec - startup.tv_sec) * 1000000 + (libosready.tv_usec - startup.tv_usec); - app_runtime = (appdie.tv_sec - libosready.tv_sec) * 1000000 + (appdie.tv_usec - libosready.tv_usec); - printf("LibOS startup time: %lu microseconds\n", libos_startup_time); - printf("Apps running time: %lu microseconds\n", app_runtime); - - // TODO: destroy the enclave gracefully - // We cannot destroy the enclave gracefully since we may still have - // running threads that are using the enclave at this point, which blocks - // sgx_destory_enclave call. This issue is related to "TODO: exit all tasks - // gracefully" above. - //sgx_destroy_enclave(global_eid); - - return status; -} diff --git a/src/pal/pal.lds b/src/pal/pal.lds new file mode 100644 index 00000000..d3383aa2 --- /dev/null +++ b/src/pal/pal.lds @@ -0,0 +1,8 @@ +{ + global: + occlum_pal_init; + occlum_pal_exec; + occlum_pal_destroy; + local: + *; +}; diff --git a/src/pal/attestation.c b/src/pal/src/ocalls/attestation.c similarity index 83% rename from src/pal/attestation.c rename to src/pal/src/ocalls/attestation.c index f8c3952c..c602514c 100644 --- a/src/pal/attestation.c +++ b/src/pal/src/ocalls/attestation.c @@ -1,13 +1,14 @@ -#include "sgx_uae_service.h" +#include +#include "ocalls.h" -sgx_status_t ocall_sgx_init_quote( - sgx_target_info_t *target_info, - sgx_epid_group_id_t *epid_group_id) +sgx_status_t occlum_ocall_sgx_init_quote( + sgx_target_info_t* target_info, + sgx_epid_group_id_t* epid_group_id) { return sgx_init_quote(target_info, epid_group_id); } -sgx_status_t ocall_sgx_get_quote( +sgx_status_t occlum_ocall_sgx_get_quote( uint8_t* sigrl, uint32_t sigrl_len, sgx_report_t* report, diff --git a/src/pal/src/ocalls/fs.c b/src/pal/src/ocalls/fs.c new file mode 100644 index 00000000..9172d307 --- /dev/null +++ b/src/pal/src/ocalls/fs.c @@ -0,0 +1,6 @@ +#include +#include "ocalls.h" + +void occlum_ocall_sync(void) { + sync(); +} diff --git a/src/pal/src/ocalls/net.c b/src/pal/src/ocalls/net.c new file mode 100644 index 00000000..92a6e3ea --- /dev/null +++ b/src/pal/src/ocalls/net.c @@ -0,0 +1,63 @@ +#include +#include +#include +#include "ocalls.h" + +ssize_t occlum_ocall_sendmsg(int sockfd, + const void *msg_name, + socklen_t msg_namelen, + const void *buf, + size_t buf_len, + const void *msg_control, + size_t msg_controllen, + int flags) +{ + struct iovec msg_iov = { .iov_base = (void*)buf, .iov_len = buf_len }; + struct iovec* p_msg_iov = buf != NULL ? &msg_iov : NULL; + size_t msg_iovlen = buf != NULL ? 1 : 0; + + struct msghdr msg = { + (void*) msg_name, + msg_namelen, + p_msg_iov, + msg_iovlen, + (void*) msg_control, + msg_controllen, + 0, + }; + return sendmsg(sockfd, &msg, flags); +} + +ssize_t occlum_ocall_recvmsg(int sockfd, + void *msg_name, + socklen_t msg_namelen, + socklen_t* msg_namelen_recv, + void *buf, + size_t buf_len, + void *msg_control, + size_t msg_controllen, + size_t* msg_controllen_recv, + int* msg_flags_recv, + int flags) +{ + struct iovec msg_iov = { .iov_base = buf, .iov_len = buf_len }; + struct iovec* p_msg_iov = buf != NULL ? &msg_iov : NULL; + size_t msg_iovlen = buf != NULL ? 1 : 0; + + struct msghdr msg = { + msg_name, + msg_namelen, + p_msg_iov, + msg_iovlen, + msg_control, + msg_controllen, + 0, + }; + ssize_t ret = recvmsg(sockfd, &msg, flags); + if (ret < 0) return ret; + + *msg_namelen_recv = msg.msg_namelen; + *msg_controllen_recv = msg.msg_controllen; + *msg_flags_recv = msg.msg_flags; + return ret; +} diff --git a/src/pal/src/ocalls/ocalls.h b/src/pal/src/ocalls/ocalls.h new file mode 100644 index 00000000..fbc75932 --- /dev/null +++ b/src/pal/src/ocalls/ocalls.h @@ -0,0 +1,5 @@ +#include "Enclave_u.h" +#include "../pal_enclave.h" +#include "../pal_error.h" +#include "../pal_syscall.h" +#include "../pal_log.h" diff --git a/src/pal/src/ocalls/sched.c b/src/pal/src/ocalls/sched.c new file mode 100644 index 00000000..cfbfab41 --- /dev/null +++ b/src/pal/src/ocalls/sched.c @@ -0,0 +1,24 @@ +#include +#include "ocalls.h" + +int occlum_ocall_sched_getaffinity(int* error, int pid, size_t cpusize, unsigned char* buf) { + int ret = syscall(__NR_sched_getaffinity, pid, cpusize, buf); + if (error) { + *error = (ret == -1) ? errno : 0; + } + return ret; +} + +int occlum_ocall_sched_setaffinity(int* error, int pid, size_t cpusize, const unsigned char* buf) { + int ret = syscall(__NR_sched_setaffinity, pid, cpusize, buf); + if (error) { + *error = (ret == -1) ? errno : 0; + } + return ret; +} + +/* In the Linux implementation, sched_yield() always succeeds */ +void occlum_ocall_sched_yield(void) { + sched_yield(); +} + diff --git a/src/pal/src/ocalls/spawn.c b/src/pal/src/ocalls/spawn.c new file mode 100644 index 00000000..bae97801 --- /dev/null +++ b/src/pal/src/ocalls/spawn.c @@ -0,0 +1,45 @@ +#include +#include +#include "ocalls.h" + +typedef struct { + sgx_enclave_id_t enclave_id; + int libos_tid; +} thread_data_t; + +void* exec_libos_thread(void* _thread_data) { + thread_data_t* thread_data = _thread_data; + sgx_enclave_id_t eid = thread_data->enclave_id; + int host_tid = gettid(); + int libos_tid = thread_data->libos_tid; + int libos_exit_status = -1; + sgx_status_t status = occlum_ecall_exec_thread(eid, &libos_exit_status, libos_tid, host_tid); + if (status != SGX_SUCCESS) { + const char* sgx_err = pal_get_sgx_error_msg(status); + PAL_ERROR("Failed to enter the enclave to execute a LibOS thread: %s", sgx_err); + exit(EXIT_FAILURE); + } + + free(thread_data); + return NULL; +} + +// Start a new host OS thread and enter the enclave to execute the LibOS thread +int occlum_ocall_exec_thread_async(int libos_tid) { + int ret = 0; + pthread_t thread; + + thread_data_t* thread_data = malloc(sizeof *thread_data); + thread_data->enclave_id = pal_get_enclave_id(); + thread_data->libos_tid = libos_tid; + + if ((ret = pthread_create(&thread, NULL, exec_libos_thread, thread_data)) < 0) { + free(thread_data); + return -1; + } + pthread_detach(thread); + + // Note: thread_data is freed just before the thread exits + + return 0; +} diff --git a/src/pal/src/ocalls/time.c b/src/pal/src/ocalls/time.c new file mode 100644 index 00000000..a76784d8 --- /dev/null +++ b/src/pal/src/ocalls/time.c @@ -0,0 +1,21 @@ +#include +#include "ocalls.h" + +void occlum_ocall_gettimeofday(long* seconds, long* microseconds) { + struct timeval tv; + gettimeofday(&tv, NULL); + *seconds = tv.tv_sec; + *microseconds = tv.tv_usec; +} + +void occlum_ocall_clock_gettime(int clockid, time_t* sec, long* ns) { + struct timespec ts; + clock_gettime(clockid, &ts); + *sec = ts.tv_sec; + *ns = ts.tv_nsec; +} + +void occlum_ocall_nanosleep(time_t sec, long nsec) { + struct timespec tv = { .tv_sec = sec, .tv_nsec = nsec }; + nanosleep(&tv, NULL); +} diff --git a/src/pal/src/pal_api.c b/src/pal/src/pal_api.c new file mode 100644 index 00000000..df46c165 --- /dev/null +++ b/src/pal/src/pal_api.c @@ -0,0 +1,93 @@ +#include +#include "Enclave_u.h" +#include "pal_enclave.h" +#include "pal_error.h" +#include "pal_log.h" +#include "pal_syscall.h" + +int occlum_pal_init(const char* instance_dir) { + errno = 0; + + if (instance_dir == NULL) { + errno = EINVAL; + return -1; + } + + sgx_enclave_id_t eid = pal_get_enclave_id(); + if (eid != SGX_INVALID_ENCLAVE_ID) { + PAL_ERROR("Enclave has been initialized."); + errno = EEXIST; + return -1; + } + + if (pal_init_enclave(instance_dir) < 0) { + return -1; + } + + // Invoke a do-nothing ECall for two purposes: + // 1) Test the enclave can work; + // 2) Initialize the global data structures inside the enclave (which is + // automatically done by Intel SGX SDK). + eid = pal_get_enclave_id(); + sgx_status_t ecall_status = occlum_ecall_nop(eid); + if (ecall_status != SGX_SUCCESS) { + const char* sgx_err = pal_get_sgx_error_msg(ecall_status); + PAL_ERROR("Failed to do ECall: %s", sgx_err); + return -1; + } + + return 0; +} + +int occlum_pal_exec(const char* cmd_path, const char** cmd_args, int* exit_status) { + errno = 0; + + if (cmd_path == NULL || cmd_args == NULL || exit_status == NULL) { + errno = EINVAL; + return -1; + } + + sgx_enclave_id_t eid = pal_get_enclave_id(); + if (eid == SGX_INVALID_ENCLAVE_ID) { + PAL_ERROR("Enclave is not initialized yet."); + errno = ENOENT; + return -1; + } + + int libos_tid = -1; + sgx_status_t ecall_status = occlum_ecall_new_process(eid, &libos_tid, cmd_path, cmd_args); + if (ecall_status != SGX_SUCCESS) { + const char* sgx_err = pal_get_sgx_error_msg(ecall_status); + PAL_ERROR("Failed to do ECall: %s", sgx_err); + return -1; + } + if (libos_tid < 0) { + return -1; + } + + int host_tid = gettid(); + ecall_status = occlum_ecall_exec_thread(eid, exit_status, libos_tid, host_tid); + if (ecall_status != SGX_SUCCESS) { + const char* sgx_err = pal_get_sgx_error_msg(ecall_status); + PAL_ERROR("Failed to do ECall: %s", sgx_err); + return -1; + } + + return 0; +} + +int occlum_pal_destroy(void) { + errno = 0; + + sgx_enclave_id_t eid = pal_get_enclave_id(); + if (eid == SGX_INVALID_ENCLAVE_ID) { + PAL_ERROR("Enclave is not initialized yet."); + errno = ENOENT; + return -1; + } + + if (pal_destroy_enclave() < 0) { + return -1; + } + return 0; +} diff --git a/src/pal/src/pal_enclave.c b/src/pal/src/pal_enclave.c new file mode 100644 index 00000000..3ccb191a --- /dev/null +++ b/src/pal/src/pal_enclave.c @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "pal_enclave.h" +#include "pal_error.h" +#include "pal_log.h" + +#define MAX_PATH FILENAME_MAX +#define TOKEN_FILENAME "enclave.token" +#define ENCLAVE_FILENAME "libocclum-libos.signed.so" + +static sgx_enclave_id_t global_eid = SGX_INVALID_ENCLAVE_ID; + +/* Get enclave debug flag according to env "OCCLUM_RELEASE_ENCLAVE" */ +static int get_enclave_debug_flag() { + const char* release_enclave_val = getenv("OCCLUM_RELEASE_ENCLAVE"); + if (release_enclave_val) { + if (!strcmp(release_enclave_val, "1") || + !strcasecmp(release_enclave_val, "y") || + !strcasecmp(release_enclave_val, "yes") || + !strcasecmp(release_enclave_val, "true")) { + return 0; + } + } + return 1; +} + +static const char* get_enclave_absolute_path(const char* instance_dir) { + static char enclave_path[MAX_PATH + 1] = {0}; + strncat(enclave_path, instance_dir, MAX_PATH); + strncat(enclave_path, "/build/lib/", MAX_PATH); + strncat(enclave_path, ENCLAVE_FILENAME, MAX_PATH); + return (const char*)enclave_path; +} + +/* Initialize the enclave: + * Step 1: try to retrieve the launch token saved by last transaction + * Step 2: call sgx_create_enclave to initialize an enclave instance + * Step 3: save the launch token if it is updated + */ +int pal_init_enclave(const char* instance_dir) { + char token_path[MAX_PATH] = {'\0'}; + sgx_launch_token_t token = {0}; + sgx_status_t ret = SGX_ERROR_UNEXPECTED; + int updated = 0; + + /* Step 1: try to retrieve the launch token saved by last transaction + * if there is no token, then create a new one. + */ + /* try to get the token saved in $HOME */ + const char *home_dir = getpwuid(getuid())->pw_dir; + + if (home_dir != NULL && + (strlen(home_dir)+strlen("/")+sizeof(TOKEN_FILENAME)+1) <= MAX_PATH) { + /* compose the token path */ + strncpy(token_path, home_dir, strlen(home_dir)); + strncat(token_path, "/", strlen("/")); + strncat(token_path, TOKEN_FILENAME, sizeof(TOKEN_FILENAME)+1); + } else { + /* if token path is too long or $HOME is NULL */ + strncpy(token_path, TOKEN_FILENAME, sizeof(TOKEN_FILENAME)); + } + + FILE *fp = fopen(token_path, "rb"); + if (fp == NULL && (fp = fopen(token_path, "wb")) == NULL) { + PAL_WARN("Warning: Failed to create/open the launch token file \"%s\".\n", token_path); + } + + if (fp != NULL) { + /* read the token from saved file */ + size_t read_num = fread(token, 1, sizeof(sgx_launch_token_t), fp); + if (read_num != 0 && read_num != sizeof(sgx_launch_token_t)) { + /* if token is invalid, clear the buffer */ + memset(&token, 0x0, sizeof(sgx_launch_token_t)); + PAL_WARN("Invalid launch token read from \"%s\".\n", token_path); + } + } + + /* Step 2: call sgx_create_enclave to initialize an enclave instance */ + /* Debug Support: set 2nd parameter to 1 */ + const char* enclave_path = get_enclave_absolute_path(instance_dir); + int sgx_debug_flag = get_enclave_debug_flag(); + ret = sgx_create_enclave(enclave_path, sgx_debug_flag, &token, &updated, &global_eid, NULL); + if (ret != SGX_SUCCESS) { + const char* sgx_err_msg = pal_get_sgx_error_msg(ret); + PAL_ERROR("Failed to create enclave: %s", sgx_err_msg); + if (fp != NULL) fclose(fp); + return -1; + } + + /* Step 3: save the launch token if it is updated */ + if (updated == 0 || fp == NULL) { + /* if the token is not updated, or file handler is invalid, do not perform saving */ + if (fp != NULL) fclose(fp); + return 0; + } + + /* reopen the file with write capablity */ + fp = freopen(token_path, "wb", fp); + if (fp == NULL) return 0; + size_t write_num = fwrite(token, 1, sizeof(sgx_launch_token_t), fp); + if (write_num != sizeof(sgx_launch_token_t)) + PAL_WARN("Warning: Failed to save launch token to \"%s\".\n", token_path); + fclose(fp); + return 0; +} + +int pal_destroy_enclave(void) { + // TODO: destroy the enclave gracefully + // We cannot destroy the enclave gracefully since we may still have + // running threads that are using the enclave at this point, which blocks + // sgx_destory_enclave call. We need to implement exit_group syscall and + // handle signal and exceptions properly. + //sgx_destroy_enclave(global_eid); + return 0; +} + +sgx_enclave_id_t pal_get_enclave_id(void) { + return global_eid; +} diff --git a/src/pal/src/pal_enclave.h b/src/pal/src/pal_enclave.h new file mode 100644 index 00000000..22deae66 --- /dev/null +++ b/src/pal/src/pal_enclave.h @@ -0,0 +1,12 @@ +#ifndef __PAL_ENCLAVE_H__ +#define __PAL_ENCLAVE_H__ + +#include + +int pal_init_enclave(const char* instance_dir); +int pal_destroy_enclave(void); + +#define SGX_INVALID_ENCLAVE_ID (-1) +sgx_enclave_id_t pal_get_enclave_id(void); + +#endif /* __PAL_ENCLAVE_H__ */ diff --git a/src/pal/src/pal_error.c b/src/pal/src/pal_error.c new file mode 100644 index 00000000..b023b129 --- /dev/null +++ b/src/pal/src/pal_error.c @@ -0,0 +1,90 @@ +#include "pal_error.h" + +typedef struct { + sgx_status_t err; + const char *msg; +} sgx_err_msg_t; + +static sgx_err_msg_t err_msg_table[] = { + { + SGX_SUCCESS, + "SGX success." + }, + { + SGX_ERROR_UNEXPECTED, + "Unexpected error occurred." + }, + { + SGX_ERROR_INVALID_PARAMETER, + "Invalid parameter." + }, + { + SGX_ERROR_OUT_OF_MEMORY, + "Out of memory." + }, + { + SGX_ERROR_ENCLAVE_LOST, + "Power transition occurred." + }, + { + SGX_ERROR_INVALID_ENCLAVE, + "Invalid enclave image." + }, + { + SGX_ERROR_INVALID_ENCLAVE_ID, + "Invalid enclave identification." + }, + { + SGX_ERROR_INVALID_SIGNATURE, + "Invalid enclave signature." + }, + { + SGX_ERROR_OUT_OF_EPC, + "Out of EPC memory." + }, + { + SGX_ERROR_NO_DEVICE, + "Invalid SGX device. Please make sure SGX module is enabled in the BIOS, and install SGX driver afterwards." + }, + { + SGX_ERROR_MEMORY_MAP_CONFLICT, + "Memory map conflicted." + }, + { + SGX_ERROR_INVALID_METADATA, + "Invalid enclave metadata." + }, + { + SGX_ERROR_DEVICE_BUSY, + "SGX device was busy." + }, + { + SGX_ERROR_INVALID_VERSION, + "Enclave version was invalid." + }, + { + SGX_ERROR_INVALID_ATTRIBUTE, + "Enclave was not authorized." + }, + { + SGX_ERROR_ENCLAVE_FILE_ACCESS, + "Can't open enclave file." + }, + { + SGX_ERROR_SERVICE_INVALID_PRIVILEGE, + "Enclave has no privilege to get run in the release mode." + "Please rebuild the Occlum enclave with a legal signing key " + "(e.g., occlum build --sign-key ), " + "to get a legal signing key, please contact Intel." + }, +}; + +const char* pal_get_sgx_error_msg(sgx_status_t error) { + int err_max = sizeof err_msg_table/sizeof err_msg_table[0]; + for (int err_i = 0; err_i < err_max; err_i++) { + if(error == err_msg_table[err_i].err) { + return err_msg_table[err_i].msg; + } + } + return "Unknown SGX error"; +} diff --git a/src/pal/src/pal_error.h b/src/pal/src/pal_error.h new file mode 100644 index 00000000..082c2618 --- /dev/null +++ b/src/pal/src/pal_error.h @@ -0,0 +1,9 @@ +#ifndef __PAL_ERROR_H__ +#define __PAL_ERROR_H__ + +#include +#include + +const char* pal_get_sgx_error_msg(sgx_status_t error); + +#endif /* __PAL_ERROR_H__ */ diff --git a/src/pal/src/pal_log.h b/src/pal/src/pal_log.h new file mode 100644 index 00000000..0fc1237a --- /dev/null +++ b/src/pal/src/pal_log.h @@ -0,0 +1,15 @@ +#ifndef __PAL_LOG_H__ +#define __PAL_LOG_H__ + +#include + +#define PAL_DEBUG(fmt, ...) \ + fprintf(stderr, "[DEBUG] occlum-pal: " fmt " (line %d, file %s)\n", ##__VA_ARGS__, __LINE__, __FILE__) +#define PAL_INFO(fmt, ...) \ + fprintf(stderr, "[INFO] occlum-pal: " fmt " (line %d, file %s)\n", ##__VA_ARGS__, __LINE__, __FILE__) +#define PAL_WARN(fmt, ...) \ + fprintf(stderr, "[WARM] occlum-pal: " fmt " (line %d, file %s)\n", ##__VA_ARGS__, __LINE__, __FILE__) +#define PAL_ERROR(fmt, ...) \ + fprintf(stderr, "[ERROR] occlum-pal: " fmt " (line %d, file %s)\n", ##__VA_ARGS__, __LINE__, __FILE__) + +#endif /* __PAL_LOG_H__ */ diff --git a/src/pal/src/pal_syscall.h b/src/pal/src/pal_syscall.h new file mode 100644 index 00000000..c29624fb --- /dev/null +++ b/src/pal/src/pal_syscall.h @@ -0,0 +1,10 @@ +#ifndef __PAL_SYSCALL_H__ +#define __PAL_SYSCALL_H__ + +#define _GNU_SOURCE +#include +#include + +#define gettid() syscall(__NR_gettid) + +#endif /* __PAL_SYSCALL_H__ */ diff --git a/src/pal/task.c b/src/pal/task.c deleted file mode 100644 index 3d6e2ad4..00000000 --- a/src/pal/task.c +++ /dev/null @@ -1,91 +0,0 @@ - -#include -#include -#include -#include -#include -#include -#include "atomic.h" -#include "futex.h" -#include "sgx_urts.h" -#include "Enclave_u.h" - -int syscall(); -#define gettid() syscall(__NR_gettid) - -static volatile int num_tasks = 0; -static volatile int any_fatal_error = 0; - -// The LibOS never returns INT_MIN. As long as the main_task_status == INT_MIN, -// the main task must not have returned. -#define MAIN_TASK_NOT_RETURNED INT_MIN -static volatile int main_task_status = MAIN_TASK_NOT_RETURNED; - -static int BEGIN_TASK(void) { - return a_fetch_and_add(&num_tasks, 1) == 0; -} - -static void END_TASK(void) { - if (a_fetch_and_add(&num_tasks, -1) == 1) { - futex_wakeup(&num_tasks); - } -} - -struct task_thread_data { - int is_main_task; - sgx_enclave_id_t eid; -}; - -static void* __run_task_thread(void* _data) { - int status = 0; - struct task_thread_data* data = _data; - - sgx_status_t sgx_ret = libos_run(data->eid, &status, gettid()); - if(sgx_ret != SGX_SUCCESS) { - // TODO: deal with ECALL error - printf("ERROR: ECall libos_run failed\n"); - any_fatal_error = 1; - } - - if (data->is_main_task) { - a_store(&main_task_status, status); - futex_wakeup(&main_task_status); - } - - free(data); - END_TASK(); - return NULL; -} - -int run_new_task(sgx_enclave_id_t eid) { - int ret = 0; - pthread_t thread; - - struct task_thread_data* data = malloc(sizeof(*data)); - data->is_main_task = BEGIN_TASK(); - data->eid = eid; - - if ((ret = pthread_create(&thread, NULL, __run_task_thread, data)) < 0) { - free(data); - END_TASK(); - return ret; - } - pthread_detach(thread); - - return 0; -} - -int wait_main_task(void) { - while ((a_load(&main_task_status)) == MAIN_TASK_NOT_RETURNED) { - futex_wait(&main_task_status, MAIN_TASK_NOT_RETURNED); - } - return main_task_status; -} - -int wait_all_tasks(void) { - int cur_num_tasks; - while ((cur_num_tasks = a_load(&num_tasks)) != 0) { - futex_wait(&num_tasks, cur_num_tasks); - } - return any_fatal_error ? -1 : main_task_status; -} diff --git a/src/pal/task.h b/src/pal/task.h deleted file mode 100644 index f8162eea..00000000 --- a/src/pal/task.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef __TASK_H_ -#define __TASK_H_ - -int run_new_task(sgx_enclave_id_t eid); -int wait_all_tasks(void); -int wait_main_task(void); - -#endif /* __TASK_H_ */ diff --git a/src/run/Makefile b/src/run/Makefile new file mode 100644 index 00000000..ad2b99fa --- /dev/null +++ b/src/run/Makefile @@ -0,0 +1,30 @@ +include ../sgxenv.mk + +BIN := $(BUILD_DIR)/bin/occlum-run + +C_SRCS := $(sort $(wildcard *.c)) +C_OBJS := $(addprefix $(BUILD_DIR)/src/run/,$(C_SRCS:.c=.o)) + +C_COMMON_FLAGS := -I$(PROJECT_DIR)/src/pal/include +C_FLAGS := $(C_COMMON_FLAGS) $(SGX_CFLAGS_U) +LINK_FLAGS := $(SGX_LFLAGS_U) -L$(BUILD_DIR)/lib -lsgx_uprotected_fs -locclum-pal + +ALL_BUILD_SUBDIRS := $(sort $(patsubst %/,%,$(dir $(BIN) $(C_OBJS)))) + +.PHONY: all clean + +all: $(ALL_BUILD_SUBDIRS) $(BIN) + +$(ALL_BUILD_SUBDIRS): + @mkdir -p $@ + +$(BIN): $(C_OBJS) + @$(CC) $^ -o $@ $(LINK_FLAGS) + @echo "LINK => $@" + +$(BUILD_DIR)/src/run/%.o: %.c + @$(CC) $(C_FLAGS) -c $< -o $@ + @echo "CC <= $@" + +clean: + @-$(RM) -f $(BIN) $(C_OBJS) diff --git a/src/run/main.c b/src/run/main.c new file mode 100644 index 00000000..01b6dc39 --- /dev/null +++ b/src/run/main.c @@ -0,0 +1,42 @@ +#include +#include +#include +#include + +static const char* get_instance_dir(void) { + const char* instance_dir_from_env = (const char*) getenv("OCCLUM_INSTANCE_DIR"); + if (instance_dir_from_env != NULL) { + return instance_dir_from_env; + } + else { + return "./.occlum"; + } +} + +int main(int argc, char* argv[]) { + // Parse arguments + if (argc < 2) { + fprintf(stderr, "[ERROR] occlum-run: at least one argument must be provided\n\n"); + fprintf(stderr, "Usage: occlum-run []\n"); + return EXIT_FAILURE; + } + const char* cmd_path = (const char*) argv[1]; + const char** cmd_args = (const char**) &argv[2]; + + // Init Occlum PAL + const char* instance_dir = get_instance_dir(); + if (occlum_pal_init(instance_dir) < 0) { + return EXIT_FAILURE; + } + + // Use Occlum PAL to execute the cmd + int exit_status = 0; + if (occlum_pal_exec(cmd_path, cmd_args, &exit_status) < 0) { + return EXIT_FAILURE; + } + + // Destroy Occlum PAL + occlum_pal_destroy(); + + return exit_status; +} diff --git a/tools/occlum b/tools/occlum index df6dd018..1c153ef8 100755 --- a/tools/occlum +++ b/tools/occlum @@ -104,10 +104,11 @@ cmd_build() { rm -rf build mkdir -p build/bin - cp "$occlum_dir/build/bin/occlum-pal" build/bin/ + cp "$occlum_dir/build/bin/occlum-run" build/bin/ mkdir -p build/lib - cp "$occlum_dir/build/lib/libocclum_core.a" build/lib/ + cp "$occlum_dir/build/lib/libocclum-libos-core.a" build/lib/ cp "$occlum_dir/build/lib/libcompiler-rt-patch.a" build/lib/ + cp "$occlum_dir/build/lib/libocclum-pal.so" build/lib/ mkdir -p build/src/libos/src/builtin chmod 531 -R $working_dir/image/bin @@ -145,7 +146,7 @@ cmd_build() { cp -r "$occlum_dir/src/libos/src/builtin" src/libos/src/builtin cd src/libos && \ make clean-builtin && \ - make "$context_dir/build/lib/libocclum.so" ONLY_REBUILD_BUILTIN=1 + make "$context_dir/build/lib/libocclum-libos.so" ONLY_REBUILD_BUILTIN=1 while [ -n "$1" ]; do case "$1" in --sign-key) [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path" ;; @@ -160,11 +161,13 @@ cmd_build() { $ENCLAVE_SIGN_TOOL sign \ -key $ENCLAVE_SIGN_KEY \ -config "$working_dir/Enclave.xml" \ - -enclave "$context_dir/build/lib/libocclum.so" \ - -out "$context_dir/build/lib/libocclum.signed.so" + -enclave "$context_dir/build/lib/libocclum-libos.so" \ + -out "$context_dir/build/lib/libocclum-libos.signed.so" cd "$context_dir" echo "built" > status + mkdir -p "$context_dir/run/mount/root" + echo "Built the Occlum image and enclave successfully" } @@ -172,8 +175,7 @@ cmd_run() { cd "$working_dir" echo "running" > "$context_dir/status" - mkdir -p "$context_dir/run/mount/root" - RUST_BACKTRACE=1 "$context_dir/build/bin/occlum-pal" "$@" + RUST_BACKTRACE=1 LD_LIBRARY_PATH="$context_dir/build/lib" "$context_dir/build/bin/occlum-run" "$@" echo "built" > "$context_dir/status" } diff --git a/tools/occlum-build-enclave b/tools/occlum-build-enclave index 0baa7df7..72e78a93 100755 --- a/tools/occlum-build-enclave +++ b/tools/occlum-build-enclave @@ -46,12 +46,12 @@ build_enclave_so() { sign_enclave_so() { cd $working_dir - rm -f libocclum.signed.so - local enclave_so_path="$project_dir/src/libos/libocclum.so" + rm -f libocclum-libos.signed.so + local enclave_so_path="$project_dir/src/libos/libocclum-libos.so" $SGX_SDK/bin/x64/sgx_sign sign \ -key $enclave_key_pem_path \ -enclave $enclave_so_path \ - -out "libocclum.signed.so" \ + -out "libocclum-libos.signed.so" \ -config $enclave_conf_xml_path } @@ -73,7 +73,7 @@ if [[ $enclave_key_pem_path != *.pem ]] ; then fi # =========================================================================== -# Build Occlum.json.protected and libocclum.signed.so +# Build Occlum.json.protected and libocclum-libos.signed.so # =========================================================================== set -e @@ -90,4 +90,4 @@ echo "EXPORT => OCCLUM_BUILTIN_VM_USER_SPACE_SIZE = $OCCLUM_BUILTIN_VM_USER_SPAC build_enclave_so sign_enclave_so -echo "SIGN => libocclum.signed.so" +echo "SIGN => libocclum-libos.signed.so"