Refactor EDL for adding custom C types easily

Before this commit, using custom C types in ECalls/OCalls defined in Occlum's
EDL is cumbersme. Now this issue is resolved by providing `occlum_edl_types.h`
header file. There are two versions of this file: one is under
`src/libos/include/edl/` for LibOS, the other is under
`src/pal/include/edl/` for PAL. So now to define a new custom C type, just
edit the two versions of `occlum_edl_types.h` to define the type.

Makefile

@@ -47,7 +47,7 @@ install:
 	install -d $(OCCLUM_PREFIX)/src/libos/src/builtin/
 	install -t $(OCCLUM_PREFIX)/src/libos/src/builtin/ -m 444 src/libos/src/builtin/*
 	install -d $(OCCLUM_PREFIX)/include/
-	install -t $(OCCLUM_PREFIX)/include/ -m 444 src/pal/include/*
+	install -t $(OCCLUM_PREFIX)/include/ -m 444 src/pal/include/*.h
 	install -d $(OCCLUM_PREFIX)/etc/template/
 	install -t $(OCCLUM_PREFIX)/etc/template/ -m 444 etc/template/*
 

src/Enclave.edl

@@ -7,6 +7,7 @@ enclave {
     from "sgx_net.edl" import *;
 
     include "sgx_quote.h"
+    include "occlum_edl_types.h"
 
     trusted {
         /*
@@ -49,16 +50,24 @@ enclave {
          */
         int occlum_ocall_exec_thread_async(int libos_tid);
 
-        void occlum_ocall_gettimeofday([out] long* sec, [out] long* us);
+        void occlum_ocall_gettimeofday([out] struct timeval* tv);
-        void occlum_ocall_clock_gettime(int clockid, [out] long* sec, [out] long* ns);
+        void occlum_ocall_clock_gettime(clockid_t clockid, [out] struct timespec* ts);
 
-        void occlum_ocall_nanosleep(long sec, long nsec);
+        void occlum_ocall_nanosleep([in] const struct timespec* req);
 
         void occlum_ocall_sync(void);
 
         void occlum_ocall_sched_yield(void);
-        int occlum_ocall_sched_getaffinity([out] int *error, int pid, size_t cpusize, [out, size=cpusize] unsigned char* buf);
+        int occlum_ocall_sched_getaffinity(
-        int occlum_ocall_sched_setaffinity([out] int *error, int pid, size_t cpusize, [in, size=cpusize] const unsigned char* buf);
+            int host_tid,
+            size_t cpusize,
+            [out, size=cpusize] unsigned char* buf
+        ) propagate_errno;
+        int occlum_ocall_sched_setaffinity(
+            int host_tid,
+            size_t cpusize,
+            [in, size=cpusize] const unsigned char* buf
+        ) propagate_errno;
 
         sgx_status_t occlum_ocall_sgx_init_quote(
             [out] sgx_target_info_t* target_info,

src/libos/Makefile

@@ -125,7 +125,7 @@ $(LIBOS_CORE_RS_A): $(RUST_SRCS)
 endif
 
 $(BUILD_DIR)/src/libos/src/Enclave_t.o: $(BUILD_DIR)/src/libos/src/Enclave_t.c
-	@$(CC) $(C_FLAGS) -c $< -o $@
+	@$(CC) $(C_FLAGS) -Iinclude/edl -c $< -o $@
 	@echo "CC <= $@"
 
 $(BUILD_DIR)/src/libos/src/Enclave_t.c: $(SGX_EDGER8R) ../Enclave.edl

src/libos/include/edl/occlum_edl_types.h

@@ -0,0 +1,13 @@
+#ifndef __OCCLUM_EDL_TYPES_H__
+#define __OCCLUM_EDL_TYPES_H__
+
+typedef long                time_t;
+typedef long                suseconds_t;
+typedef int                 clockid_t;
+
+struct timeval {
+    time_t      tv_sec;     /* seconds */
+    suseconds_t tv_usec;    /* microseconds */
+};
+
+#endif /* __OCCLUM_EDL_TYPES_H__ */

src/libos/src/process/sched.rs

@@ -1,23 +1,5 @@
 use super::*;
 
-extern "C" {
-    fn occlum_ocall_sched_getaffinity(
-        ret: *mut i32,
-        errno: *mut i32,
-        pid: i32,
-        cpusetsize: size_t,
-        mask: *mut c_uchar,
-    ) -> sgx_status_t;
-    fn occlum_ocall_sched_setaffinity(
-        ret: *mut i32,
-        errno: *mut i32,
-        pid: i32,
-        cpusetsize: size_t,
-        mask: *const c_uchar,
-    ) -> sgx_status_t;
-    fn occlum_ocall_sched_yield() -> sgx_status_t;
-}
-
 pub struct CpuSet {
     vec: Vec<u8>,
 }
@@ -86,42 +68,37 @@ fn find_host_tid(pid: pid_t) -> Result<pid_t> {
     Ok(host_tid)
 }
 
-pub fn do_sched_getaffinity(pid: pid_t, cpu_set: &mut CpuSet) -> Result<i32> {
+pub fn do_sched_getaffinity(pid: pid_t, cpu_set: &mut CpuSet) -> Result<usize> {
     let host_tid = match pid {
         0 => 0,
         _ => find_host_tid(pid)?,
     };
     let buf = cpu_set.as_mut_ptr();
     let cpusize = cpu_set.len();
-    let mut ret = 0;
+    let retval = try_libc!({
-    let mut error = 0;
+        let mut retval = 0;
-    unsafe {
+        let sgx_status = occlum_ocall_sched_getaffinity(&mut retval, host_tid as i32, cpusize, buf);
-        occlum_ocall_sched_getaffinity(&mut ret, &mut error, host_tid as i32, cpusize, buf);
+        assert!(sgx_status == sgx_status_t::SGX_SUCCESS);
-    }
+        retval
-    if (ret < 0) {
+    }) as usize;
-        let errno = Errno::from(error as u32);
+    // Note: the first retval bytes in CpuSet are valid
-        return_errno!(errno, "occlum_ocall_sched_getaffinity failed");
+    Ok(retval)
-    }
-    Ok(ret)
 }
 
-pub fn do_sched_setaffinity(pid: pid_t, cpu_set: &CpuSet) -> Result<i32> {
+pub fn do_sched_setaffinity(pid: pid_t, cpu_set: &CpuSet) -> Result<()> {
     let host_tid = match pid {
         0 => 0,
         _ => find_host_tid(pid)?,
     };
     let buf = cpu_set.as_ptr();
     let cpusize = cpu_set.len();
-    let mut ret = 0;
+    try_libc!({
-    let mut error = 0;
+        let mut retval = 0;
-    unsafe {
+        let sgx_status = occlum_ocall_sched_setaffinity(&mut retval, host_tid as i32, cpusize, buf);
-        occlum_ocall_sched_setaffinity(&mut ret, &mut error, host_tid as i32, cpusize, buf);
+        assert!(sgx_status == sgx_status_t::SGX_SUCCESS);
-    }
+        retval
-    if (ret < 0) {
+    });
-        let errno = Errno::from(error as u32);
+    Ok(())
-        return_errno!(errno, "occlum_ocall_sched_setaffinity failed");
-    }
-    Ok(ret)
 }
 
 pub fn do_sched_yield() {
@@ -130,3 +107,19 @@ pub fn do_sched_yield() {
         assert!(status == sgx_status_t::SGX_SUCCESS);
     }
 }
+
+extern "C" {
+    fn occlum_ocall_sched_getaffinity(
+        ret: *mut i32,
+        host_tid: i32,
+        cpusetsize: size_t,
+        mask: *mut c_uchar,
+    ) -> sgx_status_t;
+    fn occlum_ocall_sched_setaffinity(
+        ret: *mut i32,
+        host_tid: i32,
+        cpusetsize: size_t,
+        mask: *const c_uchar,
+    ) -> sgx_status_t;
+    fn occlum_ocall_sched_yield() -> sgx_status_t;
+}

src/libos/src/syscall/mod.rs

@@ -1046,11 +1046,10 @@ fn do_sched_getaffinity(pid: pid_t, cpusize: size_t, buf: *mut c_uchar) -> Resul
     };
     // Call the memory-safe do_sched_getaffinity
     let mut cpuset = CpuSet::new(cpusize);
-    let ret = process::do_sched_getaffinity(pid, &mut cpuset)?;
+    let retval = process::do_sched_getaffinity(pid, &mut cpuset)?;
-    debug!("sched_getaffinity cpuset: {:#x}", cpuset);
     // Copy from Rust types to C types
     buf_slice.copy_from_slice(cpuset.as_slice());
-    Ok(ret as isize)
+    Ok(retval as isize)
 }
 
 fn do_sched_setaffinity(pid: pid_t, cpusize: size_t, buf: *const c_uchar) -> Result<isize> {
@@ -1067,8 +1066,8 @@ fn do_sched_setaffinity(pid: pid_t, cpusize: size_t, buf: *const c_uchar) -> Res
     };
     debug!("sched_setaffinity cpuset: {:#x}", cpuset);
     // Call the memory-safe do_sched_setaffinity
-    let ret = process::do_sched_setaffinity(pid, &cpuset)?;
+    process::do_sched_setaffinity(pid, &cpuset)?;
-    Ok(ret as isize)
+    Ok(0)
 }
 
 fn do_socket(domain: c_int, socket_type: c_int, protocol: c_int) -> Result<isize> {

src/libos/src/time/mod.rs

@@ -21,18 +21,27 @@ impl timeval_t {
     pub fn as_usec(&self) -> usize {
         (self.sec * 1000000 + self.usec) as usize
     }
+
+    pub fn validate(&self) -> Result<()> {
+        if self.sec >= 0 && self.usec >= 0 && self.usec < 1_000_000 {
+            Ok(())
+        } else {
+            return_errno!(EINVAL, "invalid value for timeval_t");
+        }
+    }
 }
 
 pub fn do_gettimeofday() -> timeval_t {
-    let mut tv: timeval_t = Default::default();
+    extern "C" {
-    unsafe {
+        fn occlum_ocall_gettimeofday(tv: *mut timeval_t) -> sgx_status_t;
-        occlum_ocall_gettimeofday(&mut tv.sec as *mut time_t, &mut tv.usec as *mut suseconds_t);
-    }
-    tv
     }
 
-extern "C" {
+    let mut tv: timeval_t = Default::default();
-    fn occlum_ocall_gettimeofday(sec: *mut time_t, usec: *mut suseconds_t) -> sgx_status_t;
+    unsafe {
+        occlum_ocall_gettimeofday(&mut tv as *mut timeval_t);
+    }
+    tv.validate().expect("ocall returned invalid timeval_t");
+    tv
 }
 
 #[repr(C)]
@@ -51,6 +60,14 @@ impl timespec_t {
         }
         Ok(ts)
     }
+
+    pub fn validate(&self) -> Result<()> {
+        if self.sec >= 0 && self.nsec >= 0 && self.nsec < 1_000_000_000 {
+            Ok(())
+        } else {
+            return_errno!(EINVAL, "invalid value for timespec_t");
+        }
+    }
 }
 
 #[allow(non_camel_case_types)]
@@ -88,31 +105,23 @@ impl ClockID {
 
 pub fn do_clock_gettime(clockid: ClockID) -> Result<timespec_t> {
     extern "C" {
-        fn occlum_ocall_clock_gettime(
+        fn occlum_ocall_clock_gettime(clockid: clockid_t, tp: *mut timespec_t) -> sgx_status_t;
-            clockid: clockid_t,
-            sec: *mut time_t,
-            ns: *mut i64,
-        ) -> sgx_status_t;
     }
 
-    let mut sec = 0;
+    let mut tv: timespec_t = Default::default();
-    let mut nsec = 0;
     unsafe {
-        occlum_ocall_clock_gettime(
+        occlum_ocall_clock_gettime(clockid as clockid_t, &mut tv as *mut timespec_t);
-            clockid as clockid_t,
-            &mut sec as *mut time_t,
-            &mut nsec as *mut i64,
-        );
     }
-    Ok(timespec_t { sec, nsec })
+    tv.validate().expect("ocall returned invalid timespec");
+    Ok(tv)
 }
 
 pub fn do_nanosleep(req: &timespec_t) -> Result<()> {
     extern "C" {
-        fn occlum_ocall_nanosleep(sec: time_t, nsec: i64) -> sgx_status_t;
+        fn occlum_ocall_nanosleep(req: *const timespec_t) -> sgx_status_t;
     }
     unsafe {
-        occlum_ocall_nanosleep(req.sec, req.nsec);
+        occlum_ocall_nanosleep(req as *const timespec_t);
     }
     Ok(())
 }

src/pal/Makefile

@@ -12,7 +12,7 @@ C_OBJS := $(addprefix $(BUILD_DIR)/src/pal/,$(C_SRCS:.c=.o))
 CXX_SRCS := $(sort $(wildcard src/*.cpp src/*/*.cpp))
 CXX_OBJS := $(addprefix $(BUILD_DIR)/src/pal/,$(CXX_SRCS:.cpp=.o))
 
-C_COMMON_FLAGS := -I$(BUILD_DIR)/src/pal/src -Iinclude
+C_COMMON_FLAGS := -I$(BUILD_DIR)/src/pal/src -Iinclude -Iinclude/edl
 C_FLAGS := $(C_COMMON_FLAGS) $(SGX_CFLAGS_U)
 CXX_FLAGS := $(C_COMMON_FLAGS) $(SGX_CXXFLAGS_U)
 LINK_FLAGS := $(SGX_LFLAGS_U) -shared -L$(RUST_SGX_SDK_DIR)/sgx_ustdc/ -lsgx_ustdc -lsgx_uprotected_fs
@@ -36,7 +36,10 @@ $(BUILD_DIR)/src/pal/src/Enclave_u.o: $(BUILD_DIR)/src/pal/src/Enclave_u.c
 	@echo "CC <= $@"
 
 $(BUILD_DIR)/src/pal/src/Enclave_u.c: $(SGX_EDGER8R) ../Enclave.edl
-	@cd $(BUILD_DIR)/src/pal/src && $(SGX_EDGER8R) --untrusted $(CUR_DIR)/../Enclave.edl --search-path $(SGX_SDK)/include --search-path $(RUST_SGX_SDK_DIR)/edl/
+	@cd $(BUILD_DIR)/src/pal/src && \
+		$(SGX_EDGER8R) --untrusted $(CUR_DIR)/../Enclave.edl \
+		--search-path $(SGX_SDK)/include \
+		--search-path $(RUST_SGX_SDK_DIR)/edl/
 	@echo "GEN <= $@"
 
 $(BUILD_DIR)/src/pal/%.o: %.c

src/pal/include/edl/occlum_edl_types.h

@@ -0,0 +1,7 @@
+#ifndef __OCCLUM_EDL_TYPES__
+#define __OCCLUM_EDL_TYPES__
+
+#include <time.h>       // import struct timespec
+#include <sys/time.h>   // import struct timeval
+
+#endif /* __OCCLUM_EDL_TYPES__ */

src/pal/src/ocalls/sched.c

@@ -1,24 +1,16 @@
+#define _GNU_SOURCE
 #include <sched.h>
 #include "ocalls.h"
 
-int occlum_ocall_sched_getaffinity(int* error, int pid, size_t cpusize, unsigned char* buf) {
+int occlum_ocall_sched_getaffinity(int host_tid, size_t cpusize, unsigned char* buf) {
-    int ret = syscall(__NR_sched_getaffinity, pid, cpusize, buf);
+    return syscall(__NR_sched_getaffinity, host_tid, cpusize, buf);
-    if (error) {
-        *error = (ret == -1) ? errno : 0;
-    }
-    return ret;
 }
 
-int occlum_ocall_sched_setaffinity(int* error, int pid, size_t cpusize, const unsigned char* buf) {
+int occlum_ocall_sched_setaffinity(int host_tid, size_t cpusize, const unsigned char* buf) {
-    int ret = syscall(__NR_sched_setaffinity, pid, cpusize, buf);
+    return syscall(__NR_sched_setaffinity, host_tid, cpusize, buf);
-    if (error) {
-        *error = (ret == -1) ? errno : 0;
-    }
-    return ret;
 }
 
 /* In the Linux implementation, sched_yield() always succeeds */
 void occlum_ocall_sched_yield(void) {
     sched_yield();
 }
-

src/pal/src/ocalls/time.c

@@ -1,21 +1,14 @@
 #include <sys/time.h>
 #include "ocalls.h"
 
-void occlum_ocall_gettimeofday(long* seconds, long* microseconds) {
+void occlum_ocall_gettimeofday(struct timeval* tv) {
-    struct timeval tv;
+    gettimeofday(tv, NULL);
-    gettimeofday(&tv, NULL);
-    *seconds = tv.tv_sec;
-    *microseconds = tv.tv_usec;
 }
 
-void occlum_ocall_clock_gettime(int clockid, time_t* sec, long* ns) {
+void occlum_ocall_clock_gettime(int clockid, struct timespec *tp) {
-    struct timespec ts;
+    clock_gettime(clockid, tp);
-    clock_gettime(clockid, &ts);
-    *sec = ts.tv_sec;
-    *ns = ts.tv_nsec;
 }
 
-void occlum_ocall_nanosleep(time_t sec, long nsec) {
+void occlum_ocall_nanosleep(const struct timespec* req) {
-    struct timespec tv = { .tv_sec = sec, .tv_nsec = nsec };
+    nanosleep(req, NULL);
-    nanosleep(&tv, NULL);
 }