Use Intel SGX SDK reserved memory as the user space memory

src/libos/Makefile

@@ -82,10 +82,6 @@ CXX_FLAGS := $(SGX_CXXFLAGS_T) $(C_COMMON_FLAGS)
 ifdef OCCLUM_BUILTIN_CONF_FILE_MAC
 C_FLAGS += -DOCCLUM_BUILTIN_CONF_FILE_MAC='"$(OCCLUM_BUILTIN_CONF_FILE_MAC)"'
 endif
-# The total size of user-space memory must be builtin into the binary
-ifdef OCCLUM_BUILTIN_VM_USER_SPACE_SIZE
-C_FLAGS += -DOCCLUM_BUILTIN_VM_USER_SPACE_SIZE='($(OCCLUM_BUILTIN_VM_USER_SPACE_SIZE))'
-endif
 
 _Other_Link_Flags := -L$(RUST_SGX_SDK_DIR)/compiler-rt/ -L$(BUILD_DIR)/lib
 _Other_Enclave_Libs := -locclum-libos-core -lsgx_tprotected_fs

src/libos/src/builtin/vm_get_preallocated_user_space_memory.c

@@ -1,17 +0,0 @@
-#include <stddef.h>
-
-// The total size of the memory available to user programs
-// Should be provided by Makefile
-#ifndef OCCLUM_BUILTIN_VM_USER_SPACE_SIZE
-#define OCCLUM_BUILTIN_VM_USER_SPACE_SIZE      (128*1024*1024)
-#endif
-
-static char __preallocated_memory[OCCLUM_BUILTIN_VM_USER_SPACE_SIZE]
-    __attribute__ ((
-        section(".executable_data,\"awx\",@nobits#"),
-        aligned(4096))) = {0};
-
-void vm_get_preallocated_user_space_memory(void** paddr, size_t* psize) {
-    *paddr = __preallocated_memory;
-    *psize = sizeof(__preallocated_memory);
-}

src/libos/src/vm/user_space_vm.rs

@@ -1,71 +1,109 @@
 use super::*;
+use config::LIBOS_CONFIG;
 
 /// The virtual memory manager for the entire user space
-#[derive(Debug)]
 pub struct UserSpaceVMManager {
-    vm_manager: Arc<SgxMutex<VMManager>>,
+    total_size: usize,
+    free_size: SgxMutex<usize>,
 }
 
 impl UserSpaceVMManager {
-    pub unsafe fn from(addr: usize, size: usize) -> Result<UserSpaceVMManager> {
+    fn new() -> UserSpaceVMManager {
-        let vm_manager = Arc::new(SgxMutex::new(VMManager::from(addr, size)?));
+        let rsrv_mem_size = LIBOS_CONFIG.resource_limits.user_space_size;
-        Ok(UserSpaceVMManager { vm_manager })
+        UserSpaceVMManager {
+            total_size: rsrv_mem_size,
+            free_size: SgxMutex::new(rsrv_mem_size),
+        }
     }
 
     pub fn alloc(&self, size: usize) -> Result<UserSpaceVMRange> {
-        let user_vm_range = unsafe {
+        let vm_range = unsafe {
-            let mmap_options = VMMapOptionsBuilder::default().size(size).build()?;
+            let ptr = sgx_alloc_rsrv_mem(size);
+            let perm = MemPerm::READ | MemPerm::WRITE | MemPerm::EXEC;
+            if ptr.is_null() {
+                return_errno!(ENOMEM, "run out of reserved memory");
+            }
+            // Change the page permission to RWX
+            assert!(sgx_tprotect_rsrv_mem(ptr, size, perm.bits()) == sgx_status_t::SGX_SUCCESS);
 
-            let mut vm_manager = self.vm_manager.lock().unwrap();
+            let addr = ptr as usize;
-            let user_vm_addr = vm_manager.mmap(&mmap_options)?;
+            debug!("allocated rsrv addr is 0x{:x}, len is 0x{:x}", addr, size);
-            VMRange::from_unchecked(user_vm_addr, user_vm_addr + size)
+            VMRange::from_unchecked(addr, addr + size)
         };
-        Ok(UserSpaceVMRange::new(
+
-            user_vm_range,
+        *self.free_size.lock().unwrap() -= size;
-            self.vm_manager.clone(),
+        Ok(UserSpaceVMRange::new(vm_range))
-        ))
     }
 
+    fn add_free_size(&self, user_space_vmrange: &UserSpaceVMRange) {
+        *self.free_size.lock().unwrap() += user_space_vmrange.range().size();
+    }
+
+    // The empty range is not added to sub_range
     pub fn alloc_dummy(&self) -> UserSpaceVMRange {
         let empty_user_vm_range = unsafe { VMRange::from_unchecked(0, 0) };
-        UserSpaceVMRange::new(empty_user_vm_range, self.vm_manager.clone())
+        UserSpaceVMRange::new(empty_user_vm_range)
+    }
+
+    pub fn get_total_size(&self) -> usize {
+        self.total_size
+    }
+
+    pub fn get_free_size(&self) -> usize {
+        *self.free_size.lock().unwrap()
     }
 }
 
 lazy_static! {
-    pub static ref USER_SPACE_VM_MANAGER: UserSpaceVMManager = {
+    pub static ref USER_SPACE_VM_MANAGER: UserSpaceVMManager = UserSpaceVMManager::new();
-        let (addr, size) = {
+}
-            let mut addr: usize = 0;
+
-            let mut size: usize = 0;
+bitflags! {
-            unsafe { vm_get_preallocated_user_space_memory(&mut addr, &mut size) };
+    struct MemPerm: i32 {
-            (addr, size)
+        const READ  = 1;
-        };
+        const WRITE = 2;
-        let user_space_vm_manager = unsafe {
+        const EXEC  = 4;
-            match UserSpaceVMManager::from(addr, size) {
+    }
-                Ok(user_space_vm) => user_space_vm,
-                Err(_) => panic!("Failed to initialize the user space virtual memory"),
-            }
-        };
-        user_space_vm_manager
-    };
 }
 
 extern "C" {
-    pub fn vm_get_preallocated_user_space_memory(addr: &mut usize, size: &mut usize);
+    // Allocate a range of EPC memory from the reserved memory area with RW permission
+    //
+    // Parameters:
+    // Inputs: length [in]: Size of region to be allocated in bytes. Page aligned
+    // Return: Starting address of the new allocated memory area on success; otherwise NULL
+    //
+    fn sgx_alloc_rsrv_mem(length: usize) -> *const c_void;
+
+    // Free a range of EPC memory from the reserved memory area
+    //
+    // Parameters:
+    // Inputs: addr[in]: Starting address of region to be freed. Page aligned.
+    //         length[in]: The length of the memory to be freed in bytes.  Page aligned
+    // Return: 0 on success; otherwise -1
+    //
+    fn sgx_free_rsrv_mem(addr: *const c_void, length: usize) -> i32;
+
+    // Modify the access permissions of the pages in the reserved memory area
+    //
+    // Parameters:
+    // Inputs: addr[in]: Starting address of region which needs to change access
+    //         permission. Page aligned.
+    //         length[in]: The length of the memory to be manipulated in bytes. Page aligned.
+    //         prot[in]: The target memory protection.
+    // Return: sgx_status_t
+    //
+    fn sgx_tprotect_rsrv_mem(addr: *const c_void, length: usize, prot: i32) -> sgx_status_t;
 }
 
 #[derive(Debug)]
 pub struct UserSpaceVMRange {
     vm_range: VMRange,
-    vm_manager: Arc<SgxMutex<VMManager>>,
 }
 
 impl UserSpaceVMRange {
-    fn new(vm_range: VMRange, vm_manager: Arc<SgxMutex<VMManager>>) -> UserSpaceVMRange {
+    fn new(vm_range: VMRange) -> UserSpaceVMRange {
-        UserSpaceVMRange {
+        UserSpaceVMRange { vm_range }
-            vm_range,
-            vm_manager,
-        }
     }
 
     pub fn range(&self) -> &VMRange {
@@ -75,14 +113,14 @@ impl UserSpaceVMRange {
 
 impl Drop for UserSpaceVMRange {
     fn drop(&mut self) {
-        let addr = self.vm_range.start();
+        let addr = self.vm_range.start() as *const c_void;
         let size = self.vm_range.size();
         if size == 0 {
             return;
         }
-        let mut vm_manager = self.vm_manager.lock().unwrap();
+
-        vm_manager
+        USER_SPACE_VM_MANAGER.add_free_size(self);
-            .munmap(addr, size)
+
-            .expect("munmap should always succeed");
+        assert!(unsafe { sgx_free_rsrv_mem(addr, size) == 0 });
     }
 }

tools/gen_enclave_conf/src/main.rs

@@ -84,6 +84,16 @@ fn main() {
         return;
     }
 
+    // get the user space size
+    let user_space_size = parse_memory_size(&occlum_config.resource_limits.user_space_size);
+    if user_space_size.is_err() {
+        println!(
+            "The user_space_size \"{}\" is not correct.",
+            occlum_config.resource_limits.user_space_size
+        );
+        return;
+    }
+
     let sgx_enclave_configuration = EnclaveConfiguration {
         ProdID: occlum_config.metadata.product_id,
         ISVSVN: occlum_config.metadata.version_number,
@@ -97,6 +107,10 @@ fn main() {
         },
         MiscSelect: "0".to_string(),
         MiscMask: "0xFFFFFFFF".to_string(),
+        ReservedMemMaxSize: user_space_size.unwrap() as u64,
+        ReservedMemMinSize: user_space_size.unwrap() as u64,
+        ReservedMemInitSize: user_space_size.unwrap() as u64,
+        ReservedMemExecutable: 1,
     };
 
     // Generate the enclave configuration
@@ -175,4 +189,8 @@ struct EnclaveConfiguration {
     DisableDebug: u32,
     MiscSelect: String,
     MiscMask: String,
+    ReservedMemMaxSize: u64,
+    ReservedMemMinSize: u64,
+    ReservedMemInitSize: u64,
+    ReservedMemExecutable: u32,
 }