Add support for the encrypted fs image

.github/workflows/demo_test.yml

@@ -37,6 +37,14 @@ jobs:
             cd occlum_instance && SGX_MODE=SIM occlum build;
             occlum run /bin/hello_world"
 
+    - name: C with encrypted image test
+      run: docker exec language_support_test bash -c "cd /root/occlum/demos/hello_c && make;
+            rm -rf occlum_instance && occlum new occlum_instance;
+            echo \"ff-2a-f9-29-ce-6d-95-04-93-70-6e-83-64-1b-d6-0c\" > occlum_instance/image_key;
+            cp hello_world occlum_instance/image/bin;
+            cd occlum_instance && SGX_MODE=SIM occlum build --image-key ./image_key;
+            occlum run /bin/hello_world"
+
     - name: C++ test
       run: docker exec language_support_test bash -c "cd /root/occlum/demos/hello_cc && make;
             occlum new occlum_instance;

.github/workflows/hw_mode_test.yml

@@ -181,6 +181,14 @@ jobs:
             cd occlum_instance && occlum build;
             occlum run /bin/hello_world"
 
+    - name: C with encrypted image test
+      run: docker exec $language_support_test bash -c "cd /root/occlum/demos/hello_c && make;
+            rm -rf occlum_instance && occlum new occlum_instance;
+            echo \"ff-2a-f9-29-ce-6d-95-04-93-70-6e-83-64-1b-d6-0c\" > occlum_instance/image_key;
+            cp hello_world occlum_instance/image/bin;
+            cd occlum_instance && occlum build --image-key ./image_key;
+            occlum run /bin/hello_world"
+
     - name: C++ test
       run: docker exec $language_support_test bash -c "cd /root/occlum/demos/hello_cc && make;
             occlum new occlum_instance;

README.md

@@ -128,8 +128,7 @@ Occlum can be configured easily via a configuration file named `Occlum.json`, wh
     },
     // Mount points and their file systems
     //
-    // Limitation: configuring mount points by modifying this config file is not
+    // The default configuration is shown below.
-    // supported at the moment. The default configuration is shown below.
     "mount": [
         {
             "target": "/",
@@ -139,14 +138,15 @@ Occlum can be configured easily via a configuration file named `Occlum.json`, wh
                     {
                         "target": "/",
                         "type": "sefs",
-                        "source": "./image",
+                        "source": "./build/mount/__ROOT",
                         "options": {
-                            "integrity_only": true
+                            "MAC": ""
                         }
                     },
                     {
                         "target": "/",
-                        "type": "sefs"
+                        "type": "sefs",
+                        "source": "./run/mount/__ROOT"
                     }
                 ]
             }
@@ -156,17 +156,18 @@ Occlum can be configured easily via a configuration file named `Occlum.json`, wh
             "type": "hostfs",
             "source": "."
         },
-        {
-            "target": "/proc",
-            "type": "procfs"
-        },
         {
             "target": "/tmp",
             "type": "sefs",
+            "source": "./run/mount/tmp",
             "options": {
                 "temporary": true
             }
         },
+        {
+            "target": "/proc",
+            "type": "procfs"
+        },
         {
             "target": "/dev",
             "type": "devfs"

deps/sefs

@@ -1 +1 @@
-Subproject commit 45cb68d334fc97921fcf8d4ab6c9f0ba8d4dea1a
+Subproject commit 88784548d5db1156f2d9e71f968d0d78fa7957db

etc/template/Occlum.json

@@ -1,8 +1,8 @@
 {
     "resource_limits": {
-        "user_space_size": "300MB",
         "kernel_space_heap_size": "32MB",
         "kernel_space_stack_size": "1MB",
+        "user_space_size": "300MB",
         "max_num_of_threads": 32
     },
     "process": {
@@ -35,14 +35,15 @@
                     {
                         "target": "/",
                         "type": "sefs",
-                        "source": "./image",
+                        "source": "./build/mount/__ROOT",
                         "options": {
-                            "integrity_only": true
+                            "MAC": ""
                         }
                     },
                     {
                         "target": "/",
-                        "type": "sefs"
+                        "type": "sefs",
+                        "source": "./run/mount/__ROOT"
                     }
                 ]
             }
@@ -52,17 +53,18 @@
             "type": "hostfs",
             "source": "."
         },
-        {
-            "target": "/proc",
-            "type": "procfs"
-        },
         {
             "target": "/tmp",
             "type": "sefs",
+            "source": "./run/mount/tmp",
             "options": {
                 "temporary": true
             }
         },
+        {
+            "target": "/proc",
+            "type": "procfs"
+        },
         {
             "target": "/dev",
             "type": "devfs"

src/libos/src/config.rs

@@ -9,34 +9,10 @@ use std::sgxfs::SgxFile;
 
 lazy_static! {
     pub static ref LIBOS_CONFIG: Config = {
-        fn load_config(config_path: &str) -> Result<Config> {
+        let config_path =
-            let mut config_file = {
+            unsafe { format!("{}{}", INSTANCE_DIR, "/build/.Occlum_sys.json.protected") };
-                let config_file =
+        let expected_mac = conf_get_hardcoded_file_mac();
-                    SgxFile::open_integrity_only(config_path).map_err(|e| errno!(e))?;
+        match load_config(&config_path, &expected_mac) {
-
-                let actual_mac = config_file.get_mac().map_err(|e| errno!(e))?;
-                let expected_mac = conf_get_hardcoded_file_mac();
-                if actual_mac != expected_mac {
-                    return_errno!(EINVAL, "unexpected file MAC");
-                }
-
-                config_file
-            };
-            let config_json = {
-                let mut config_json = String::new();
-                config_file
-                    .read_to_string(&mut config_json)
-                    .map_err(|e| errno!(e))?;
-                config_json
-            };
-            let config_input: InputConfig =
-                serde_json::from_str(&config_json).map_err(|e| errno!(e))?;
-            let config = Config::from_input(&config_input)
-                .cause_err(|e| errno!(EINVAL, "invalid config JSON"))?;
-            Ok(config)
-        }
-        let config_path = unsafe { format!("{}{}", INSTANCE_DIR, "/build/Occlum.json.protected") };
-        match load_config(&config_path) {
             Err(e) => {
                 error!("failed to load config: {}", e.backtrace());
                 panic!();
@@ -46,6 +22,28 @@ lazy_static! {
     };
 }
 
+pub fn load_config(config_path: &str, expected_mac: &sgx_aes_gcm_128bit_tag_t) -> Result<Config> {
+    let mut config_file = {
+        let config_file = SgxFile::open_integrity_only(config_path).map_err(|e| errno!(e))?;
+        let actual_mac = config_file.get_mac().map_err(|e| errno!(e))?;
+        if actual_mac != *expected_mac {
+            return_errno!(EINVAL, "unexpected file MAC");
+        }
+        config_file
+    };
+    let config_json = {
+        let mut config_json = String::new();
+        config_file
+            .read_to_string(&mut config_json)
+            .map_err(|e| errno!(e))?;
+        config_json
+    };
+    let config_input: InputConfig = serde_json::from_str(&config_json).map_err(|e| errno!(e))?;
+    let config =
+        Config::from_input(&config_input).cause_err(|e| errno!(EINVAL, "invalid config JSON"))?;
+    Ok(config)
+}
+
 // This value will be modified during occlum build
 #[no_mangle]
 #[link_section = ".builtin_config"]
@@ -131,7 +129,6 @@ pub enum ConfigMountFsType {
 
 #[derive(Debug)]
 pub struct ConfigMountOptions {
-    pub integrity_only: bool,
     pub mac: Option<sgx_aes_gcm_128bit_tag_t>,
     pub layers: Option<Vec<ConfigMount>>,
     pub temporary: bool,
@@ -240,13 +237,10 @@ impl ConfigMount {
 
 impl ConfigMountOptions {
     fn from_input(input: &InputConfigMountOptions) -> Result<ConfigMountOptions> {
-        let (integrity_only, mac) = if !input.integrity_only {
+        let mac = if input.mac.is_some() {
-            (false, None)
+            Some(parse_mac(&input.mac.as_ref().unwrap())?)
         } else {
-            if input.mac.is_none() {
+            None
-                return_errno!(EINVAL, "MAC is expected");
-            }
-            (true, Some(parse_mac(&input.mac.as_ref().unwrap())?))
         };
         let layers = if let Some(layers) = &input.layers {
             let layers = layers
@@ -258,7 +252,6 @@ impl ConfigMountOptions {
             None
         };
         Ok(ConfigMountOptions {
-            integrity_only,
             mac,
             layers,
             temporary: input.temporary,
@@ -391,8 +384,6 @@ struct InputConfigMount {
 #[derive(Deserialize, Debug, Default)]
 #[serde(deny_unknown_fields)]
 struct InputConfigMountOptions {
-    #[serde(default)]
-    pub integrity_only: bool,
     #[serde(rename = "MAC")]
     #[serde(default)]
     pub mac: Option<String>,

src/libos/src/entry.rs

@@ -266,7 +266,7 @@ fn do_exec_thread(libos_tid: pid_t, host_tid: pid_t) -> Result<i32> {
     // sync file system
     // TODO: only sync when all processes exit
     use rcore_fs::vfs::FileSystem;
-    crate::fs::ROOT_INODE.fs().sync()?;
+    crate::fs::ROOT_INODE.read().unwrap().fs().sync()?;
 
     // Not to be confused with the return value of a main function.
     // The exact meaning of status is described in wait(2) man page.

src/libos/src/fs/fs_ops/mod.rs

@@ -2,8 +2,10 @@ use super::*;
 
 pub use self::chdir::do_chdir;
 pub use self::getcwd::do_getcwd;
+pub use self::mount::do_mount_rootfs;
 pub use self::sync::do_sync;
 
 mod chdir;
 mod getcwd;
+mod mount;
 mod sync;

src/libos/src/fs/fs_ops/mount.rs

@@ -0,0 +1,30 @@
+use std::sync::Once;
+
+use super::rootfs::{mount_nonroot_fs_according_to, open_root_fs_according_to};
+use super::*;
+
+lazy_static! {
+    static ref MOUNT_ONCE: Once = Once::new();
+}
+
+pub fn do_mount_rootfs(
+    mount_configs: &Vec<ConfigMount>,
+    user_key: &Option<sgx_key_128bit_t>,
+) -> Result<()> {
+    debug!("mount rootfs");
+
+    if MOUNT_ONCE.is_completed() {
+        return_errno!(EPERM, "rootfs cannot be mounted more than once");
+    }
+    let new_root_inode = {
+        let rootfs = open_root_fs_according_to(mount_configs, user_key)?;
+        rootfs.root_inode()
+    };
+    mount_nonroot_fs_according_to(&new_root_inode, mount_configs, user_key)?;
+    MOUNT_ONCE.call_once(|| {
+        let mut root_inode = ROOT_INODE.write().unwrap();
+        root_inode.fs().sync().expect("failed to sync old rootfs");
+        *root_inode = new_root_inode;
+    });
+    Ok(())
+}

src/libos/src/fs/fs_ops/sync.rs

@@ -2,6 +2,6 @@ use super::*;
 
 pub fn do_sync() -> Result<()> {
     debug!("sync:");
-    ROOT_INODE.fs().sync()?;
+    ROOT_INODE.read().unwrap().fs().sync()?;
     Ok(())
 }

src/libos/src/fs/fs_view.rs

@@ -162,12 +162,17 @@ impl FsView {
         if path.len() > 0 && path.as_bytes()[0] == b'/' {
             // absolute path
             let abs_path = path.trim_start_matches('/');
-            let inode = ROOT_INODE.lookup_follow(abs_path, MAX_SYMLINKS)?;
+            let inode = ROOT_INODE
+                .read()
+                .unwrap()
+                .lookup_follow(abs_path, MAX_SYMLINKS)?;
             Ok(inode)
         } else {
             // relative path
             let cwd = self.cwd().trim_start_matches('/');
             let inode = ROOT_INODE
+                .read()
+                .unwrap()
                 .lookup_follow(cwd, MAX_SYMLINKS)?
                 .lookup_follow(path, MAX_SYMLINKS)?;
             Ok(inode)

src/libos/src/fs/mod.rs

@@ -10,6 +10,8 @@ use std::mem::MaybeUninit;
 use std::path::Path;
 use untrusted::{SliceAsMutPtrAndLen, SliceAsPtrAndLen};
 
+use crate::config::ConfigMount;
+
 pub use self::event_file::{AsEvent, EventCreationFlags, EventFile};
 pub use self::events::{AtomicIoEvents, IoEvents, IoNotifier};
 pub use self::file::{File, FileRef};

src/libos/src/fs/rootfs.rs

@@ -3,8 +3,9 @@ use super::hostfs::HostFS;
 use super::procfs::ProcFS;
 use super::sefs::{SgxStorage, SgxUuidProvider};
 use super::*;
-use config::{ConfigMount, ConfigMountFsType};
+use config::ConfigMountFsType;
 use std::path::{Path, PathBuf};
+use std::untrusted::path::PathEx;
 
 use rcore_fs_mountfs::{MNode, MountFS};
 use rcore_fs_ramfs::RamFS;
@@ -14,83 +15,71 @@ use rcore_fs_unionfs::UnionFS;
 
 lazy_static! {
     /// The root of file system
-    pub static ref ROOT_INODE: Arc<dyn INode> = {
+    pub static ref ROOT_INODE: RwLock<Arc<dyn INode>> = {
         fn init_root_inode() -> Result<Arc<dyn INode>> {
             let mount_config = &config::LIBOS_CONFIG.mount;
             let root_inode = {
-                let rootfs = open_root_fs_according_to(mount_config)?;
+                let rootfs = open_root_fs_according_to(mount_config, &None)?;
                 rootfs.root_inode()
             };
-            mount_nonroot_fs_according_to(mount_config, &root_inode)?;
+            mount_nonroot_fs_according_to(&root_inode, mount_config, &None)?;
             Ok(root_inode)
         }
 
-        init_root_inode().unwrap_or_else(|e| {
+        let root_inode = init_root_inode().unwrap_or_else(|e| {
             error!("failed to init root inode: {}", e.backtrace());
             panic!();
-        })
+        });
+        RwLock::new(root_inode)
     };
 }
 
-fn open_root_fs_according_to(mount_configs: &Vec<ConfigMount>) -> Result<Arc<MountFS>> {
+pub fn open_root_fs_according_to(
-    let mount_config = mount_configs
+    mount_configs: &Vec<ConfigMount>,
+    user_key: &Option<sgx_key_128bit_t>,
+) -> Result<Arc<MountFS>> {
+    let root_mount_config = mount_configs
         .iter()
         .find(|m| m.target == Path::new("/") && m.type_ == ConfigMountFsType::TYPE_UNIONFS)
         .ok_or_else(|| errno!(Errno::ENOENT, "the root UnionFS is not valid"))?;
-    if mount_config.options.layers.is_none() {
+    if root_mount_config.options.layers.is_none() {
-        return_errno!(EINVAL, "The root UnionFS must be given the layers");
+        return_errno!(EINVAL, "the root UnionFS must be given the layers");
     }
-    let layer_mount_configs = mount_config.options.layers.as_ref().unwrap();
+    let layer_mount_configs = root_mount_config.options.layers.as_ref().unwrap();
     // image SEFS in layers
-    let (root_image_sefs_mac, root_image_sefs_source) = {
+    let root_image_sefs_mount_config = layer_mount_configs
-        let mount_config = layer_mount_configs
+        .iter()
-            .iter()
+        .find(|m| {
-            .find(|m| m.type_ == ConfigMountFsType::TYPE_SEFS && m.options.integrity_only)
+            m.target == Path::new("/")
-            .ok_or_else(|| errno!(Errno::ENOENT, "the image SEFS in layers is not valid"))?;
+                && m.type_ == ConfigMountFsType::TYPE_SEFS
-        (
+                && m.options.mac.is_some()
-            mount_config.options.mac,
+        })
-            mount_config.source.as_ref().unwrap(),
+        .ok_or_else(|| errno!(Errno::ENOENT, "the image SEFS in layers is not valid"))?;
-        )
+    let root_image_sefs =
-    };
+        open_or_create_sefs_according_to(&root_image_sefs_mount_config, user_key)?;
-    let root_image_sefs = SEFS::open(
-        Box::new(SgxStorage::new(
-            root_image_sefs_source,
-            true,
-            root_image_sefs_mac,
-        )),
-        &time::OcclumTimeProvider,
-        &SgxUuidProvider,
-    )?;
     // container SEFS in layers
-    let root_container_sefs_source = {
+    let root_container_sefs_mount_config = layer_mount_configs
-        let mount_config = layer_mount_configs
+        .iter()
-            .iter()
+        .find(|m| {
-            .find(|m| m.type_ == ConfigMountFsType::TYPE_SEFS && !m.options.integrity_only)
+            m.target == Path::new("/")
-            .ok_or_else(|| errno!(Errno::ENOENT, "the container SEFS in layers is not valid"))?;
+                && m.type_ == ConfigMountFsType::TYPE_SEFS
-        mount_config.source.as_ref().unwrap()
+                && m.options.mac.is_none()
-    };
+        })
-    let root_container_sefs = {
+        .ok_or_else(|| errno!(Errno::ENOENT, "the container SEFS in layers is not valid"))?;
-        SEFS::open(
+    let root_container_sefs =
-            Box::new(SgxStorage::new(root_container_sefs_source, false, None)),
+        open_or_create_sefs_according_to(&root_container_sefs_mount_config, user_key)?;
-            &time::OcclumTimeProvider,
+    // create UnionFS
-            &SgxUuidProvider,
-        )
-    }
-    .or_else(|_| {
-        SEFS::create(
-            Box::new(SgxStorage::new(root_container_sefs_source, false, None)),
-            &time::OcclumTimeProvider,
-            &SgxUuidProvider,
-        )
-    })?;
-
     let root_unionfs = UnionFS::new(vec![root_container_sefs, root_image_sefs])?;
     let root_mountable_unionfs = MountFS::new(root_unionfs);
     Ok(root_mountable_unionfs)
 }
 
-fn mount_nonroot_fs_according_to(mount_config: &Vec<ConfigMount>, root: &MNode) -> Result<()> {
+pub fn mount_nonroot_fs_according_to(
-    for mc in mount_config {
+    root: &MNode,
+    mount_configs: &Vec<ConfigMount>,
+    user_key: &Option<sgx_key_128bit_t>,
+) -> Result<()> {
+    for mc in mount_configs {
         if mc.target == Path::new("/") {
             continue;
         }
@@ -102,35 +91,7 @@ fn mount_nonroot_fs_according_to(mount_config: &Vec<ConfigMount>, root: &MNode)
         use self::ConfigMountFsType::*;
         match mc.type_ {
             TYPE_SEFS => {
-                if mc.options.integrity_only {
+                let sefs = open_or_create_sefs_according_to(&mc, user_key)?;
-                    return_errno!(EINVAL, "Cannot mount integrity-only SEFS at non-root path");
-                }
-                if mc.source.is_none() {
-                    return_errno!(EINVAL, "Source is expected for SEFS");
-                }
-                let source_path = mc.source.as_ref().unwrap();
-                let sefs = if !mc.options.temporary {
-                    {
-                        SEFS::open(
-                            Box::new(SgxStorage::new(source_path, false, None)),
-                            &time::OcclumTimeProvider,
-                            &SgxUuidProvider,
-                        )
-                    }
-                    .or_else(|_| {
-                        SEFS::create(
-                            Box::new(SgxStorage::new(source_path, false, None)),
-                            &time::OcclumTimeProvider,
-                            &SgxUuidProvider,
-                        )
-                    })?
-                } else {
-                    SEFS::create(
-                        Box::new(SgxStorage::new(source_path, false, None)),
-                        &time::OcclumTimeProvider,
-                        &SgxUuidProvider,
-                    )?
-                };
                 mount_fs_at(sefs, root, &mc.target)?;
             }
             TYPE_HOSTFS => {
@@ -179,3 +140,47 @@ fn mount_fs_at(fs: Arc<dyn FileSystem>, parent_inode: &MNode, abs_path: &Path) -
     mount_dir.mount(fs);
     Ok(())
 }
+
+fn open_or_create_sefs_according_to(
+    mc: &ConfigMount,
+    user_key: &Option<sgx_key_128bit_t>,
+) -> Result<Arc<SEFS>> {
+    assert!(mc.type_ == ConfigMountFsType::TYPE_SEFS);
+
+    if mc.source.is_none() {
+        return_errno!(EINVAL, "Source is expected for SEFS");
+    }
+    if mc.options.temporary && mc.options.mac.is_some() {
+        return_errno!(EINVAL, "Integrity protected SEFS cannot be temporary");
+    }
+    let source_path = mc.source.as_ref().unwrap();
+    let root_mac = mc.options.mac;
+    let sefs = if !mc.options.temporary {
+        if root_mac.is_some() {
+            SEFS::open(
+                Box::new(SgxStorage::new(source_path, user_key, &root_mac)),
+                &time::OcclumTimeProvider,
+                &SgxUuidProvider,
+            )?
+        } else if source_path.join("metadata").exists() {
+            SEFS::open(
+                Box::new(SgxStorage::new(source_path, user_key, &root_mac)),
+                &time::OcclumTimeProvider,
+                &SgxUuidProvider,
+            )?
+        } else {
+            SEFS::create(
+                Box::new(SgxStorage::new(source_path, user_key, &root_mac)),
+                &time::OcclumTimeProvider,
+                &SgxUuidProvider,
+            )?
+        }
+    } else {
+        SEFS::create(
+            Box::new(SgxStorage::new(source_path, user_key, &root_mac)),
+            &time::OcclumTimeProvider,
+            &SgxUuidProvider,
+        )?
+    };
+    Ok(sefs)
+}

src/libos/src/fs/sefs/mod.rs

@@ -1,4 +1,4 @@
-use super::sgx_aes_gcm_128bit_tag_t;
+use super::{sgx_aes_gcm_128bit_tag_t, sgx_key_128bit_t};
 
 pub use self::sgx_storage::SgxStorage;
 pub use self::sgx_uuid_provider::SgxUuidProvider;

src/libos/src/fs/sefs/sgx_storage.rs

@@ -28,23 +28,21 @@ macro_rules! convert_result {
 
 pub struct SgxStorage {
     path: PathBuf,
-    integrity_only: bool,
+    encrypt_mode: EncryptMode,
     file_cache: Mutex<BTreeMap<u64, LockedFile>>,
-    root_mac: Option<sgx_aes_gcm_128bit_tag_t>,
 }
 
 impl SgxStorage {
     pub fn new(
         path: impl AsRef<Path>,
-        integrity_only: bool,
+        key: &Option<sgx_key_128bit_t>,
-        file_mac: Option<sgx_aes_gcm_128bit_tag_t>,
+        root_mac: &Option<sgx_aes_gcm_128bit_tag_t>,
     ) -> Self {
         // assert!(path.as_ref().is_dir());
         SgxStorage {
             path: path.as_ref().to_path_buf(),
-            integrity_only: integrity_only,
+            encrypt_mode: EncryptMode::new(key, root_mac),
             file_cache: Mutex::new(BTreeMap::new()),
-            root_mac: file_mac,
         }
     }
     /// Get file by `file_id`.
@@ -84,15 +82,6 @@ impl SgxStorage {
     ) -> Result<LockedFile> {
         open_fn(self)
     }
-
-    /// Set the expected root MAC of the SGX storage.
-    ///
-    /// By giving this root MAC, we can be sure that the root file (file_id = 0) opened
-    /// by the storage has a MAC that is equal to the given root MAC.
-    pub fn set_root_mac(&mut self, mac: sgx_aes_gcm_128bit_tag_t) -> Result<()> {
-        self.root_mac = Some(mac);
-        Ok(())
-    }
 }
 
 impl Storage for SgxStorage {
@@ -105,19 +94,21 @@ impl Storage for SgxStorage {
                 options.read(true).update(true);
                 options
             };
-            let file = if !self.integrity_only {
+            let file = match self.encrypt_mode {
-                options.open(path)?
+                EncryptMode::IntegrityOnly(_) => options.open_integrity_only(path)?,
-            } else {
+                EncryptMode::EncryptWithIntegrity(key, _) | EncryptMode::Encrypt(key) => {
-                options.open_integrity_only(path)?
+                    options.open_ex(path, &key)?
+                }
+                EncryptMode::EncryptAutoKey => options.open(path)?,
             };
 
             // Check the MAC of the root file against the given root MAC of the storage
-            if file_id == "metadata" && self.root_mac.is_some() {
+            if file_id == "metadata" && self.protect_integrity() {
                 let root_file_mac = file.get_mac().expect("Failed to get mac");
-                if root_file_mac != self.root_mac.unwrap() {
+                if root_file_mac != self.encrypt_mode.root_mac().unwrap() {
                     error!(
                         "MAC validation for metadata file failed: expected = {:#?}, found = {:?}",
-                        self.root_mac.unwrap(),
+                        self.encrypt_mode.root_mac().unwrap(),
                         root_file_mac
                     );
                     return_errno!(EACCES);
@@ -138,10 +129,12 @@ impl Storage for SgxStorage {
                 options.write(true).update(true);
                 options
             };
-            let file = if !self.integrity_only {
+            let file = match self.encrypt_mode {
-                options.open(path)?
+                EncryptMode::IntegrityOnly(_) => options.open_integrity_only(path)?,
-            } else {
+                EncryptMode::EncryptWithIntegrity(key, _) | EncryptMode::Encrypt(key) => {
-                options.open_integrity_only(path)?
+                    options.open_ex(path, &key)?
+                }
+                EncryptMode::EncryptAutoKey => options.open(path)?,
             };
             Ok(LockedFile(Arc::new(Mutex::new(file))))
         })?;
@@ -161,8 +154,11 @@ impl Storage for SgxStorage {
         })
     }
 
-    fn is_integrity_only(&self) -> bool {
+    fn protect_integrity(&self) -> bool {
-        self.integrity_only
+        match self.encrypt_mode {
+            EncryptMode::IntegrityOnly(_) | EncryptMode::EncryptWithIntegrity(_, _) => true,
+            _ => false,
+        }
     }
 
     fn clear(&self) -> DevResult<()> {
@@ -179,6 +175,36 @@ impl Storage for SgxStorage {
     }
 }
 
+enum EncryptMode {
+    IntegrityOnly(sgx_aes_gcm_128bit_tag_t),
+    EncryptWithIntegrity(sgx_key_128bit_t, sgx_aes_gcm_128bit_tag_t),
+    Encrypt(sgx_key_128bit_t),
+    EncryptAutoKey,
+}
+
+impl EncryptMode {
+    pub fn new(
+        key: &Option<sgx_key_128bit_t>,
+        root_mac: &Option<sgx_aes_gcm_128bit_tag_t>,
+    ) -> Self {
+        match (key, root_mac) {
+            (Some(key), Some(root_mac)) => Self::EncryptWithIntegrity(*key, *root_mac),
+            (Some(key), None) => Self::Encrypt(*key),
+            (None, Some(root_mac)) => Self::IntegrityOnly(*root_mac),
+            (None, None) => Self::EncryptAutoKey,
+        }
+    }
+
+    pub fn root_mac(&self) -> Option<sgx_aes_gcm_128bit_tag_t> {
+        match self {
+            Self::IntegrityOnly(root_mac) | Self::EncryptWithIntegrity(_, root_mac) => {
+                Some(*root_mac)
+            }
+            _ => None,
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct LockedFile(Arc<Mutex<SgxFile>>);
 

src/libos/src/fs/syscalls.rs

@@ -518,3 +518,23 @@ pub fn do_ioctl(fd: FileDesc, cmd: u32, argp: *mut u8) -> Result<isize> {
     file_ops::do_ioctl(fd, &mut ioctl_cmd)?;
     Ok(0)
 }
+
+pub fn do_mount_rootfs(
+    key_ptr: *const sgx_key_128bit_t,
+    occlum_json_mac_ptr: *const sgx_aes_gcm_128bit_tag_t,
+) -> Result<isize> {
+    let key = if key_ptr.is_null() {
+        None
+    } else {
+        Some(unsafe { key_ptr.read() })
+    };
+    if occlum_json_mac_ptr.is_null() {
+        return_errno!(EINVAL, "occlum_json_mac_ptr cannot be null");
+    }
+    let expected_occlum_json_mac = unsafe { occlum_json_mac_ptr.read() };
+    let user_config_path = unsafe { format!("{}{}", INSTANCE_DIR, "/build/Occlum.json.protected") };
+    let user_mount_config =
+        config::load_config(&user_config_path, &expected_occlum_json_mac)?.mount;
+    fs_ops::do_mount_rootfs(&user_mount_config, &key)?;
+    Ok(0)
+}

src/libos/src/process/do_spawn/mod.rs

@@ -10,7 +10,6 @@ use super::thread::ThreadName;
 use super::{table, task, ProcessRef, ThreadRef};
 use crate::fs::{
     CreationFlags, File, FileDesc, FileTable, FsView, HostStdioFds, StdinFile, StdoutFile,
-    ROOT_INODE,
 };
 use crate::prelude::*;
 use crate::vm::ProcessVM;

src/libos/src/syscall/mod.rs

@@ -26,10 +26,10 @@ use crate::fs::{
     do_eventfd2, do_faccessat, do_fchmod, do_fchmodat, do_fchown, do_fchownat, do_fcntl,
     do_fdatasync, do_fstat, do_fstatat, do_fsync, do_ftruncate, do_getcwd, do_getdents,
     do_getdents64, do_ioctl, do_lchown, do_link, do_linkat, do_lseek, do_lstat, do_mkdir,
-    do_mkdirat, do_open, do_openat, do_pipe, do_pipe2, do_pread, do_pwrite, do_read, do_readlink,
+    do_mkdirat, do_mount_rootfs, do_open, do_openat, do_pipe, do_pipe2, do_pread, do_pwrite,
-    do_readlinkat, do_readv, do_rename, do_renameat, do_rmdir, do_sendfile, do_stat, do_symlink,
+    do_read, do_readlink, do_readlinkat, do_readv, do_rename, do_renameat, do_rmdir, do_sendfile,
-    do_symlinkat, do_sync, do_truncate, do_unlink, do_unlinkat, do_write, do_writev, iovec_t, File,
+    do_stat, do_symlink, do_symlinkat, do_sync, do_truncate, do_unlink, do_unlinkat, do_write,
-    FileDesc, FileRef, HostStdioFds, Stat,
+    do_writev, iovec_t, File, FileDesc, FileRef, HostStdioFds, Stat,
 };
 use crate::interrupt::{do_handle_interrupt, sgx_interrupt_info_t};
 use crate::misc::{resource_t, rlimit_t, sysinfo_t, utsname_t};
@@ -416,6 +416,7 @@ macro_rules! process_syscall_table_with_callback {
             (SpawnMusl = 360) => do_spawn_for_musl(child_pid_ptr: *mut u32, path: *const i8, argv: *const *const i8, envp: *const *const i8, fdop_list: *const FdOp),
             (HandleException = 361) => do_handle_exception(info: *mut sgx_exception_info_t, fpregs: *mut FpRegs, context: *mut CpuContext),
             (HandleInterrupt = 362) => do_handle_interrupt(info: *mut sgx_interrupt_info_t, fpregs: *mut FpRegs, context: *mut CpuContext),
+            (MountRootFS = 363) => do_mount_rootfs(key_ptr: *const sgx_key_128bit_t, occlum_json_mac_ptr: *const sgx_aes_gcm_128bit_tag_t),
         }
     };
 }

src/pal/src/pal_api.c

@@ -14,6 +14,53 @@ int occlum_pal_get_version(void) {
     return OCCLUM_PAL_VERSION;
 }
 
+int pal_run_init_process() {
+    const char *init_path = "/bin/init";
+    const char *init_argv[2] = {
+        "init",
+        NULL,
+    };
+    struct occlum_stdio_fds init_io_fds = {
+        .stdin_fd = STDIN_FILENO,
+        .stdout_fd = STDOUT_FILENO,
+        .stderr_fd = STDERR_FILENO,
+    };
+    int libos_tid = 0;
+    struct occlum_pal_create_process_args init_process_args = {
+        .path = init_path,
+        .argv = init_argv,
+        .env = NULL,
+        .stdio = &init_io_fds,
+        .pid = &libos_tid,
+    };
+    if (occlum_pal_create_process(&init_process_args) < 0) {
+        return -1;
+    }
+
+    int exit_status = 0;
+    struct occlum_pal_exec_args init_exec_args = {
+        .pid = libos_tid,
+        .exit_value = &exit_status,
+    };
+    if (occlum_pal_exec(&init_exec_args) < 0) {
+        return -1;
+    }
+
+    // Convert the exit status to a value in a shell-like encoding
+    if (WIFEXITED(exit_status)) { // terminated normally
+        exit_status = WEXITSTATUS(exit_status) & 0x7F; // [0, 127]
+    } else { // killed by signal
+        exit_status = 128 + WTERMSIG(exit_status); // [128 + 1, 128 + 64]
+    }
+    if (exit_status != 0) {
+        errno = EINVAL;
+        PAL_ERROR("The init process exit with code: %d", exit_status);
+        return -1;
+    }
+
+    return 0;
+}
+
 int occlum_pal_init(const struct occlum_pal_attr *attr) {
     if (attr == NULL) {
         errno = EINVAL;
@@ -68,6 +115,11 @@ int occlum_pal_init(const struct occlum_pal_attr *attr) {
     }
 #endif
 
+    if (pal_run_init_process() < 0) {
+        PAL_ERROR("Failed to run the init process: %s", errno2str(errno));
+        goto on_destroy_enclave;
+    }
+
     return 0;
 on_destroy_enclave:
     if (pal_destroy_enclave() < 0) {

test/Occlum.json

@@ -38,14 +38,15 @@
                     {
                         "target": "/",
                         "type": "sefs",
-                        "source": "./image",
+                        "source": "./build/mount/__ROOT",
                         "options": {
-                            "integrity_only": true
+                            "MAC": ""
                         }
                     },
                     {
                         "target": "/",
-                        "type": "sefs"
+                        "type": "sefs",
+                        "source": "./run/mount/__ROOT"
                     }
                 ]
             }
@@ -55,17 +56,18 @@
             "type": "hostfs",
             "source": "."
         },
-        {
-            "target": "/proc",
-            "type": "procfs"
-        },
         {
             "target": "/tmp",
             "type": "sefs",
+            "source": "./run/mount/tmp",
             "options": {
                 "temporary": true
             }
         },
+        {
+            "target": "/proc",
+            "type": "procfs"
+        },
         {
             "target": "/dev",
             "type": "devfs"

test/env/main.c

@@ -209,7 +209,7 @@ int main(int argc, const char *argv[]) {
     g_argc = argc;
     g_argv = argv;
     // Test argc
-    if (getpid() > 1) {
+    if (getpid() > 2) {
         return test_suite_run(child_test_cases, ARRAY_SIZE(child_test_cases));
     } else  {
         return test_suite_run(test_cases, ARRAY_SIZE(test_cases));

tools/Makefile

@@ -9,6 +9,7 @@ all:
 	@ln -s -f ../../tools/occlum ../$(BUILD_DIR)/bin/occlum
 	@$(MAKE) --no-print-directory -C protect-integrity
 	@$(MAKE) --no-print-directory -C gen_internal_conf
+	@$(MAKE) --no-print-directory -C init
 
 format:
 	@$(MAKE) --no-print-directory -C protect-integrity format
@@ -19,3 +20,4 @@ format-check:
 clean:
 	@$(MAKE) --no-print-directory -C protect-integrity clean
 	@$(MAKE) --no-print-directory -C gen_internal_conf clean
+	@$(MAKE) --no-print-directory -C init clean

tools/gen_internal_conf/src/main.rs

@@ -6,7 +6,7 @@ extern crate serde;
 extern crate serde_derive;
 extern crate serde_xml_rs;
 
-use clap::{App, Arg};
+use clap::{App, Arg, SubCommand};
 use log::debug;
 use serde_derive::{Deserialize, Serialize};
 use serde_json::json;
@@ -14,10 +14,6 @@ use std::fs::File;
 use std::io::Write;
 use std::path::Path;
 
-// This is not really used anymore.
-// Just keep it here in case we need it in the future.
-const OCCLUM_INSTANCE_DIR: &str = ".";
-
 fn main() {
     env_logger::init();
 
@@ -27,48 +23,76 @@ fn main() {
         .arg(
             Arg::with_name("user_json")
                 .long("user_json")
-                .value_name("input: user_json")
+                .value_name("input user json")
                 .required(true)
                 .validator(|f| match Path::new(&f).exists() {
                     true => Ok(()),
                     false => {
-                        let err_message = String::from(f) + " is not exist";
+                        let err_message = f + " is not exist";
                         Err(err_message)
                     }
                 })
                 .takes_value(true),
         )
-        // Input: Secure Occlum FS image MAC
+        .subcommand(
-        .arg(
+            SubCommand::with_name("gen_user_conf")
-            Arg::with_name("fs_mac")
+                .about("Generate user image config")
-                .long("fs_mac")
+                // Input: User's Secure Occlum FS image MAC
-                .value_name("input: fs_mac")
+                .arg(
-                .required(true)
+                    Arg::with_name("user_fs_mac")
-                .takes_value(true),
+                        .long("user_fs_mac")
+                        .value_name("input MAC of user image fs")
+                        .required(true)
+                        .takes_value(true),
+                )
+                // Output: JSON file used by libOS and users shouldn't touch
+                .arg(
+                    Arg::with_name("output_user_json")
+                        .long("output_user_json")
+                        .value_name("output user json")
+                        .required(true)
+                        .validator(|f| match File::create(f) {
+                            Ok(_) => Ok(()),
+                            Err(e) => Err(e.to_string()),
+                        })
+                        .takes_value(true),
+                )
+                // Output: XML file used by Intel SGX SDK
+                .arg(
+                    Arg::with_name("sdk_xml")
+                        .long("sdk_xml")
+                        .value_name("output sdk's xml")
+                        .required(true)
+                        .validator(|f| match File::create(f) {
+                            Ok(_e) => Ok(()),
+                            Err(e) => Err(e.to_string()),
+                        })
+                        .takes_value(true),
+                ),
         )
-        // Output: XML file used by Intel SGX SDK
+        .subcommand(
-        .arg(
+            SubCommand::with_name("gen_sys_conf")
-            Arg::with_name("sdk_xml")
+                .about("Generate initfs image config")
-                .long("sdk_xml")
+                // Input: InitFS image MAC
-                .value_name("output: sdk_xml")
+                .arg(
-                .required(true)
+                    Arg::with_name("init_fs_mac")
-                .validator(|f| match File::create(f) {
+                        .long("init_fs_mac")
-                    Ok(_e) => Ok(()),
+                        .value_name("input MAC of init image fs")
-                    Err(e) => Err(e.to_string()),
+                        .required(true)
-                })
+                        .takes_value(true),
-                .takes_value(true),
+                )
-        )
+                // Output: JSON file for initfs and users shouldn't touch
-        // Output: JSON file used by libOS and users shouldn't touch
+                .arg(
-        .arg(
+                    Arg::with_name("sys_json")
-            Arg::with_name("sys_json")
+                        .long("sys_json")
-                .long("sys_json")
+                        .value_name("output sys json")
-                .value_name("output: sys_json")
+                        .required(true)
-                .required(true)
+                        .validator(|f| match File::create(f) {
-                .validator(|f| match File::create(f) {
+                            Ok(_) => Ok(()),
-                    Ok(_) => Ok(()),
+                            Err(e) => Err(e.to_string()),
-                    Err(e) => Err(e.to_string()),
+                        })
-                })
+                        .takes_value(true),
-                .takes_value(true),
+                ),
         )
         .get_matches();
 
@@ -77,25 +101,6 @@ fn main() {
         "Occlum config (json) file name {:?}",
         occlum_config_file_path
     );
-
-    let occlum_conf_root_fs_mac = matches.value_of("fs_mac").unwrap();
-    debug!(
-        "Occlum config root FS MAC {:?}",
-        occlum_conf_root_fs_mac
-    );
-
-    let enclave_config_file_path = matches.value_of("sdk_xml").unwrap();
-    debug!(
-        "Enclave config (xml) file name {:?}",
-        enclave_config_file_path
-    );
-
-    let occlum_internal_json_file_path = matches.value_of("sys_json").unwrap();
-    debug!(
-        "Genereated Occlum internal config (json) file name {:?}",
-        occlum_internal_json_file_path
-    );
-
     // Read the occlum configuration file
     let occlum_config_file =
         File::open(occlum_config_file_path).expect("The Occlum configuration file does not exist");
@@ -103,91 +108,151 @@ fn main() {
         .expect("It is not a valid Occlum configuration file.");
     debug!("The occlum config is:{:?}", occlum_config);
 
-    // get the kernel stack size
+    // Match subcommand
-    let stack_max_size = parse_memory_size(&occlum_config.resource_limits.kernel_space_stack_size);
+    if let Some(sub_matches) = matches.subcommand_matches("gen_user_conf") {
-    if stack_max_size.is_err() {
+        let occlum_conf_user_fs_mac = sub_matches.value_of("user_fs_mac").unwrap();
-        println!(
+        debug!("Occlum config user FS MAC {:?}", occlum_conf_user_fs_mac);
-            "The kernel_space_stack_size \"{}\" is not correct.",
+
-            occlum_config.resource_limits.kernel_space_stack_size
+        let occlum_user_json_file_path = sub_matches.value_of("output_user_json").unwrap();
+        debug!(
+            "Genereated Occlum user config (json) file name {:?}",
+            occlum_user_json_file_path
         );
-        return;
-    }
 
-    // get the kernel heap size
+        let enclave_config_file_path = sub_matches.value_of("sdk_xml").unwrap();
-    let heap_max_size = parse_memory_size(&occlum_config.resource_limits.kernel_space_heap_size);
+        debug!(
-    if heap_max_size.is_err() {
+            "Enclave config (xml) file name {:?}",
-        println!(
+            enclave_config_file_path
-            "The kernel_space_heap_size \"{}\" is not correct.",
-            occlum_config.resource_limits.kernel_space_heap_size
         );
-        return;
-    }
 
-    // get the user space size
+        // get the kernel stack size
-    let user_space_size = parse_memory_size(&occlum_config.resource_limits.user_space_size);
+        let stack_max_size =
-    if user_space_size.is_err() {
+            parse_memory_size(&occlum_config.resource_limits.kernel_space_stack_size);
-        println!(
+        if stack_max_size.is_err() {
-            "The user_space_size \"{}\" is not correct.",
+            println!(
-            occlum_config.resource_limits.user_space_size
+                "The kernel_space_stack_size \"{}\" is not correct.",
+                occlum_config.resource_limits.kernel_space_stack_size
+            );
+            return;
+        }
+        // get the kernel heap size
+        let heap_max_size =
+            parse_memory_size(&occlum_config.resource_limits.kernel_space_heap_size);
+        if heap_max_size.is_err() {
+            println!(
+                "The kernel_space_heap_size \"{}\" is not correct.",
+                occlum_config.resource_limits.kernel_space_heap_size
+            );
+            return;
+        }
+        // get the user space size
+        let user_space_size = parse_memory_size(&occlum_config.resource_limits.user_space_size);
+        if user_space_size.is_err() {
+            println!(
+                "The user_space_size \"{}\" is not correct.",
+                occlum_config.resource_limits.user_space_size
+            );
+            return;
+        }
+
+        // Generate the enclave configuration
+        let sgx_enclave_configuration = EnclaveConfiguration {
+            ProdID: occlum_config.metadata.product_id,
+            ISVSVN: occlum_config.metadata.version_number,
+            StackMaxSize: stack_max_size.unwrap() as u64,
+            StackMinSize: stack_max_size.unwrap() as u64, // just use the same size as max size
+            HeapMaxSize: heap_max_size.unwrap() as u64,
+            HeapMinSize: heap_max_size.unwrap() as u64, // just use the same size as max size
+            TCSNum: occlum_config.resource_limits.max_num_of_threads,
+            TCSPolicy: 1,
+            DisableDebug: match occlum_config.metadata.debuggable {
+                true => 0,
+                false => 1,
+            },
+            MiscSelect: "0".to_string(),
+            MiscMask: "0xFFFFFFFF".to_string(),
+            ReservedMemMaxSize: user_space_size.unwrap() as u64,
+            ReservedMemMinSize: user_space_size.unwrap() as u64,
+            ReservedMemInitSize: user_space_size.unwrap() as u64,
+            ReservedMemExecutable: 1,
+        };
+        let enclave_config = serde_xml_rs::to_string(&sgx_enclave_configuration).unwrap();
+        debug!("The enclave config:{:?}", enclave_config);
+
+        // Generate user Occlum.json - "output_user_json"
+        let user_mount_config = {
+            let user_mount_config =
+                gen_user_mount_config(occlum_config.mount, occlum_conf_user_fs_mac.to_string());
+            if user_mount_config.is_err() {
+                println!("Mount configuration invalid: {:?}", user_mount_config);
+                return;
+            }
+            user_mount_config.unwrap()
+        };
+        let user_occlum_json_config = InternalOcclumJson {
+            resource_limits: InternalResourceLimits {
+                user_space_size: occlum_config.resource_limits.user_space_size.to_string(),
+            },
+            process: OcclumProcess {
+                default_stack_size: occlum_config.process.default_stack_size,
+                default_heap_size: occlum_config.process.default_heap_size,
+                default_mmap_size: occlum_config.process.default_mmap_size,
+            },
+            entry_points: occlum_config.entry_points,
+            env: occlum_config.env,
+            mount: serde_json::to_value(user_mount_config).unwrap(),
+        };
+        let user_occlum_json_str = serde_json::to_string_pretty(&user_occlum_json_config).unwrap();
+        debug!("The user Occlum.json config:\n{:?}", user_occlum_json_str);
+
+        // Update the output file
+        let mut enclave_config_file = File::create(enclave_config_file_path)
+            .expect("Could not open the target Enclave configuration file.");
+        enclave_config_file
+            .write_all(enclave_config.as_bytes())
+            .expect("Failed to update the Enclave configuration file.");
+
+        let mut user_occlum_json = File::create(occlum_user_json_file_path)
+            .expect("Could not open the output user Occlum.json file.");
+        user_occlum_json
+            .write_all(user_occlum_json_str.as_bytes())
+            .expect("Failed to update the output user Occlum.json file.");
+    } else if let Some(sub_matches) = matches.subcommand_matches("gen_sys_conf") {
+        let occlum_conf_init_fs_mac = sub_matches.value_of("init_fs_mac").unwrap();
+        debug!("Occlum config init FS MAC {:?}", occlum_conf_init_fs_mac);
+
+        let occlum_sys_json_file_path = sub_matches.value_of("sys_json").unwrap();
+        debug!(
+            "Genereated Occlum sys config (json) file name {:?}",
+            occlum_sys_json_file_path
         );
-        return;
+
+        // Generate sys Occlum.json - "sys_json"
+        let sys_occlum_json_config = InternalOcclumJson {
+            resource_limits: InternalResourceLimits {
+                user_space_size: occlum_config.resource_limits.user_space_size.to_string(),
+            },
+            process: OcclumProcess {
+                default_stack_size: occlum_config.process.default_stack_size,
+                default_heap_size: occlum_config.process.default_heap_size,
+                default_mmap_size: occlum_config.process.default_mmap_size,
+            },
+            entry_points: occlum_config.entry_points,
+            env: occlum_config.env,
+            mount: gen_sys_mount_config(occlum_conf_init_fs_mac.to_string()),
+        };
+
+        // Update the output file
+        let sys_occlum_json_str = serde_json::to_string_pretty(&sys_occlum_json_config).unwrap();
+        debug!("The sys Occlum.json config:\n{:?}", sys_occlum_json_str);
+        let mut sys_occlum_json = File::create(occlum_sys_json_file_path)
+            .expect("Could not open the output sys Occlum.json file.");
+        sys_occlum_json
+            .write_all(sys_occlum_json_str.as_bytes())
+            .expect("Failed to update the output sys Occlum.json file.");
+    } else {
+        unreachable!();
     }
-
-    // Generate the enclave configuration
-    let sgx_enclave_configuration = EnclaveConfiguration {
-        ProdID: occlum_config.metadata.product_id,
-        ISVSVN: occlum_config.metadata.version_number,
-        StackMaxSize: stack_max_size.unwrap() as u64,
-        StackMinSize: stack_max_size.unwrap() as u64, // just use the same size as max size
-        HeapMaxSize: heap_max_size.unwrap() as u64,
-        HeapMinSize: heap_max_size.unwrap() as u64, // just use the same size as max size
-        TCSNum: occlum_config.resource_limits.max_num_of_threads,
-        TCSPolicy: 1,
-        DisableDebug: match occlum_config.metadata.debuggable {
-            true => 0,
-            false => 1,
-        },
-        MiscSelect: "0".to_string(),
-        MiscMask: "0xFFFFFFFF".to_string(),
-        ReservedMemMaxSize: user_space_size.unwrap() as u64,
-        ReservedMemMinSize: user_space_size.unwrap() as u64,
-        ReservedMemInitSize: user_space_size.unwrap() as u64,
-        ReservedMemExecutable: 1,
-    };
-
-    let enclave_config = serde_xml_rs::to_string(&sgx_enclave_configuration).unwrap();
-    debug!("The enclave config:{:?}", enclave_config);
-
-    // Generate internal Occlum.json - "sys_json"
-    let internal_occlum_json_config = InternalOcclumJson {
-        resource_limits: InternalResourceLimits {
-            user_space_size: occlum_config.resource_limits.user_space_size.to_string(),
-        },
-        process: OcclumProcess {
-            default_stack_size: occlum_config.process.default_stack_size,
-            default_heap_size: occlum_config.process.default_heap_size,
-            default_mmap_size: occlum_config.process.default_mmap_size,
-        },
-        entry_points: occlum_config.entry_points,
-        env: occlum_config.env,
-        mount: gen_mount_config(occlum_conf_root_fs_mac.to_string()),
-    };
-    let internal_occlum_json_str =
-        serde_json::to_string_pretty(&internal_occlum_json_config).unwrap();
-    debug!("The internal Occlum.json config:\n{:?}", internal_occlum_json_str);
-
-    // Update the output file
-    let mut enclave_config_file = File::create(enclave_config_file_path)
-        .expect("Could not open the target Enclave configuration file.");
-    enclave_config_file
-        .write_all(enclave_config.as_bytes())
-        .expect("Failed to update the Enclave configuration file.");
-
-    let mut internal_occlum_json = File::create(occlum_internal_json_file_path)
-        .expect("Could not open the internal Occlum.json file.");
-    internal_occlum_json
-        .write_all(internal_occlum_json_str.as_bytes())
-        .expect("Failed to update the internal Occlum.json file.");
 }
 
 fn parse_memory_size(mem_str: &str) -> Result<usize, &str> {
@@ -204,7 +269,7 @@ fn parse_memory_size(mem_str: &str) -> Result<usize, &str> {
     let (mem_unit, unit_factor) = UNIT2FACTOR
         .iter()
         .position(|(mem_unit, _)| mem_str.ends_with(mem_unit))
-        .ok_or_else(|| "No unit")
+        .ok_or("No unit")
         .map(|unit_i| &UNIT2FACTOR[unit_i])?;
 
     // Extract the value part of the memory size
@@ -221,8 +286,38 @@ fn parse_memory_size(mem_str: &str) -> Result<usize, &str> {
     Ok(mem_val * unit_factor)
 }
 
-fn gen_mount_config(occlum_conf_root_fs_mac: String) -> serde_json::Value {
+fn gen_user_mount_config(
-    let mut internal_mount_config: serde_json::Value = json!({
+    mount_conf: Vec<OcclumMount>,
+    occlum_conf_user_fs_mac: String,
+) -> Result<Vec<OcclumMount>, &'static str> {
+    let mut user_mount_config = mount_conf;
+    let root_mc = user_mount_config
+        .iter_mut()
+        .find(|m| m.target == String::from("/") && m.type_ == String::from("unionfs"))
+        .ok_or("the root UnionFS is not valid")?;
+    if root_mc.options.layers.is_none() {
+        return Err("the root UnionFS must be given layers");
+    }
+    let mut root_image_sefs_mc = root_mc
+        .options
+        .layers
+        .as_mut()
+        .unwrap()
+        .iter_mut()
+        .find(|m| {
+            m.target == String::from("/")
+                && m.type_ == String::from("sefs")
+                && m.options.mac.is_some()
+        })
+        .ok_or("the image SEFS in layers is not valid")?;
+    root_image_sefs_mc.options.mac = Some(occlum_conf_user_fs_mac);
+
+    debug!("user Occlum.json mount config:\n{:?}", user_mount_config);
+    Ok(user_mount_config)
+}
+
+fn gen_sys_mount_config(occlum_conf_init_fs_mac: String) -> serde_json::Value {
+    let mut init_fs_mount_config: serde_json::Value = json!({
         "mount": [
             {
                 "target": "/",
@@ -232,37 +327,23 @@ fn gen_mount_config(occlum_conf_root_fs_mac: String) -> serde_json::Value {
                         {
                             "target": "/",
                             "type": "sefs",
-                            "source": "",
+                            "source": "./build/initfs/__ROOT",
                             "options": {
-                                "integrity_only": true,
                                 "MAC": ""
                             }
                         },
                         {
                             "target": "/",
                             "type": "sefs",
-                            "source": ""
+                            "source": "./run/initfs/__ROOT"
                         }
                     ]
                 }
             },
-            {
-                "target": "/host",
-                "type": "hostfs",
-                "source": "."
-            },
             {
                 "target": "/proc",
                 "type": "procfs"
             },
-            {
-                "target": "/tmp",
-                "type": "sefs",
-                "source": "",
-                "options": {
-                    "temporary": true
-                }
-            },
             {
                 "target": "/dev",
                 "type": "devfs"
@@ -270,26 +351,13 @@ fn gen_mount_config(occlum_conf_root_fs_mac: String) -> serde_json::Value {
         ]
     });
 
-    let unionfs_base_source_path = format!("{}{}", OCCLUM_INSTANCE_DIR, "/build/mount/__ROOT");
+    *init_fs_mount_config
-    let unionfs_run_source_path = format!("{}{}", OCCLUM_INSTANCE_DIR, "/run/mount/__ROOT");
-    let tmp_run_source_path = format!("{}{}", OCCLUM_INSTANCE_DIR, "/run/mount/tmp");
-
-    *internal_mount_config
-        .pointer_mut("/mount/0/options/layers/0/source")
-        .unwrap() = serde_json::Value::String(unionfs_base_source_path);
-    *internal_mount_config
         .pointer_mut("/mount/0/options/layers/0/options/MAC")
-        .unwrap() = serde_json::Value::String(occlum_conf_root_fs_mac);
+        .unwrap() = serde_json::Value::String(occlum_conf_init_fs_mac);
-    *internal_mount_config
-        .pointer_mut("/mount/0/options/layers/1/source")
-        .unwrap() = serde_json::Value::String(unionfs_run_source_path);
-    *internal_mount_config
-        .pointer_mut("/mount/3/source")
-        .unwrap() = serde_json::Value::String(tmp_run_source_path);
 
-    debug!("internal Occlum.json mount config:\n{:?}", internal_mount_config);
+    debug!("initfs mount config:\n{:?}", init_fs_mount_config);
 
-    internal_mount_config["mount"].to_owned()
+    init_fs_mount_config["mount"].to_owned()
 }
 
 #[derive(Debug, PartialEq, Deserialize)]
@@ -299,7 +367,7 @@ struct OcclumConfiguration {
     entry_points: serde_json::Value,
     env: serde_json::Value,
     metadata: OcclumMetadata,
-    mount: serde_json::Value,
+    mount: Vec<OcclumMount>,
 }
 
 #[derive(Debug, PartialEq, Deserialize)]
@@ -310,7 +378,7 @@ struct OcclumResourceLimits {
     user_space_size: String,
 }
 
-#[derive(Debug, PartialEq, Deserialize, Serialize)]
+#[derive(Debug, PartialEq, Clone, Deserialize, Serialize)]
 struct OcclumProcess {
     default_stack_size: String,
     default_heap_size: String,
@@ -324,6 +392,39 @@ struct OcclumMetadata {
     debuggable: bool,
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+struct OcclumMount {
+    #[serde(rename = "type")]
+    type_: String,
+    target: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    source: Option<String>,
+    #[serde(default, skip_serializing_if = "is_default")]
+    options: OcclumMountOptions,
+}
+
+#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)]
+struct OcclumMountOptions {
+    #[serde(rename = "MAC")]
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    mac: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub layers: Option<Vec<OcclumMount>>,
+    #[serde(default, skip_serializing_if = "is_false")]
+    pub temporary: bool,
+}
+
+#[inline]
+fn is_false(v: &bool) -> bool {
+    !(*v)
+}
+
+#[inline]
+fn is_default(option: &OcclumMountOptions) -> bool {
+    let default_option: OcclumMountOptions = Default::default();
+    option == &default_option
+}
+
 #[allow(non_snake_case)]
 #[derive(Debug, PartialEq, Serialize)]
 struct EnclaveConfiguration {
@@ -344,12 +445,12 @@ struct EnclaveConfiguration {
     ReservedMemExecutable: u32,
 }
 
-#[derive(Debug, PartialEq, Serialize)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 struct InternalResourceLimits {
     user_space_size: String,
 }
 
-#[derive(Debug, PartialEq, Serialize)]
+#[derive(Debug, PartialEq, Clone, Serialize)]
 struct InternalOcclumJson {
     resource_limits: InternalResourceLimits,
     process: OcclumProcess,

tools/init/Cargo.lock

@@ -0,0 +1,94 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+[[package]]
+name = "init"
+version = "0.0.1"
+dependencies = [
+ "libc",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "itoa"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
+
+[[package]]
+name = "libc"
+version = "0.2.84"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
+
+[[package]]
+name = "serde"
+version = "1.0.123"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.123"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.62"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.60"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"

tools/init/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "init"
+version = "0.0.1"
+authors = ["LI Qing geding.lq@antgroup.com"]
+edition = "2018"
+
+[dependencies]
+libc = "0.2.84"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"

tools/init/Makefile

@@ -0,0 +1,16 @@
+include ../../src/sgxenv.mk
+
+SRC_FILES := $(shell find . -type f -name '*.rs') Cargo.toml
+RUST_TARGET_DIR := $(BUILD_DIR)/internal/tools/init/cargo-target
+RUST_OUT_DIR := $(BUILD_DIR)/bin
+TARGET_BINARY := $(RUST_OUT_DIR)/init
+
+.PHONY: all clean
+
+all: $(SRC_FILES)
+	@RUSTC_BOOTSTRAP=1 occlum-cargo build --release --target-dir=$(RUST_TARGET_DIR) -Z unstable-options --out-dir=$(RUST_OUT_DIR)
+	@echo "CARGO (release) => init"
+
+clean:
+	@occlum-cargo clean --target-dir=$(RUST_TARGET_DIR)
+	@-$(RM) -f $(TARGET_BINARY)

tools/init/src/main.rs

@@ -0,0 +1,89 @@
+extern crate libc;
+extern crate serde;
+extern crate serde_json;
+
+use libc::syscall;
+use serde::Deserialize;
+
+use std::error::Error;
+use std::fs::File;
+use std::io::{ErrorKind, Read};
+
+fn main() -> Result<(), Box<dyn Error>> {
+    // Load the configuration from initfs
+    const IMAGE_CONFIG_FILE: &str = "/etc/image_config.json";
+    let image_config = load_config(IMAGE_CONFIG_FILE)?;
+
+    // Get the MAC of Occlum.json.protected file
+    let occlum_json_mac = {
+        let mut mac: sgx_aes_gcm_128bit_tag_t = Default::default();
+        parse_str_to_bytes(&image_config.occlum_json_mac, &mut mac)?;
+        mac
+    };
+    let occlum_json_mac_ptr = &occlum_json_mac as *const sgx_aes_gcm_128bit_tag_t;
+
+    // Get the key of encrypted SEFS image if exists
+    let key = match image_config.key {
+        Some(key_str) => {
+            let mut key: sgx_key_128bit_t = Default::default();
+            parse_str_to_bytes(&key_str, &mut key)?;
+            Some(key)
+        }
+        None => None,
+    };
+    let key_ptr = key
+        .as_ref()
+        .map(|key| key as *const sgx_key_128bit_t)
+        .unwrap_or(std::ptr::null());
+
+    // Mount the image
+    const SYS_MOUNT_FS: i64 = 363;
+    let ret = unsafe { syscall(SYS_MOUNT_FS, key_ptr, occlum_json_mac_ptr) };
+    if ret < 0 {
+        return Err(Box::new(std::io::Error::last_os_error()));
+    }
+
+    Ok(())
+}
+
+#[allow(non_camel_case_types)]
+type sgx_key_128bit_t = [u8; 16];
+#[allow(non_camel_case_types)]
+type sgx_aes_gcm_128bit_tag_t = [u8; 16];
+
+#[derive(Deserialize, Debug)]
+#[serde(deny_unknown_fields)]
+struct ImageConfig {
+    occlum_json_mac: String,
+    #[serde(default)]
+    key: Option<String>,
+}
+
+fn load_config(config_path: &str) -> Result<ImageConfig, Box<dyn Error>> {
+    let mut config_file = File::open(config_path)?;
+    let config_json = {
+        let mut config_json = String::new();
+        config_file.read_to_string(&mut config_json)?;
+        config_json
+    };
+    let config: ImageConfig = serde_json::from_str(&config_json)?;
+    Ok(config)
+}
+
+fn parse_str_to_bytes(arg_str: &str, bytes: &mut [u8]) -> Result<(), Box<dyn Error>> {
+    let bytes_str_vec = {
+        let bytes_str_vec: Vec<&str> = arg_str.split('-').collect();
+        if bytes_str_vec.len() != bytes.len() {
+            return Err(Box::new(std::io::Error::new(
+                ErrorKind::InvalidData,
+                "The length or format of Key/MAC string is invalid",
+            )));
+        }
+        bytes_str_vec
+    };
+
+    for (byte_i, byte_str) in bytes_str_vec.iter().enumerate() {
+        bytes[byte_i] = u8::from_str_radix(byte_str, 16)?;
+    }
+    Ok(())
+}

tools/occlum

@@ -52,11 +52,16 @@ Usage:
     occlum init
         Initialize a directory as the Occlum instance.
 
-    occlum build [--sign-key <key_path>] [--sign-tool <tool_path>] [-f/--force]
+    occlum build [--sign-key <key_path>] [--sign-tool <tool_path>] [--image-key <key_path>] [--no-buildin-image-config] [-f/--force]
         Build and sign an Occlum SGX enclave (.so) and generate its associated secure
         FS image according to the user-provided image directory and Occlum.json config file.
         The whole building process is incremental: the building artifacts are built only
         when needed.
+        A Configuration file (i.e., image_config.json) consists of the MAC of Occlum.json and
+        the key (if provided) of the secure FS image will be built into the initfs by default,
+        the init program can use it to mount the secure FS image as the rootfs.
+        To protect the security of FS image, give the [--no-buildin-image-config] flag,
+        and modifies the init program to acquire the MAC and key with LA or RA.
         To force rebuilding all artifacts, give the [-f/--force] flag.
 
     occlum run <program_name> <program_args>
@@ -73,7 +78,7 @@ Usage:
     occlum gdb <program_name> <program_args>
         Debug the program running inside an SGX enclave with GDB.
 
-    occlum mount [--sign-key <key_path>] [--sign-tool <tool_path>] <path>
+    occlum mount [--sign-key <key_path>] [--sign-tool <tool_path>] [--image-key <key_path>] <path>
         Mount the secure FS image of the Occlum instance as a Linux FS at an existing <path>.
         This makes it easy to access and manipulate Occlum's secure FS for debug purpose.
 EOF
@@ -192,6 +197,25 @@ cmd_init() {
             "$cpu_lib/online"
     fi
 
+    mkdir -p initfs
+    mkdir -p initfs/bin
+    mkdir -p initfs/lib
+    mkdir -p initfs/dev
+    mkdir -p initfs/proc
+    mkdir -p initfs/etc
+    # add default /etc/hosts
+    echo "127.0.0.1   localhost" > initfs/etc/hosts
+    # add musl
+    local occlum_musl_lib=/usr/local/occlum/x86_64-linux-musl/lib
+    cp -t initfs/lib \
+        /lib/ld-musl-x86_64.so.1 \
+        "$occlum_musl_lib/libc.so" \
+        "$occlum_musl_lib/libstdc++.so.6" \
+        "$occlum_musl_lib/libgcc_s.so.1" \
+        "$occlum_musl_lib/libgomp.so.1"
+
+    cp "$occlum_dir"/build/bin/init initfs/bin/
+
     cp "$occlum_dir"/etc/template/Occlum.json "$instance_dir"/
     chmod 644 "$instance_dir"/Occlum.json
 
@@ -203,20 +227,27 @@ cmd_build() {
 
     pal_lib=libocclum-pal.so
     libos_lib=libocclum-libos.so
+    BUILDIN_IMAGE_CONF=true
 
     while [ -n "$1" ]; do
         case "$1" in
         --sign-key)     [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path"     ;;
         --sign-tool)    [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path"   ;;
         --sgx-mode)     [[ -n "$2" && "$2" != "HW" ]] && export SGX_MODE=SIM ; shift 2 || exit_error "empty sgx mode";;
+        --image-key)    [ -n "$2" ] && SECURE_IMAGE_KEY=$2 ; shift 2 || exit_error "empty secure image key path"   ;;
+        --no-buildin-image-config)  BUILDIN_IMAGE_CONF=false ; shift ;;
         --force | -f)   MAKE_OPTION="--always-make" ; shift ;;
         *) exit_error "Unknown option: $1"                                                                      ;;
         esac
     done
     [ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY"
     [ -e "$ENCLAVE_SIGN_TOOL" ] || exit_error "invalid signing tool path: $ENCLAVE_SIGN_TOOL"
+    if [ -n "$SECURE_IMAGE_KEY" ]; then
+        [ -e "$SECURE_IMAGE_KEY" ] || exit_error "invalid secure image key path: $SECURE_IMAGE_KEY"
+    fi
     echo "Enclave sign-tool: $ENCLAVE_SIGN_TOOL"
     echo "Enclave sign-key: $ENCLAVE_SIGN_KEY"
+    [ -n "$SECURE_IMAGE_KEY" ] && echo "Image encryption key: $SECURE_IMAGE_KEY"
 
     if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then
         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SGX_SDK/sdk_libs
@@ -244,6 +275,7 @@ cmd_build() {
     occlum_dir=$occlum_dir instance_dir=$instance_dir pal_lib=$pal_lib major_ver=$major_ver \
     occlum_version=$occlum_version libos_lib=$libos_lib \
     ENCLAVE_SIGN_KEY=$ENCLAVE_SIGN_KEY ENCLAVE_SIGN_TOOL=$ENCLAVE_SIGN_TOOL \
+    SECURE_IMAGE_KEY=$SECURE_IMAGE_KEY BUILDIN_IMAGE_CONF=$BUILDIN_IMAGE_CONF \
     make -f $build_makefile $MAKE_OPTION
 
     cd "$instance_dir"
@@ -258,6 +290,8 @@ cmd_build() {
     mkdir -p "$instance_dir/run/mount/__ROOT"
     mkdir -p "$instance_dir/run/mount/tmp"
 
+    mkdir -p "$instance_dir/run/initfs/__ROOT"
+
     echo "Built the Occlum image and enclave successfully"
 }
 
@@ -363,7 +397,9 @@ cmd_package() {
 
     cd .. && tar -cvzf $instance_dir/$package_name $instance_base_name/Occlum.json $instance_base_name/build/bin \
         $instance_base_name/build/lib/libocclum-libos.signed.so $instance_base_name/build/lib/libocclum-pal.so* \
-        $instance_base_name/build/mount $instance_base_name/build/Occlum.json.protected $instance_base_name/run \
+        $instance_base_name/build/mount $instance_base_name/build/Occlum.json.protected \
+        $instance_base_name/build/initfs $instance_base_name/build/.Occlum_sys.json.protected \
+        $instance_base_name/initfs $instance_base_name/run \
         $instance_base_name/.__occlum_status $instance_base_name/.sgx_mode
 
     if [ "`get_enclave_debuggable_flag`" == "true" ]; then
@@ -398,14 +434,19 @@ cmd_mount() {
         case "$1" in
         --sign-key)     [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path"     ;;
         --sign-tool)    [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path"   ;;
+        --image-key)    [ -n "$2" ] && SECURE_IMAGE_KEY=$2 ; shift 2 || exit_error "empty secure image key path"   ;;
         *) MNT_POINT=$1 ; shift ;;
         esac
     done
     [ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY"
     [ -e "$ENCLAVE_SIGN_TOOL" ] || exit_error "invalid signing tool path: $ENCLAVE_SIGN_TOOL"
+    if [ -n "$SECURE_IMAGE_KEY" ]; then
+        [ -e "$SECURE_IMAGE_KEY" ] || exit_error "invalid secure image key path: $SECURE_IMAGE_KEY"
+    fi
     [ -d "$MNT_POINT" ] || exit_error "invalid mount point: $MNT_POINT"
     echo "Mount tool sign-tool: $ENCLAVE_SIGN_TOOL"
     echo "Mount tool sign-key: $ENCLAVE_SIGN_KEY"
+    [ -n "$SECURE_IMAGE_KEY" ] && echo "Image decryption key: $SECURE_IMAGE_KEY"
     echo "Mount point: $MNT_POINT"
 
     SGX_MODE=$(cat $instance_dir/.sgx_mode)
@@ -434,10 +475,12 @@ cmd_mount() {
         container_fs=""
     fi
 
+    [ -n "$SECURE_IMAGE_KEY" ] && SECURE_IMAGE_KEY_OPTION="--key $SECURE_IMAGE_KEY"
     echo "Start to mount the FS..."
     LD_LIBRARY_PATH="$SGX_SDK/sdk_libs" "$sefs_cli" \
         --enclave "$signed_sefs_cli_lib" \
         mount \
+        $SECURE_IMAGE_KEY_OPTION \
         "$image_fs" \
         "$container_fs" \
         "$MNT_POINT"

tools/occlum_build.mk

@@ -2,6 +2,11 @@ SGX_SDK ?= /opt/occlum/sgxsdk-tools
 
 IMAGE := $(instance_dir)/image
 SECURE_IMAGE := $(instance_dir)/build/mount/__ROOT/metadata
+SECURE_IMAGE_MAC := $(instance_dir)/build/mount/.ROOT_MAC
+IMAGE_CONFIG_JSON := $(instance_dir)/build/image_config.json
+INITFS := $(instance_dir)/initfs
+INITFS_IMAGE := $(instance_dir)/build/initfs/__ROOT/metadata
+INITFS_IMAGE_MAC := $(instance_dir)/build/initfs/.ROOT_MAC
 JSON_CONF := $(instance_dir)/Occlum.json
 
 LIBOS := $(instance_dir)/build/lib/$(libos_lib).$(occlum_version)
@@ -21,19 +26,23 @@ ifneq (, $(wildcard $(IMAGE)/. ))
 	IMAGE_FILES := $(shell find $(IMAGE) -type f 2>/dev/null | sed 's/ /\\ /g' || true)
 endif
 
+ifneq (, $(wildcard $(INITFS)/. ))
+	INITFS_DIRS := $(shell find $(INITFS) -type d 2>/dev/null | sed 's/ /\\ /g' || true)
+	INITFS_FILES := $(shell find $(INITFS) -type f 2>/dev/null | sed 's/ /\\ /g' || true)
+endif
+
 SHELL:=/bin/bash
 
-define get_conf_root_fs_mac
+define get_occlum_sys_conf_file_mac
 	LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" \
-		"$(occlum_dir)/build/bin/occlum-protect-integrity" show-mac "$(instance_dir)/build/mount/__ROOT/metadata"
+		"$(occlum_dir)/build/bin/occlum-protect-integrity" show-mac "$(instance_dir)/build/.Occlum_sys.json.protected"
 endef
 
-define get_occlum_conf_file_mac
+define get_occlum_user_conf_file_mac
 	LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" \
 		"$(occlum_dir)/build/bin/occlum-protect-integrity" show-mac "$(instance_dir)/build/Occlum.json.protected"
 endef
 
-
 .PHONY : all
 
 ALL_TARGETS := $(SIGNED_ENCLAVE) $(BIN_LINKS) $(LIB_LINKS)
@@ -49,25 +58,23 @@ $(SIGNED_ENCLAVE): $(LIBOS)
 		-enclave "$(instance_dir)/build/lib/libocclum-libos.so.$(major_ver)" \
 		-out "$(instance_dir)/build/lib/libocclum-libos.signed.so"
 
-$(LIBOS): $(instance_dir)/build/Occlum.json.protected
+$(LIBOS): $(instance_dir)/build/.Occlum_sys.json.protected
 	@echo "Building libOS..."
-	@export OCCLUM_BUILTIN_CONF_FILE_MAC=`$(get_occlum_conf_file_mac)` ; \
+	@export OCCLUM_BUILTIN_SYS_CONF_FILE_MAC=`$(get_occlum_sys_conf_file_mac)` ; \
-		echo "EXPORT => OCCLUM_BUILTIN_CONF_FILE_MAC = $$OCCLUM_BUILTIN_CONF_FILE_MAC" ; \
 		cd $(instance_dir)/build/lib && \
 		cp "$(occlum_dir)/build/lib/$(libos_lib).$(occlum_version)" . && ln -sf "$(libos_lib).$(occlum_version)" "libocclum-libos.so.$(major_ver)" && \
 		ln -sf "libocclum-libos.so.$(major_ver)" libocclum-libos.so ; \
-		echo -e "$$OCCLUM_BUILTIN_CONF_FILE_MAC\c" > temp_mac_file && \
+		echo -e "$$OCCLUM_BUILTIN_SYS_CONF_FILE_MAC\c" > temp_mac_file && \
 		objcopy --update-section .builtin_config=temp_mac_file libocclum-libos.so && \
 		rm temp_mac_file
 
-$(instance_dir)/build/Occlum.json.protected: $(instance_dir)/build/Occlum.json
+$(instance_dir)/build/.Occlum_sys.json.protected: $(instance_dir)/build/.Occlum_sys.json
 	@cd "$(instance_dir)/build" ; \
-		LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" "$(occlum_dir)/build/bin/occlum-protect-integrity" protect Occlum.json ;
+		LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" "$(occlum_dir)/build/bin/occlum-protect-integrity" protect .Occlum_sys.json ;
 
-$(instance_dir)/build/Enclave.xml:
+$(instance_dir)/build/.Occlum_sys.json: $(INITFS_IMAGE) $(INITFS_IMAGE_MAC) $(JSON_CONF)
-$(instance_dir)/build/Occlum.json: $(SECURE_IMAGE) $(JSON_CONF) | $(instance_dir)/build/lib
+	@$(occlum_dir)/build/bin/gen_internal_conf --user_json "$(JSON_CONF)" gen_sys_conf \
-	@$(occlum_dir)/build/bin/gen_internal_conf --user_json "$(instance_dir)/Occlum.json" --fs_mac `$(get_conf_root_fs_mac)` \
+		--init_fs_mac "`cat $(INITFS_IMAGE_MAC)`" --sys_json $(instance_dir)/build/.Occlum_sys.json
-		--sdk_xml "$(instance_dir)/build/Enclave.xml" --sys_json $(instance_dir)/build/Occlum.json
 
 $(BIN_LINKS): $(instance_dir)/build/bin/%: $(occlum_dir)/build/bin/% | $(instance_dir)/build/bin
 	@ln -sf $< $@
@@ -84,18 +91,53 @@ $(instance_dir)/build/lib/libocclum-pal.so.0: | $(instance_dir)/build/lib
 $(instance_dir)/build/lib:
 	@mkdir -p build/lib
 
-# If image dir not exist, just use the secure Occlum FS image
+$(INITFS_IMAGE_MAC):
-ifneq ($(wildcard $(IMAGE)/. ),)
+$(INITFS_IMAGE): $(INITFS) $(INITFS_DIRS) $(INITFS_FILES) $(IMAGE_CONFIG_JSON) $(SEFS_CLI_SIM) $(SIGNED_SEFS_CLI_LIB)
-$(SECURE_IMAGE): $(IMAGE) $(IMAGE_DIRS) $(IMAGE_FILES) $(SEFS_CLI_SIM) $(SIGNED_SEFS_CLI_LIB)
+	@echo "Building the initfs..."
-	@echo "Building new image..."
+	@rm -rf build/initfs
-
+	@mkdir -p build/initfs
-	@rm -rf build/mount
+	@[ "$(BUILDIN_IMAGE_CONF)" == "true" ] && \
-
+		cp "$(IMAGE_CONFIG_JSON)" "$(INITFS)/etc/" || \
-	@mkdir -p build/mount/
+		rm -f "$(INITFS)/etc/`basename $(IMAGE_CONFIG_JSON)`"
 	@LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" $(SEFS_CLI_SIM) \
 		--enclave "$(SIGNED_SEFS_CLI_LIB)" \
 		zip \
-		"$(instance_dir)/image" \
+		"$(INITFS)" \
-		"$(instance_dir)/build/mount/__ROOT" \
+		"$(instance_dir)/build/initfs/__ROOT" \
-		--integrity-only
+		"$(INITFS_IMAGE_MAC)"
+
+$(IMAGE_CONFIG_JSON): $(instance_dir)/build/Occlum.json.protected
+	@export OCCLUM_CONF_FILE_MAC=`$(get_occlum_user_conf_file_mac)` ; \
+		echo "EXPORT => OCCLUM_CONF_FILE_MAC = $$OCCLUM_CONF_FILE_MAC" ; \
+		[ -n "$(SECURE_IMAGE_KEY)" ] && \
+		jq -n --arg mac_val "$$OCCLUM_CONF_FILE_MAC" --arg key_val "`cat $(SECURE_IMAGE_KEY)`" \
+		'{occlum_json_mac: $$mac_val, key: $$key_val}' > $(IMAGE_CONFIG_JSON) || \
+		jq -n --arg mac_val "$$OCCLUM_CONF_FILE_MAC" \
+		'{occlum_json_mac: $$mac_val}' > $(IMAGE_CONFIG_JSON)
+
+$(instance_dir)/build/Occlum.json.protected: $(instance_dir)/build/Occlum.json
+	@cd "$(instance_dir)/build" ; \
+		LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" "$(occlum_dir)/build/bin/occlum-protect-integrity" protect Occlum.json ;
+
+$(instance_dir)/build/Enclave.xml:
+$(instance_dir)/build/Occlum.json: $(SECURE_IMAGE) $(SECURE_IMAGE_MAC) $(JSON_CONF) | $(instance_dir)/build/lib
+	@$(occlum_dir)/build/bin/gen_internal_conf --user_json "$(JSON_CONF)" gen_user_conf \
+		--user_fs_mac "`cat $(SECURE_IMAGE_MAC)`" --sdk_xml "$(instance_dir)/build/Enclave.xml"  \
+		--output_user_json $(instance_dir)/build/Occlum.json
+
+# If image dir not exist, just use the secure Occlum FS image
+ifneq ($(wildcard $(IMAGE)/. ),)
+$(SECURE_IMAGE_MAC):
+$(SECURE_IMAGE): $(IMAGE) $(IMAGE_DIRS) $(IMAGE_FILES) $(SEFS_CLI_SIM) $(SIGNED_SEFS_CLI_LIB)
+	@echo "Building new image..."
+	@rm -rf build/mount
+	@mkdir -p build/mount/
+	@[ -n "$(SECURE_IMAGE_KEY)" ] && export SECURE_IMAGE_KEY_OPTION="--key $(SECURE_IMAGE_KEY)" ; \
+		LD_LIBRARY_PATH="$(SGX_SDK)/sdk_libs" $(SEFS_CLI_SIM) \
+			--enclave "$(SIGNED_SEFS_CLI_LIB)" \
+			zip \
+			$$SECURE_IMAGE_KEY_OPTION \
+			"$(IMAGE)" \
+			"$(instance_dir)/build/mount/__ROOT" \
+			"$(SECURE_IMAGE_MAC)"
 endif