Improve simulation mode user experience

Simulation mode and hardware mode can both work without rebuild Occlum.

.gitignore

@@ -2,3 +2,4 @@
 *.a
 *.so
 build/
+build_sim/

Makefile

@@ -23,8 +23,18 @@ submodule: githooks
 	@# Try to apply the patches. If failed, check if the patches are already applied
 	cd deps/rust-sgx-sdk && git apply ../rust-sgx-sdk.patch >/dev/null 2>&1 || git apply ../rust-sgx-sdk.patch -R --check
 	cd deps/serde-json-sgx && git apply ../serde-json-sgx.patch >/dev/null 2>&1 || git apply ../serde-json-sgx.patch -R --check
-	cd deps/sefs/sefs-fuse && make
+
-	cd tools/ && make
+	@# Build tools and sefs-fuse for both HW mode and SIM mode
+	@$(MAKE) SGX_MODE=SIM --no-print-directory -C tools
+	@$(MAKE) --no-print-directory -C deps/sefs/sefs-fuse clean
+	@$(MAKE) SGX_MODE=SIM --no-print-directory -C deps/sefs/sefs-fuse
+	@cp deps/sefs/sefs-fuse/bin/sefs-fuse build_sim/bin
+	@cp deps/sefs/sefs-fuse/lib/libsefs-fuse.signed.so build_sim/lib
+	@$(MAKE) --no-print-directory -C tools
+	@$(MAKE) --no-print-directory -C deps/sefs/sefs-fuse clean
+	@$(MAKE) --no-print-directory -C deps/sefs/sefs-fuse
+	@cp deps/sefs/sefs-fuse/bin/sefs-fuse build/bin
+	@cp deps/sefs/sefs-fuse/lib/libsefs-fuse.signed.so build/lib
 
 src:
 	@$(MAKE) --no-print-directory -C src
@@ -34,12 +44,17 @@ test:
 
 OCCLUM_PREFIX ?= /opt/occlum
 install:
-	install -d $(OCCLUM_PREFIX)/deps/sefs/sefs-fuse/bin/
+	@# Install both libraries for HW mode and SIM mode
-	install -t $(OCCLUM_PREFIX)/deps/sefs/sefs-fuse/bin/ deps/sefs/sefs-fuse/bin/*
+	@$(MAKE) --no-print-directory -C src
+	@$(MAKE) SGX_MODE=SIM --no-print-directory -C src
 	install -d $(OCCLUM_PREFIX)/build/bin/
 	install -t $(OCCLUM_PREFIX)/build/bin/ -D build/bin/*
 	install -d $(OCCLUM_PREFIX)/build/lib/
 	install -t $(OCCLUM_PREFIX)/build/lib/ -D build/lib/*
+	install -d $(OCCLUM_PREFIX)/build_sim/bin/
+	install -t $(OCCLUM_PREFIX)/build_sim/bin/ -D build_sim/bin/*
+	install -d $(OCCLUM_PREFIX)/build_sim/lib/
+	install -t $(OCCLUM_PREFIX)/build_sim/lib/ -D build_sim/lib/*
 	install -d $(OCCLUM_PREFIX)/src/
 	install -t $(OCCLUM_PREFIX)/src/ -m 444 src/sgxenv.mk
 	install -d $(OCCLUM_PREFIX)/src/libos/
@@ -54,3 +69,5 @@ install:
 clean:
 	@$(MAKE) --no-print-directory -C src clean
 	@$(MAKE) --no-print-directory -C test clean
+	@$(MAKE) SGX_MODE=SIM --no-print-directory -C src clean
+	@$(MAKE) SGX_MODE=SIM --no-print-directory -C test clean

deps/sefs

@@ -1 +1 @@
-Subproject commit 6bdce43eafde51ec9eff4fb71c0106747b18a7d1
+Subproject commit 984dd12e414e554000034514ac41a69117ccffa7

src/sgxenv.mk

@@ -2,12 +2,22 @@ MAIN_MAKEFILE := $(firstword $(MAKEFILE_LIST))
 INCLUDE_MAKEFILE := $(lastword $(MAKEFILE_LIST))
 CUR_DIR := $(shell dirname $(realpath $(MAIN_MAKEFILE)))
 PROJECT_DIR := $(realpath $(CUR_DIR)/../../)
-BUILD_DIR := $(PROJECT_DIR)/build
 
 SGX_SDK ?= /opt/intel/sgxsdk
 SGX_MODE ?= HW
 SGX_ARCH ?= x64
 
+ifneq ($(SGX_MODE), HW)
+	BUILD_DIR := $(PROJECT_DIR)/build_sim
+else
+	BUILD_DIR := $(PROJECT_DIR)/build
+endif
+
+# for sgxenv.mk in .occlum
+ifeq ($(CONTEXT), 1)
+	BUILD_DIR := $(PROJECT_DIR)/build
+endif
+
 # If OCCLUM_RELEASE_BUILD equals to 1, y, or yes, then build in release mode
 OCCLUM_RELEASE_BUILD ?= 0
 ifeq ($(OCCLUM_RELEASE_BUILD), yes)

test/Makefile

@@ -1,6 +1,12 @@
 CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 PROJECT_DIR := $(realpath $(CUR_DIR)/../)
-BUILD_DIR := $(PROJECT_DIR)/build
+SGX_MODE ?= HW
+
+ifneq ($(SGX_MODE), HW)
+	BUILD_DIR := $(PROJECT_DIR)/build_sim
+else
+	BUILD_DIR := $(PROJECT_DIR)/build
+endif
 
 # Dependencies: need to be compiled but not to run by any Makefile target
 TEST_DEPS := client data_sink
@@ -40,7 +46,7 @@ prebuild:
 	@$(RM) -rf $(BUILD_DIR)/test
 	@mkdir -p $(BUILD_DIR)/test
 	@cd $(BUILD_DIR)/test && \
-		$(PROJECT_DIR)/build/bin/occlum init
+		$(BUILD_DIR)/bin/occlum init
 	@cp Occlum.json Enclave.xml $(BUILD_DIR)/test/
 
 $(BUILD_TARGETS): %:
@@ -50,7 +56,7 @@ $(BUILD_TARGETS): %:
 
 postbuild:
 	@cd $(BUILD_DIR)/test && \
-		$(PROJECT_DIR)/build/bin/occlum build
+		$(BUILD_DIR)/bin/occlum build
 
 #############################################################################
 # Test targets

test/test_common.mk

@@ -2,7 +2,13 @@ MAIN_MAKEFILE := $(firstword $(MAKEFILE_LIST))
 INCLUDE_MAKEFILE := $(lastword $(MAKEFILE_LIST))
 CUR_DIR := $(shell dirname $(realpath $(MAIN_MAKEFILE)))
 PROJECT_DIR := $(realpath $(CUR_DIR)/../../)
-BUILD_DIR := $(PROJECT_DIR)/build
+SGX_MODE ?= HW
+
+ifneq ($(SGX_MODE), HW)
+	BUILD_DIR := $(PROJECT_DIR)/build_sim
+else
+	BUILD_DIR := $(PROJECT_DIR)/build
+endif
 
 TEST_NAME := $(shell basename $(CUR_DIR))
 IMAGE_DIR := $(BUILD_DIR)/test/image
@@ -58,7 +64,7 @@ $(BUILD_DIR)/test/obj/$(TEST_NAME)/%.o: %.cc
 
 test:
 	@cd $(BUILD_DIR)/test && \
-		$(PROJECT_DIR)/build/bin/occlum run /bin/$(TEST_NAME) $(BIN_ARGS)
+		$(BUILD_DIR)/bin/occlum run /bin/$(TEST_NAME) $(BIN_ARGS)
 
 test-native:
 	@LD_LIBRARY_PATH=/usr/local/occlum/lib cd $(IMAGE_DIR) && ./bin/$(TEST_NAME) $(BIN_ARGS)

tools/Makefile

@@ -1,10 +1,18 @@
+SGX_MODE ?= HW
+
+ifneq ($(SGX_MODE), HW)
+	BUILD_DIR := build_sim
+else
+	BUILD_DIR := build
+endif
+
 .PHONY: all clean
 
 all:
-	@mkdir -p ../build/bin/
+	@mkdir -p ../$(BUILD_DIR)/bin/
-	@ln -s -f ../../tools/occlum ../build/bin/occlum
+	@ln -s -f ../../tools/occlum ../$(BUILD_DIR)/bin/occlum
-	@ln -s -f ../../tools/occlum-build-enclave ../build/bin/occlum-build-enclave
+	@ln -s -f ../../tools/occlum-build-enclave ../$(BUILD_DIR)/bin/occlum-build-enclave
-	@ln -s -f ../../tools/occlum-gen-default-occlum-json ../build/bin/occlum-gen-default-occlum-json
+	@ln -s -f ../../tools/occlum-gen-default-occlum-json ../$(BUILD_DIR)/bin/occlum-gen-default-occlum-json
 	@$(MAKE) --no-print-directory -C protect-integrity
 
 clean:

tools/occlum

@@ -68,7 +68,7 @@ get_conf_entry_points() {
 }
 
 get_occlum_conf_file_mac() {
-    "$occlum_dir/build/bin/occlum-protect-integrity" show-mac "$context_dir/build/Occlum.json.protected"
+    "$occlum_dir/$build_dir/bin/occlum-protect-integrity" show-mac "$context_dir/build/Occlum.json.protected"
 }
 
 parse_occlum_user_space_size() {
@@ -108,8 +108,27 @@ cmd_init() {
 }
 
 cmd_build() {
+    build_dir=build
+
+    while [ -n "$1" ]; do
+        case "$1" in
+        --sign-key)     [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path"     ;;
+        --sign-tool)    [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path"   ;;
+        --sgx-mode)     [[ -n "$2" && "$2" != "HW" ]] && export SGX_MODE=SW && build_dir=build_sim ; shift 2 || exit_error "empty sgx mode";;
+        *) exit_error "Unknown option: $1"                                                                      ;;
+        esac
+    done
+    [ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY"
+    [ -e "$ENCLAVE_SIGN_TOOL" ] || exit_error "invalid signing tool path: $ENCLAVE_SIGN_TOOL"
+    echo "Enclave sign-tool: $ENCLAVE_SIGN_TOOL"
+    echo "Enclave sign-key: $ENCLAVE_SIGN_KEY"
+
     if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then
-        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SGX_SDK/lib64
+        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SGX_SDK/sdk_libs
+        build_dir=build_sim
+        echo "SGX mode: $SGX_MODE"
+    else
+        echo "SGX mode: HW"
     fi
 
     cd "$context_dir"
@@ -118,24 +137,24 @@ cmd_build() {
     rm -rf build
 
     mkdir -p build/bin
-    cp "$occlum_dir/build/bin/occlum-run" build/bin/
+    cp "$occlum_dir/$build_dir/bin/occlum-run" build/bin/
     mkdir -p build/lib
-    cp "$occlum_dir/build/lib/libocclum-libos-core.a" build/lib/
+    cp "$occlum_dir/$build_dir/lib/libocclum-libos-core.a" build/lib/
-    cp "$occlum_dir/build/lib/libcompiler-rt-patch.a" build/lib/
+    cp "$occlum_dir/$build_dir/lib/libcompiler-rt-patch.a" build/lib/
-    cp "$occlum_dir/build/lib/libocclum-pal.so" build/lib/
+    cp "$occlum_dir/$build_dir/lib/libocclum-pal.so" build/lib/
     mkdir -p build/src/libos/src/builtin
 
     chmod 531 -R $working_dir/image/bin
     chmod 531 -R $working_dir/image/lib
     mkdir -p build/mount/
-    cd "$occlum_dir/deps/sefs/sefs-fuse/bin/" && \
+    cd "$occlum_dir/$build_dir/bin/" && \
-        ./app \
+    ./sefs-fuse \
-            --integrity-only \
+        --integrity-only \
-            "$context_dir/build/mount/__ROOT" \
+        "$context_dir/build/mount/__ROOT" \
-            "$working_dir/image" \
+        "$working_dir/image" \
-            zip
+        zip
 
-    export OCCLUM_CONF_ROOT_FS_MAC=`"$occlum_dir/build/bin/occlum-protect-integrity" show-mac "$context_dir/build/mount/__ROOT/metadata"`
+    export OCCLUM_CONF_ROOT_FS_MAC=`"$occlum_dir/$build_dir/bin/occlum-protect-integrity" show-mac "$context_dir/build/mount/__ROOT/metadata"`
     export OCCLUM_CONF_USER_SPACE_SIZE=`get_conf_user_space_size`
     export OCCLUM_CONF_DEFAULT_STACK_SIZE=`get_conf_default_stack_size`
     export OCCLUM_CONF_DEFAULT_HEAP_SIZE=`get_conf_default_heap_size`
@@ -143,9 +162,9 @@ cmd_build() {
     export OCCLUM_CONF_ENV=`get_conf_env`
     export OCCLUM_CONF_ENTRY_POINTS=`get_conf_entry_points`
     cd "$context_dir/build"
-    "$occlum_dir/build/bin/occlum-gen-default-occlum-json"\
+    "$occlum_dir/$build_dir/bin/occlum-gen-default-occlum-json"\
         > "Occlum.json"
-    "$occlum_dir/build/bin/occlum-protect-integrity" protect Occlum.json
+    "$occlum_dir/$build_dir/bin/occlum-protect-integrity" protect Occlum.json
 
     export OCCLUM_BUILTIN_CONF_FILE_MAC=`get_occlum_conf_file_mac`
     echo "EXPORT => OCCLUM_BUILTIN_CONF_FILE_MAC = $OCCLUM_BUILTIN_CONF_FILE_MAC"
@@ -160,18 +179,7 @@ cmd_build() {
     cp -r "$occlum_dir/src/libos/src/builtin" src/libos/src/builtin
     cd src/libos && \
         make clean-builtin && \
-        make "$context_dir/build/lib/libocclum-libos.so" ONLY_REBUILD_BUILTIN=1
+        make "$context_dir/build/lib/libocclum-libos.so" ONLY_REBUILD_BUILTIN=1 CONTEXT=1
-    while [ -n "$1" ]; do
-        case "$1" in
-        --sign-key)     [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path"     ;;
-        --sign-tool)    [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path"   ;;
-        *) exit_error "Unknown option: $1"                                                                      ;;
-        esac
-    done
-    [ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY"
-    [ -e "$ENCLAVE_SIGN_TOOL" ] || exit_error "invalid signing tool path: $ENCLAVE_SIGN_TOOL"
-    echo "Enclave sign-tool: $ENCLAVE_SIGN_TOOL"
-    echo "Enclave sign-key: $ENCLAVE_SIGN_KEY"
     $ENCLAVE_SIGN_TOOL sign \
         -key $ENCLAVE_SIGN_KEY \
         -config "$working_dir/Enclave.xml" \
@@ -182,12 +190,19 @@ cmd_build() {
 
     mkdir -p "$context_dir/run/mount/root"
 
+    if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then
+        echo "SIM" > .sgx_mode
+    else
+        echo "HW" > .sgx_mode
+    fi
+
     echo "Built the Occlum image and enclave successfully"
 }
 
 cmd_run() {
+    SGX_MODE=$(cat $context_dir/.sgx_mode)
     if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then
-        export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/lib64/"
+        export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/sdk_libs/"
     else
         export LD_LIBRARY_PATH="$context_dir/build/lib"
     fi
@@ -201,8 +216,9 @@ cmd_run() {
 }
 
 cmd_gdb() {
+    SGX_MODE=$(cat $context_dir/.sgx_mode)
     if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then
-        export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/lib64/"
+        export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/sdk_libs/"
     else
         export LD_LIBRARY_PATH="$context_dir/build/lib"
     fi

tools/protect-integrity/Makefile

@@ -2,14 +2,19 @@
 
 CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST))))
 PROJECT_DIR := $(realpath $(CUR_DIR)/../../)
-BUILD_DIR := $(PROJECT_DIR)/build
-OBJS_DIR := $(BUILD_DIR)/tools/protect-integrity
 
 SGX_SDK ?= /opt/intel/sgxsdk
 SGX_MODE ?= HW
 SGX_ARCH ?= x64
 SGX_DEBUG ?= 1
 
+ifneq ($(SGX_MODE), HW)
+	BUILD_DIR := $(PROJECT_DIR)/build_sim
+else
+	BUILD_DIR := $(PROJECT_DIR)/build
+endif
+OBJS_DIR := $(BUILD_DIR)/tools/protect-integrity
+
 ifeq ($(shell getconf LONG_BIT), 32)
 	SGX_ARCH := x86
 else ifeq ($(findstring -m32, $(CXXFLAGS)), -m32)