From 4ebedd9bf5f6fda39cfffc27ffcd1ccc08ed5968 Mon Sep 17 00:00:00 2001 From: "Hui,Chunyang" Date: Tue, 3 Mar 2020 02:51:43 +0000 Subject: [PATCH] Improve simulation mode user experience Simulation mode and hardware mode can both work without rebuild Occlum. --- .gitignore | 1 + Makefile | 25 +++++++++-- deps/sefs | 2 +- src/sgxenv.mk | 12 +++++- test/Makefile | 12 ++++-- test/test_common.mk | 10 ++++- tools/Makefile | 16 +++++-- tools/occlum | 74 +++++++++++++++++++------------- tools/protect-integrity/Makefile | 9 +++- 9 files changed, 115 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 6fda0f01..99682c95 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.a *.so build/ +build_sim/ diff --git a/Makefile b/Makefile index 57b808d1..81968c96 100644 --- a/Makefile +++ b/Makefile @@ -23,8 +23,18 @@ submodule: githooks @# Try to apply the patches. If failed, check if the patches are already applied cd deps/rust-sgx-sdk && git apply ../rust-sgx-sdk.patch >/dev/null 2>&1 || git apply ../rust-sgx-sdk.patch -R --check cd deps/serde-json-sgx && git apply ../serde-json-sgx.patch >/dev/null 2>&1 || git apply ../serde-json-sgx.patch -R --check - cd deps/sefs/sefs-fuse && make - cd tools/ && make + + @# Build tools and sefs-fuse for both HW mode and SIM mode + @$(MAKE) SGX_MODE=SIM --no-print-directory -C tools + @$(MAKE) --no-print-directory -C deps/sefs/sefs-fuse clean + @$(MAKE) SGX_MODE=SIM --no-print-directory -C deps/sefs/sefs-fuse + @cp deps/sefs/sefs-fuse/bin/sefs-fuse build_sim/bin + @cp deps/sefs/sefs-fuse/lib/libsefs-fuse.signed.so build_sim/lib + @$(MAKE) --no-print-directory -C tools + @$(MAKE) --no-print-directory -C deps/sefs/sefs-fuse clean + @$(MAKE) --no-print-directory -C deps/sefs/sefs-fuse + @cp deps/sefs/sefs-fuse/bin/sefs-fuse build/bin + @cp deps/sefs/sefs-fuse/lib/libsefs-fuse.signed.so build/lib src: @$(MAKE) --no-print-directory -C src @@ -34,12 +44,17 @@ test: OCCLUM_PREFIX ?= /opt/occlum install: - install -d $(OCCLUM_PREFIX)/deps/sefs/sefs-fuse/bin/ - install -t $(OCCLUM_PREFIX)/deps/sefs/sefs-fuse/bin/ deps/sefs/sefs-fuse/bin/* + @# Install both libraries for HW mode and SIM mode + @$(MAKE) --no-print-directory -C src + @$(MAKE) SGX_MODE=SIM --no-print-directory -C src install -d $(OCCLUM_PREFIX)/build/bin/ install -t $(OCCLUM_PREFIX)/build/bin/ -D build/bin/* install -d $(OCCLUM_PREFIX)/build/lib/ install -t $(OCCLUM_PREFIX)/build/lib/ -D build/lib/* + install -d $(OCCLUM_PREFIX)/build_sim/bin/ + install -t $(OCCLUM_PREFIX)/build_sim/bin/ -D build_sim/bin/* + install -d $(OCCLUM_PREFIX)/build_sim/lib/ + install -t $(OCCLUM_PREFIX)/build_sim/lib/ -D build_sim/lib/* install -d $(OCCLUM_PREFIX)/src/ install -t $(OCCLUM_PREFIX)/src/ -m 444 src/sgxenv.mk install -d $(OCCLUM_PREFIX)/src/libos/ @@ -54,3 +69,5 @@ install: clean: @$(MAKE) --no-print-directory -C src clean @$(MAKE) --no-print-directory -C test clean + @$(MAKE) SGX_MODE=SIM --no-print-directory -C src clean + @$(MAKE) SGX_MODE=SIM --no-print-directory -C test clean diff --git a/deps/sefs b/deps/sefs index 6bdce43e..984dd12e 160000 --- a/deps/sefs +++ b/deps/sefs @@ -1 +1 @@ -Subproject commit 6bdce43eafde51ec9eff4fb71c0106747b18a7d1 +Subproject commit 984dd12e414e554000034514ac41a69117ccffa7 diff --git a/src/sgxenv.mk b/src/sgxenv.mk index f791766b..61181057 100644 --- a/src/sgxenv.mk +++ b/src/sgxenv.mk @@ -2,12 +2,22 @@ MAIN_MAKEFILE := $(firstword $(MAKEFILE_LIST)) INCLUDE_MAKEFILE := $(lastword $(MAKEFILE_LIST)) CUR_DIR := $(shell dirname $(realpath $(MAIN_MAKEFILE))) PROJECT_DIR := $(realpath $(CUR_DIR)/../../) -BUILD_DIR := $(PROJECT_DIR)/build SGX_SDK ?= /opt/intel/sgxsdk SGX_MODE ?= HW SGX_ARCH ?= x64 +ifneq ($(SGX_MODE), HW) + BUILD_DIR := $(PROJECT_DIR)/build_sim +else + BUILD_DIR := $(PROJECT_DIR)/build +endif + +# for sgxenv.mk in .occlum +ifeq ($(CONTEXT), 1) + BUILD_DIR := $(PROJECT_DIR)/build +endif + # If OCCLUM_RELEASE_BUILD equals to 1, y, or yes, then build in release mode OCCLUM_RELEASE_BUILD ?= 0 ifeq ($(OCCLUM_RELEASE_BUILD), yes) diff --git a/test/Makefile b/test/Makefile index 0d90a9fc..24e3def6 100644 --- a/test/Makefile +++ b/test/Makefile @@ -1,6 +1,12 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) PROJECT_DIR := $(realpath $(CUR_DIR)/../) -BUILD_DIR := $(PROJECT_DIR)/build +SGX_MODE ?= HW + +ifneq ($(SGX_MODE), HW) + BUILD_DIR := $(PROJECT_DIR)/build_sim +else + BUILD_DIR := $(PROJECT_DIR)/build +endif # Dependencies: need to be compiled but not to run by any Makefile target TEST_DEPS := client data_sink @@ -40,7 +46,7 @@ prebuild: @$(RM) -rf $(BUILD_DIR)/test @mkdir -p $(BUILD_DIR)/test @cd $(BUILD_DIR)/test && \ - $(PROJECT_DIR)/build/bin/occlum init + $(BUILD_DIR)/bin/occlum init @cp Occlum.json Enclave.xml $(BUILD_DIR)/test/ $(BUILD_TARGETS): %: @@ -50,7 +56,7 @@ $(BUILD_TARGETS): %: postbuild: @cd $(BUILD_DIR)/test && \ - $(PROJECT_DIR)/build/bin/occlum build + $(BUILD_DIR)/bin/occlum build ############################################################################# # Test targets diff --git a/test/test_common.mk b/test/test_common.mk index 26733c90..e9f67431 100644 --- a/test/test_common.mk +++ b/test/test_common.mk @@ -2,7 +2,13 @@ MAIN_MAKEFILE := $(firstword $(MAKEFILE_LIST)) INCLUDE_MAKEFILE := $(lastword $(MAKEFILE_LIST)) CUR_DIR := $(shell dirname $(realpath $(MAIN_MAKEFILE))) PROJECT_DIR := $(realpath $(CUR_DIR)/../../) -BUILD_DIR := $(PROJECT_DIR)/build +SGX_MODE ?= HW + +ifneq ($(SGX_MODE), HW) + BUILD_DIR := $(PROJECT_DIR)/build_sim +else + BUILD_DIR := $(PROJECT_DIR)/build +endif TEST_NAME := $(shell basename $(CUR_DIR)) IMAGE_DIR := $(BUILD_DIR)/test/image @@ -58,7 +64,7 @@ $(BUILD_DIR)/test/obj/$(TEST_NAME)/%.o: %.cc test: @cd $(BUILD_DIR)/test && \ - $(PROJECT_DIR)/build/bin/occlum run /bin/$(TEST_NAME) $(BIN_ARGS) + $(BUILD_DIR)/bin/occlum run /bin/$(TEST_NAME) $(BIN_ARGS) test-native: @LD_LIBRARY_PATH=/usr/local/occlum/lib cd $(IMAGE_DIR) && ./bin/$(TEST_NAME) $(BIN_ARGS) diff --git a/tools/Makefile b/tools/Makefile index d6b73bab..64f1af89 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -1,10 +1,18 @@ +SGX_MODE ?= HW + +ifneq ($(SGX_MODE), HW) + BUILD_DIR := build_sim +else + BUILD_DIR := build +endif + .PHONY: all clean all: - @mkdir -p ../build/bin/ - @ln -s -f ../../tools/occlum ../build/bin/occlum - @ln -s -f ../../tools/occlum-build-enclave ../build/bin/occlum-build-enclave - @ln -s -f ../../tools/occlum-gen-default-occlum-json ../build/bin/occlum-gen-default-occlum-json + @mkdir -p ../$(BUILD_DIR)/bin/ + @ln -s -f ../../tools/occlum ../$(BUILD_DIR)/bin/occlum + @ln -s -f ../../tools/occlum-build-enclave ../$(BUILD_DIR)/bin/occlum-build-enclave + @ln -s -f ../../tools/occlum-gen-default-occlum-json ../$(BUILD_DIR)/bin/occlum-gen-default-occlum-json @$(MAKE) --no-print-directory -C protect-integrity clean: diff --git a/tools/occlum b/tools/occlum index 5f85a19f..a8bb2e72 100755 --- a/tools/occlum +++ b/tools/occlum @@ -68,7 +68,7 @@ get_conf_entry_points() { } get_occlum_conf_file_mac() { - "$occlum_dir/build/bin/occlum-protect-integrity" show-mac "$context_dir/build/Occlum.json.protected" + "$occlum_dir/$build_dir/bin/occlum-protect-integrity" show-mac "$context_dir/build/Occlum.json.protected" } parse_occlum_user_space_size() { @@ -108,8 +108,27 @@ cmd_init() { } cmd_build() { + build_dir=build + + while [ -n "$1" ]; do + case "$1" in + --sign-key) [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path" ;; + --sign-tool) [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path" ;; + --sgx-mode) [[ -n "$2" && "$2" != "HW" ]] && export SGX_MODE=SW && build_dir=build_sim ; shift 2 || exit_error "empty sgx mode";; + *) exit_error "Unknown option: $1" ;; + esac + done + [ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY" + [ -e "$ENCLAVE_SIGN_TOOL" ] || exit_error "invalid signing tool path: $ENCLAVE_SIGN_TOOL" + echo "Enclave sign-tool: $ENCLAVE_SIGN_TOOL" + echo "Enclave sign-key: $ENCLAVE_SIGN_KEY" + if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SGX_SDK/lib64 + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$SGX_SDK/sdk_libs + build_dir=build_sim + echo "SGX mode: $SGX_MODE" + else + echo "SGX mode: HW" fi cd "$context_dir" @@ -118,24 +137,24 @@ cmd_build() { rm -rf build mkdir -p build/bin - cp "$occlum_dir/build/bin/occlum-run" build/bin/ + cp "$occlum_dir/$build_dir/bin/occlum-run" build/bin/ mkdir -p build/lib - cp "$occlum_dir/build/lib/libocclum-libos-core.a" build/lib/ - cp "$occlum_dir/build/lib/libcompiler-rt-patch.a" build/lib/ - cp "$occlum_dir/build/lib/libocclum-pal.so" build/lib/ + cp "$occlum_dir/$build_dir/lib/libocclum-libos-core.a" build/lib/ + cp "$occlum_dir/$build_dir/lib/libcompiler-rt-patch.a" build/lib/ + cp "$occlum_dir/$build_dir/lib/libocclum-pal.so" build/lib/ mkdir -p build/src/libos/src/builtin chmod 531 -R $working_dir/image/bin chmod 531 -R $working_dir/image/lib mkdir -p build/mount/ - cd "$occlum_dir/deps/sefs/sefs-fuse/bin/" && \ - ./app \ - --integrity-only \ - "$context_dir/build/mount/__ROOT" \ - "$working_dir/image" \ - zip + cd "$occlum_dir/$build_dir/bin/" && \ + ./sefs-fuse \ + --integrity-only \ + "$context_dir/build/mount/__ROOT" \ + "$working_dir/image" \ + zip - export OCCLUM_CONF_ROOT_FS_MAC=`"$occlum_dir/build/bin/occlum-protect-integrity" show-mac "$context_dir/build/mount/__ROOT/metadata"` + export OCCLUM_CONF_ROOT_FS_MAC=`"$occlum_dir/$build_dir/bin/occlum-protect-integrity" show-mac "$context_dir/build/mount/__ROOT/metadata"` export OCCLUM_CONF_USER_SPACE_SIZE=`get_conf_user_space_size` export OCCLUM_CONF_DEFAULT_STACK_SIZE=`get_conf_default_stack_size` export OCCLUM_CONF_DEFAULT_HEAP_SIZE=`get_conf_default_heap_size` @@ -143,9 +162,9 @@ cmd_build() { export OCCLUM_CONF_ENV=`get_conf_env` export OCCLUM_CONF_ENTRY_POINTS=`get_conf_entry_points` cd "$context_dir/build" - "$occlum_dir/build/bin/occlum-gen-default-occlum-json"\ + "$occlum_dir/$build_dir/bin/occlum-gen-default-occlum-json"\ > "Occlum.json" - "$occlum_dir/build/bin/occlum-protect-integrity" protect Occlum.json + "$occlum_dir/$build_dir/bin/occlum-protect-integrity" protect Occlum.json export OCCLUM_BUILTIN_CONF_FILE_MAC=`get_occlum_conf_file_mac` echo "EXPORT => OCCLUM_BUILTIN_CONF_FILE_MAC = $OCCLUM_BUILTIN_CONF_FILE_MAC" @@ -160,18 +179,7 @@ cmd_build() { cp -r "$occlum_dir/src/libos/src/builtin" src/libos/src/builtin cd src/libos && \ make clean-builtin && \ - make "$context_dir/build/lib/libocclum-libos.so" ONLY_REBUILD_BUILTIN=1 - while [ -n "$1" ]; do - case "$1" in - --sign-key) [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path" ;; - --sign-tool) [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path" ;; - *) exit_error "Unknown option: $1" ;; - esac - done - [ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY" - [ -e "$ENCLAVE_SIGN_TOOL" ] || exit_error "invalid signing tool path: $ENCLAVE_SIGN_TOOL" - echo "Enclave sign-tool: $ENCLAVE_SIGN_TOOL" - echo "Enclave sign-key: $ENCLAVE_SIGN_KEY" + make "$context_dir/build/lib/libocclum-libos.so" ONLY_REBUILD_BUILTIN=1 CONTEXT=1 $ENCLAVE_SIGN_TOOL sign \ -key $ENCLAVE_SIGN_KEY \ -config "$working_dir/Enclave.xml" \ @@ -182,12 +190,19 @@ cmd_build() { mkdir -p "$context_dir/run/mount/root" + if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then + echo "SIM" > .sgx_mode + else + echo "HW" > .sgx_mode + fi + echo "Built the Occlum image and enclave successfully" } cmd_run() { + SGX_MODE=$(cat $context_dir/.sgx_mode) if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then - export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/lib64/" + export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/sdk_libs/" else export LD_LIBRARY_PATH="$context_dir/build/lib" fi @@ -201,8 +216,9 @@ cmd_run() { } cmd_gdb() { + SGX_MODE=$(cat $context_dir/.sgx_mode) if [[ -n $SGX_MODE && "$SGX_MODE" != "HW" ]]; then - export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/lib64/" + export LD_LIBRARY_PATH="$context_dir/build/lib:$SGX_SDK/sdk_libs/" else export LD_LIBRARY_PATH="$context_dir/build/lib" fi diff --git a/tools/protect-integrity/Makefile b/tools/protect-integrity/Makefile index b3c35ec9..58928456 100644 --- a/tools/protect-integrity/Makefile +++ b/tools/protect-integrity/Makefile @@ -2,14 +2,19 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) PROJECT_DIR := $(realpath $(CUR_DIR)/../../) -BUILD_DIR := $(PROJECT_DIR)/build -OBJS_DIR := $(BUILD_DIR)/tools/protect-integrity SGX_SDK ?= /opt/intel/sgxsdk SGX_MODE ?= HW SGX_ARCH ?= x64 SGX_DEBUG ?= 1 +ifneq ($(SGX_MODE), HW) + BUILD_DIR := $(PROJECT_DIR)/build_sim +else + BUILD_DIR := $(PROJECT_DIR)/build +endif +OBJS_DIR := $(BUILD_DIR)/tools/protect-integrity + ifeq ($(shell getconf LONG_BIT), 32) SGX_ARCH := x86 else ifeq ($(findstring -m32, $(CXXFLAGS)), -m32)