From f2b4e96ed0d55d3e4342d15936f1340130905181 Mon Sep 17 00:00:00 2001 From: "Tate, Hongliang Tian" Date: Sat, 4 Jan 2020 18:18:28 +0000 Subject: [PATCH] Add a demo for the embedded mode --- demos/README.md | 3 +- demos/embedded_mode/.gitignore | 2 + demos/embedded_mode/Makefile | 25 +++++ demos/embedded_mode/README.md | 65 +++++++++++++ demos/embedded_mode/bench_driver/.gitignore | 1 + demos/embedded_mode/bench_driver/Makefile | 37 ++++++++ demos/embedded_mode/bench_driver/main.c | 76 ++++++++++++++++ .../trusted_memcpy_bench/.gitignore | 1 + .../trusted_memcpy_bench/Makefile | 32 +++++++ .../embedded_mode/trusted_memcpy_bench/main.c | 91 +++++++++++++++++++ 10 files changed, 332 insertions(+), 1 deletion(-) create mode 100644 demos/embedded_mode/.gitignore create mode 100644 demos/embedded_mode/Makefile create mode 100644 demos/embedded_mode/README.md create mode 100644 demos/embedded_mode/bench_driver/.gitignore create mode 100644 demos/embedded_mode/bench_driver/Makefile create mode 100644 demos/embedded_mode/bench_driver/main.c create mode 100644 demos/embedded_mode/trusted_memcpy_bench/.gitignore create mode 100644 demos/embedded_mode/trusted_memcpy_bench/Makefile create mode 100644 demos/embedded_mode/trusted_memcpy_bench/main.c diff --git a/demos/README.md b/demos/README.md index c72cb8b6..d64a8ea8 100644 --- a/demos/README.md +++ b/demos/README.md @@ -20,6 +20,7 @@ This set of demos shows how real-world apps can be easily run inside SGX enclave * `tensorflow_lite/`: A demo and benchmark of [Tensorflow Lite](https://www.tensorflow.org/lite) inference engine. * `xgboost/`: A demo of [XGBoost](https://xgboost.readthedocs.io/en/latest/). -## SGX capability demos +## Other demos * `remote_attestation/`: This project demonstrates how an app running upon Occlum can perform SGX remote attestation. +* `embedded_mode/`: A cross-enclave memory throughput benchmark enabled by the embedded mode of Occlum. diff --git a/demos/embedded_mode/.gitignore b/demos/embedded_mode/.gitignore new file mode 100644 index 00000000..d271a39c --- /dev/null +++ b/demos/embedded_mode/.gitignore @@ -0,0 +1,2 @@ +.occlum +occlum_context diff --git a/demos/embedded_mode/Makefile b/demos/embedded_mode/Makefile new file mode 100644 index 00000000..113a3ff4 --- /dev/null +++ b/demos/embedded_mode/Makefile @@ -0,0 +1,25 @@ +.PHONY: all build_src test clean + +all: occlum_context + +occlum_context: build_src + @mkdir -p occlum_context + @cd occlum_context && \ + occlum init && \ + cp ../trusted_memcpy_bench/build/trusted_memcpy_bench image/bin/ && \ + occlum build + @cp -r occlum_context/.occlum .occlum + +build_src: + @$(MAKE) --no-print-directory -C trusted_memcpy_bench + @$(MAKE) --no-print-directory -C bench_driver + +TOTAL_BYTES := 10000000000 # 10GB +test: + LD_LIBRARY_PATH=/opt/occlum/build/lib RUST_BACKTRACE=1 \ + ./bench_driver/build/bench_driver $(TOTAL_BYTES) + +clean: + @$(MAKE) --no-print-directory -C trusted_memcpy_bench clean + @$(MAKE) --no-print-directory -C bench_driver clean + @rm -rf .occlum occlum_context diff --git a/demos/embedded_mode/README.md b/demos/embedded_mode/README.md new file mode 100644 index 00000000..c42ad474 --- /dev/null +++ b/demos/embedded_mode/README.md @@ -0,0 +1,65 @@ +# A Demo for Occlum's Embedded Mode + +## Background + +There are two main approaches to building SGX applications: the SDK-based +approach (e.g., using Intel SGX SDK) and the LibOS-based approach (e.g., using +Occlum). The SDK-based approach usually requires developers to build an SGX +application by partitioning it into trusted and untrusted halves, while the +LibOS-based approach runs the entire application inside an enclave. + +Both approaches have their pros and cons. The SDK-based approach lets the +developers decide which components are to be or not to be put into the enclave. +Thus, it provides the flexibility and customizability that is attractive to +advanced developers. However, this requires non-trivial efforts from the +developers, especially when porting existing applications or libraries into +enclaves. Furthermore, whichever the SDK being used, the developers are +typically bound to a specific programming language and only provided with a +subset of the functionality or features that are supported by the programming +language and its libraries. + +In contrast, the LibOS-based approach offers binary-level or code-level +compatibility so that a legacy application or library can be ported into an +enclave with minimal effort. But as the whole application is hosted by the +LibOS inside the enclave, the developers are given no mechanism to efficiently +offloading some functionalities of the application outside the enclave. + +## The Embedded Mode + +The embedded mode of Occlum brings the advantages of the SDK-based approach to +the LibOS-based approach. As the name suggests, this mode enables developers to +embed Occlum in an SGX application: link Occlum as a shared library to the SGX +application and use Occlum's APIs to load and execute trusted programs in an +enclave. This gives the developers both a complete control over the untrusted +components outside the enclave and a Linux-compatible environment for the +trusted programs inside the enclave. The trusted programs and the untrusted +components can communicate with each other efficiently via shared (untrusted) +memory. In short, the embedded mode combines the best of the two approaches. + +## This Demo + +To demonstrate the usage and the advantage of the embedded mode, we provide a +benchmark program that measures the cross-enclave memory throughput, where data +is `memcpy`'ed from an untrusted component outside the enclave to a trusted +program inside the enclave. + +The trusted program is under `trusted_memcpy_bench/`. Running upon Occlum, this +program is given an untrusted buffer outside the enclave and measures the I/O +throughput achieved by repeatedly `memcpy` from it. + +The untrusted component is under `bench_driver/`, which is a normal Linux +program except that is linked with the Occlum PAL library and uses Occlum PAL +APIS to load and execute `trusted_memcpy_bench` program. The untrusted buffer +required by `trusted_memcpy_bench` is prepared by `bench_driver`. + +## How to Build and Run + +To build the two components, use the following command +``` +make +``` + +To run the benchmark, use the following command +``` +make test +``` diff --git a/demos/embedded_mode/bench_driver/.gitignore b/demos/embedded_mode/bench_driver/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/demos/embedded_mode/bench_driver/.gitignore @@ -0,0 +1 @@ +build diff --git a/demos/embedded_mode/bench_driver/Makefile b/demos/embedded_mode/bench_driver/Makefile new file mode 100644 index 00000000..bf0be648 --- /dev/null +++ b/demos/embedded_mode/bench_driver/Makefile @@ -0,0 +1,37 @@ +SGX_SDK ?= /opt/intel/sgxsdk +OCCLUM_PREFIX ?= /opt/occlum + +BUILD_DIR := build +BIN := $(BUILD_DIR)/bench_driver + +C_SRCS := $(sort $(wildcard *.c)) +C_OBJS := $(addprefix $(BUILD_DIR)/,$(C_SRCS:.c=.o)) + +C_FLAGS := -Wall \ + -I$(SGX_SDK)/include \ + -I$(OCCLUM_PREFIX)/include +LINK_FLAGS := $(C_FLAGS) -lpthread \ + -L$(SGX_SDK)/lib64 -lsgx_urts -lsgx_uprotected_fs \ + -L$(OCCLUM_PREFIX)/build/lib -locclum-pal + +ALL_BUILD_SUBDIRS := $(sort $(patsubst %/,%,$(dir $(BIN) $(C_OBJS)))) + +.PHONY: all clean + +all: $(BIN) + +$(BIN) $(C_OBJS): $(ALL_BUILD_SUBDIRS) + +$(BIN): $(C_OBJS) + @$(CC) $(C_OBJS) -o $@ $(LINK_FLAGS) + @echo "LINK => $@" + +$(BUILD_DIR)/%.o: %.c + @$(CC) $(C_FLAGS) -c $< -o $@ + @echo "CC <= $@" + +$(ALL_BUILD_SUBDIRS): + @mkdir -p $@ + +clean: + @-$(RM) -rf $(BUILD_DIR) diff --git a/demos/embedded_mode/bench_driver/main.c b/demos/embedded_mode/bench_driver/main.c new file mode 100644 index 00000000..615c2bdc --- /dev/null +++ b/demos/embedded_mode/bench_driver/main.c @@ -0,0 +1,76 @@ +#include +#include +#include +#include + +//============================================================================ +// Help message +//============================================================================ + +#define HELP_MSG \ + "%s\n" \ + "A benchmark program that measures the memory throughput across the enclave.\n" \ + "\n" \ + "Usage:\n" \ + " %s \n" \ + "\n" \ + "Arguments:\n" \ + " The total number of bytes that are copied from the outside of an enclave to the inside" \ + "\n" \ + "Note:\n" \ + " This simple benchmark program showcases the power of the embedded mode of Occlum, " \ + "which enables sharing memory between the inside and outside of an enclave." \ + "The embedded mode makes it possible to build Occlum-based SGX apps " \ + "that comprise of trusted and untrused halves.\n" + +static void print_help_msg(const char* prog_name) { + fprintf(stderr, HELP_MSG, prog_name, prog_name); +} + +//============================================================================ +// Main +//============================================================================ + +int main(int argc, char* argv[]) { + // Parse arguments + const char* prog_name = (const char*)argv[0]; + if (argc < 2) { + fprintf(stderr, "error: require one argument\n\n"); + print_help_msg(prog_name); + return EXIT_FAILURE; + } + const char* total_bytes_str = argv[1]; + + // Init Occlum PAL + const char* occlum_instance_dir = ".occlum"; + if (occlum_pal_init(occlum_instance_dir) < 0) { + return EXIT_FAILURE; + } + + // The buffer shared between the outside and inside the enclave + char shared_buf[1024 * 1024] = {0}; + + // Prepare cmd path and arguments + const char* cmd_path = "/bin/trusted_memcpy_bench"; + char buf_ptr_str[32] = {0}; + char buf_size_str[32] = {0}; + snprintf(buf_ptr_str, sizeof buf_ptr_str, "%lu", (unsigned long) shared_buf); + snprintf(buf_size_str, sizeof buf_size_str, "%lu", sizeof shared_buf); + const char* cmd_args[] = { + buf_ptr_str, // buf_ptr + buf_size_str, // buf_size + total_bytes_str, // total_bytes + NULL + }; + + // Use Occlum PAL to execute the cmd + int exit_status = 0; + if (occlum_pal_exec(cmd_path, cmd_args, &exit_status) < 0) { + return EXIT_FAILURE; + } + + // Destroy Occlum PAL + occlum_pal_destroy(); + + return exit_status; +} diff --git a/demos/embedded_mode/trusted_memcpy_bench/.gitignore b/demos/embedded_mode/trusted_memcpy_bench/.gitignore new file mode 100644 index 00000000..378eac25 --- /dev/null +++ b/demos/embedded_mode/trusted_memcpy_bench/.gitignore @@ -0,0 +1 @@ +build diff --git a/demos/embedded_mode/trusted_memcpy_bench/Makefile b/demos/embedded_mode/trusted_memcpy_bench/Makefile new file mode 100644 index 00000000..a7c83ccb --- /dev/null +++ b/demos/embedded_mode/trusted_memcpy_bench/Makefile @@ -0,0 +1,32 @@ +# Program targeted for Occlum need to be built with the Occlum toolchain +CC := occlum-gcc + +BUILD_DIR := build +BIN := $(BUILD_DIR)/trusted_memcpy_bench + +C_SRCS := $(wildcard *.c) +C_OBJS := $(addprefix $(BUILD_DIR)/,$(C_SRCS:.c=.o)) +C_FLAGS := -Wall +LINK_FLAGS := $(C_FLAGS) + +ALL_BUILD_SUBDIRS := $(sort $(patsubst %/,%,$(dir $(BIN) $(C_OBJS)))) + +.PHONY: all test clean + +all: $(BIN) + +$(BIN) $(C_OBJS): $(ALL_BUILD_SUBDIRS) + +$(BIN): $(C_OBJS) + @$(CC) $(C_OBJS) -o $@ $(LINK_FLAGS) + @echo "LINK => $@" + +$(BUILD_DIR)/%.o: %.c + @$(CC) $(C_FLAGS) -c $< -o $@ + @echo "CC <= $@" + +$(ALL_BUILD_SUBDIRS): + @mkdir -p $@ + +clean: + @-$(RM) -rf $(BUILD_DIR) diff --git a/demos/embedded_mode/trusted_memcpy_bench/main.c b/demos/embedded_mode/trusted_memcpy_bench/main.c new file mode 100644 index 00000000..578cdc66 --- /dev/null +++ b/demos/embedded_mode/trusted_memcpy_bench/main.c @@ -0,0 +1,91 @@ +#include +#include +#include +#include + +//============================================================================ +// Help message +//============================================================================ + +#define HELP_MSG \ + "Usage: %s \n" \ + "\n" \ + "Arguments:\n" \ + " The pointer to an untrusted buffer outside the enclave\n" \ + " The size of the untrusted buffer\n" \ + " The total number of bytes to copy from the buffer into the enclave\n" + +static void print_help_msg(const char* prog_name) { + fprintf(stderr, HELP_MSG, prog_name); +} + +//============================================================================ +// Data consumption +//============================================================================ + +#define MIN(x, y) ((x) <= (y) ? (x) : (y)) + +static int copy_into_enclave(const char* src_buf, size_t buf_size, size_t total_bytes) { + char* dst_buf = malloc(buf_size); + if (dst_buf == NULL) { + fprintf(stderr, "ERROR: out of memory"); + return -1; + } + + while (total_bytes > 0) { + size_t copy_bytes = MIN(buf_size, total_bytes); + memcpy(dst_buf, src_buf, copy_bytes); + total_bytes -= copy_bytes; + } + + free(dst_buf); + return 0; +} + +//============================================================================ +// Main +//============================================================================ + +int main(int argc, char* argv[]) { + // Parse arguments + const char* prog_name = argv[0]; + if (argc < 4) { + print_help_msg(prog_name); + return EXIT_FAILURE; + } + const char* buf_ptr = (const char*) strtoul(argv[1], NULL, 10); + size_t buf_size = (size_t) strtoul(argv[2], NULL, 10); + size_t total_bytes = (size_t) strtoul(argv[3], NULL, 10); + if (buf_ptr == NULL || buf_size == 0 || total_bytes == 0) { + print_help_msg(prog_name); + return EXIT_FAILURE; + } + + // Benchmark memcpy from outside the enclave to inside the enclave + printf("Start copying data from the given buffer (ptr = %p, len = %lu) for a total of %lu bytes...\n", + buf_ptr, buf_size, total_bytes); + + // Time begin + struct timeval time_begin, time_end; + gettimeofday(&time_begin, NULL); + // Do memcpy for a total of `total_bytes` bytes + int ret = copy_into_enclave(buf_ptr, buf_size, total_bytes); + if (ret < 0) { + return EXIT_FAILURE; + } + // Time end + gettimeofday(&time_end, NULL); + printf("Done.\n"); + + // Calculate the throughput + unsigned long elapsed_us = (time_end.tv_sec - time_begin.tv_sec) * 1000000 + + (time_end.tv_usec - time_begin.tv_usec); + if (elapsed_us == 0) { + fprintf(stderr, "ERROR: elapsed time (in us) cannot be zero"); + print_help_msg(prog_name); + return EXIT_FAILURE; + } + printf("Cross-enclave memcpy throughput = %lu MB/s\n", total_bytes / elapsed_us); + + return EXIT_SUCCESS; +}