Upgrade Dockerfile for dynamic-linking support

src/libos/src/process/spawn/mod.rs

@@ -58,7 +58,7 @@ pub fn do_spawn<P: AsRef<Path>>(
     };
 
     let mut ldso_elf_buf = {
-        let ldso_path = "/ld.so";
+        let ldso_path = "/lib/ld-musl-x86_64.so.1";
         let ldso_inode = ROOT_INODE.lookup(ldso_path)?;
         ldso_inode.read_as_vec()?
     };

src/libos/src/vm/process_vm.rs

@@ -26,8 +26,8 @@ macro_rules! impl_setter_for_process_vm_builder {
 
 impl ProcessVMBuilder {
     pub const DEFAULT_STACK_SIZE: usize = 1 * 1024 * 1024;
-    pub const DEFAULT_HEAP_SIZE: usize = 8 * 1024 * 1024;
+    pub const DEFAULT_HEAP_SIZE: usize = 16 * 1024 * 1024;
-    pub const DEFAULT_MMAP_SIZE: usize = 8 * 1024 * 1024;
+    pub const DEFAULT_MMAP_SIZE: usize = 16 * 1024 * 1024;
 
     pub fn new(code_size: usize, data_size: usize) -> ProcessVMBuilder {
         ProcessVMBuilder {

src/pal/pal.c

@@ -242,17 +242,18 @@ int SGX_CDECL main(int argc, const char *argv[])
         return -1;
     }
 
-    sgx_ret = libos_boot(global_eid, &status, executable_path, &argv[2]);
-    if(sgx_ret != SGX_SUCCESS) {
-        print_error_message(sgx_ret);
-        return status;
-    }
     // First ecall do a lot initializations.
     // Count it as startup time.
     dummy_ecall(global_eid, &status);
 
     gettimeofday(&libosready, NULL);
 
+    sgx_ret = libos_boot(global_eid, &status, executable_path, &argv[2]);
+    if(sgx_ret != SGX_SUCCESS) {
+        print_error_message(sgx_ret);
+        return status;
+    }
+
     status = wait_all_tasks();
 
     gettimeofday(&appdie, NULL);

test/Makefile

@@ -45,7 +45,11 @@ $(BUILD_TARGETS): %:
 
 sefs:
 	@$(RM) -rf $(SEFS_PATH)
-	@cp /lib/ld-musl-x86_64.so.1 $(FS_PATH)/ld.so
+	@mkdir -p $(FS_PATH)/lib/
+	@cp /lib/ld-musl-x86_64.so.1 $(FS_PATH)/lib/
+	@cp /usr/local/occlum/lib/libc++.so.1 $(FS_PATH)/lib/
+	@cp /usr/local/occlum/lib/libc++abi.so.1 $(FS_PATH)/lib/
+	@cp /usr/local/occlum/lib/libunwind.so.1 $(FS_PATH)/lib/
 	@cd $(PROJECT_DIR)/deps/sefs/sefs-fuse/bin/ && \
 		./app \
 			$(CUR_DIR)/$(SEFS_PATH) \

test/test_common.mk

@@ -20,7 +20,6 @@ CLANG_BIN_PATH := $(shell clang -print-prog-name=clang)
 LLVM_PATH := $(abspath $(dir $(CLANG_BIN_PATH))../)
 
 C_FLAGS = -Wall -I../include -O2 -fPIC $(EXTRA_C_FLAGS)
-C_FLAGS += -Xclang -load -Xclang $(LLVM_PATH)/lib/LLVMMDSFIIRInserter.so
 LINK_FLAGS = $(C_FLAGS) -pie $(EXTRA_LINK_FLAGS)
 
 .PHONY: all test debug clean
@@ -36,18 +35,17 @@ $(BIN_PATH): $(BIN_NAME)
 	@cp $^ $@
 	@echo "COPY => $@"
 
-debug: $(OBJDUMP_FILE) $(READELF_FILE)
+# Compile C/C++ test program
-
+#
-$(OBJDUMP_FILE): $(BIN_NAME)
+# When compiling programs, we do not use CXX if we're not compilng any C++ files.
-	@objdump -d $(BIN_NAME) > $(OBJDUMP_FILE)
+# This ensures C++ libraries are only linked and loaded for C++ programs, not C 
-	@echo "OBJDUMP => $@"
+# programs.
-
-$(READELF_FILE): $(BIN_NAME)
-	@readelf -a -d $(BIN_NAME) > $(READELF_FILE)
-	@echo "READELF => $@"
-
 $(BIN_NAME): $(C_OBJS) $(CXX_OBJS)
-	@$(CXX) $^ $(LINK_FLAGS) -o $(BIN_NAME)
+	@if [ -z $(CXX_OBJS) ] ; then \
+		$(CC) $^ $(LINK_FLAGS) -o $(BIN_NAME); \
+	else \
+		$(CXX) $^ $(LINK_FLAGS) -o $(BIN_NAME); \
+	fi ;
 	@echo "LINK => $@"
 
 $(C_OBJS): %.o: %.c

tools/docker/Dockerfile

@@ -39,16 +39,13 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     rm -rf /var/lib/apt/lists/*
 
 # Install SGX SDK
-WORKDIR /root/occlum/linux-sgx
+WORKDIR /tmp
-RUN git clone -b for_occlum https://github.com/occlum/linux-sgx . && \
+RUN git clone https://github.com/occlum/linux-sgx . && \
     ./download_prebuilt.sh && \
-    make && \
+    ./compile.sh && \
-    make sdk_install_pkg && \
+    ./install.sh && \
-    make deb_sgx_enclave_common_pkg && \
-    dpkg -i ./linux/installer/deb/libsgx-enclave-common/libsgx-enclave-common_*.deb && \
-    echo -e 'no\n/opt/intel' | ./linux/installer/bin/sgx_linux_x64_sdk_*.bin && \
     echo 'source /opt/intel/sgxsdk/environment' >> /root/.bashrc && \
-    rm -rf /root/occlum/linux-sgx
+    rm -rf /tmp/linux-sgx
 
 # Install Rust
 ENV OCCLUM_RUST_VERSION=nightly-2019-01-28
@@ -58,7 +55,9 @@ RUN curl https://sh.rustup.rs -sSf | \
     rm -rf /root/.cargo/registry && rm -rf /root/.cargo/git
 
 # Install Occlum toolchain
-WORKDIR /root/occlum/
+WORKDIR /tmp
-COPY build_toolchain.sh /root/occlum/
+COPY build_toolchain.sh /tmp
 RUN ./build_toolchain.sh
 ENV PATH="/usr/local/occlum/bin:$PATH"
+
+WORKDIR /root

tools/docker/build_toolchain.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-BUILD_DIR=/root/occlum/toolchain
+BUILD_DIR=/tmp/occlum_toolchain
 INSTALL_DIR=/usr/local/occlum
 
 # Exit if any command fails
@@ -14,14 +14,15 @@ mkdir -p ${BUILD_DIR}
 cd ${BUILD_DIR}
 
 # Download all source code
-git clone -b for_occlum https://github.com/occlum/llvm
+# TODO: use Occlum's fork of LLVM for SFI
-git clone -b for_occlum https://github.com/occlum/musl
+git clone -b release_70 https://github.com/llvm-mirror/llvm
-git clone -b for_occlum https://github.com/occlum/lld
+git clone -b release_70 https://github.com/llvm-mirror/lld
 git clone -b release_70 https://github.com/llvm-mirror/clang
 git clone -b release_70 https://github.com/llvm-mirror/libcxx
 git clone -b release_70 https://github.com/llvm-mirror/libcxxabi
 git clone -b release_70 https://github.com/llvm-mirror/libunwind
 git clone -b release_70 https://github.com/llvm-mirror/compiler-rt
+git clone https://github.com/occlum/musl
 
 # Build LLVM
 mkdir llvm-build
@@ -31,7 +32,8 @@ cmake -DCMAKE_BUILD_TYPE=Release \
     -DLLVM_ENABLE_PROJECTS="clang;lld" \
     -DLLVM_TARGETS_TO_BUILD="X86" \
     ../llvm
-# Compile LLVM in a single thread (parallel compilation would consume too much memory)
+# Compiling LLVM in many threads may consume too much memory
+make -j2
 make install
 cd ..
 
@@ -46,7 +48,7 @@ cd ..
 
 # Link Linux headers
 ln -s /usr/include/linux ${INSTALL_DIR}/include/linux
-ln -s /usr/include/asm ${INSTALL_DIR}/include/asm
+ln -s /usr/include/x86_64-linux-gnu/asm ${INSTALL_DIR}/include/asm
 ln -s /usr/include/asm-generic ${INSTALL_DIR}/include/asm-generic
 
 # Build libunwind
@@ -54,11 +56,10 @@ mkdir libunwind-build
 cd libunwind-build
 cmake -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=musl-clang \
-    -DCMAKE_C_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_C_FLAGS="-O2 -fPIC" \
     -DCMAKE_CXX_COMPILER=musl-clang \
-    -DCMAKE_CXX_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_CXX_FLAGS="-O2 -fPIC" \
     -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-    -DLIBUNWIND_ENABLE_SHARED=OFF \
     -DLLVM_ENABLE_LIBCXX=ON \
     ../libunwind
 make install -j
@@ -69,11 +70,10 @@ mkdir libcxx-prebuild
 cd libcxx-prebuild
 cmake -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=musl-clang \
-    -DCMAKE_C_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_C_FLAGS="-O2 -fPIC" \
     -DCMAKE_CXX_COMPILER=musl-clang \
-    -DCMAKE_CXX_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_CXX_FLAGS="-O2 -fPIC" \
     -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-    -DLIBCXX_ENABLE_SHARED=OFF \
     -DLIBCXX_HAS_MUSL_LIBC=ON \
     ../libcxx
 make install -j
@@ -84,12 +84,11 @@ mkdir libcxxabi-build
 cd libcxxabi-build
 cmake -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=musl-clang \
-    -DCMAKE_C_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_C_FLAGS="-O2 -fPIC" \
     -DCMAKE_CXX_COMPILER=musl-clang \
-    -DCMAKE_CXX_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_CXX_FLAGS="-O2 -fPIC" \
     -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
     -DLIBCXXABI_ENABLE_PIC=ON \
-    -DLIBCXXABI_ENABLE_SHARED=OFF \
     -DLIBCXXABI_ENABLE_STATIC_UNWINDER=OFF \
     -DLIBCXXABI_LIBCXX_PATH=${INSTALL_DIR} \
     -DLIBCXXABI_USE_LLVM_UNWINDER=ON \
@@ -103,11 +102,10 @@ mkdir libcxx-build
 cd libcxx-build
 cmake -DCMAKE_BUILD_TYPE=Release \
     -DCMAKE_C_COMPILER=musl-clang \
-    -DCMAKE_C_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_C_FLAGS="-O2 -fPIC" \
     -DCMAKE_CXX_COMPILER=musl-clang \
-    -DCMAKE_CXX_FLAGS="-O2 -fPIC -locclum_stub" \
+    -DCMAKE_CXX_FLAGS="-O2 -fPIC" \
     -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR} \
-    -DLIBCXX_ENABLE_SHARED=OFF \
     -DLIBCXX_HAS_MUSL_LIBC=ON \
     -DLIBCXX_CXX_ABI=libcxxabi \
     -DLIBCXX_CXX_ABI_INCLUDE_PATHS=../libcxxabi/include \