From 22b02850a3290dd0a729a9fab529552334047818 Mon Sep 17 00:00:00 2001 From: LI Qing Date: Mon, 9 Nov 2020 16:18:49 +0800 Subject: [PATCH] Add Glibc as an optional libc and fix test cases --- Makefile | 3 ++ test/Makefile | 3 ++ test/Occlum.json | 4 +-- test/chown/main.c | 1 + test/eventfd/Makefile | 2 +- test/exit_group/Makefile | 2 +- test/fcntl/main.c | 8 ++++- test/link/main.c | 1 + test/pipe/main.c | 2 ++ test/prctl/Makefile | 2 +- test/pthread/Makefile | 2 +- test/readdir/main.c | 12 +++++++ test/server_epoll/main.c | 1 + test/signal/Makefile | 2 +- test/stat/main.c | 2 ++ test/symlink/main.c | 2 +- test/test_common.mk | 9 +++-- tools/docker/Dockerfile.centos8.1 | 13 +++++-- tools/docker/Dockerfile.ubuntu18.04 | 13 +++++-- tools/occlum | 35 +++++++++++++++---- tools/toolchains/glibc/build.sh | 27 ++++++++++++++ .../0014-libgomp-futex-occlum.diff | 0 tools/toolchains/{gcc => musl-gcc}/build.sh | 0 .../{gcc => musl-gcc}/install_zlib.sh | 0 24 files changed, 123 insertions(+), 23 deletions(-) create mode 100755 tools/toolchains/glibc/build.sh rename tools/toolchains/{gcc => musl-gcc}/0014-libgomp-futex-occlum.diff (100%) rename tools/toolchains/{gcc => musl-gcc}/build.sh (100%) rename tools/toolchains/{gcc => musl-gcc}/install_zlib.sh (100%) diff --git a/Makefile b/Makefile index 74195ce6..aa9ead39 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,9 @@ src: test: @$(MAKE) --no-print-directory -C test test +test-glibc: + @$(MAKE) --no-print-directory -C test test-glibc + OCCLUM_PREFIX ?= /opt/occlum install: $(OCCLUM_PREFIX)/sgxsdk-tools/lib64/libsgx_uae_service_sim.so @# Install both libraries for HW mode and SIM mode diff --git a/test/Makefile b/test/Makefile index deb7a8b5..1087a852 100644 --- a/test/Makefile +++ b/test/Makefile @@ -66,6 +66,9 @@ postbuild: # Test targets ############################################################################# +test-glibc: + @OCCLUM_TEST_GLIBC=1 $(MAKE) test + test: build pretest $(TEST_TARGETS) posttest pretest: diff --git a/test/Occlum.json b/test/Occlum.json index 1490c4ff..d6a981f6 100644 --- a/test/Occlum.json +++ b/test/Occlum.json @@ -2,13 +2,13 @@ "resource_limits": { "kernel_space_heap_size": "40MB", "kernel_space_stack_size": "1MB", - "user_space_size": "128MB", + "user_space_size": "224MB", "max_num_of_threads": 32 }, "process": { "default_stack_size": "4MB", "default_heap_size": "8MB", - "default_mmap_size": "32MB" + "default_mmap_size": "80MB" }, "entry_points": [ "/bin" diff --git a/test/chown/main.c b/test/chown/main.c index 9c70132e..4c8e8109 100644 --- a/test/chown/main.c +++ b/test/chown/main.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE #include #include #include diff --git a/test/eventfd/Makefile b/test/eventfd/Makefile index 8c5a2fb4..3ec3f733 100644 --- a/test/eventfd/Makefile +++ b/test/eventfd/Makefile @@ -1,5 +1,5 @@ include ../test_common.mk EXTRA_C_FLAGS := -Wno-incompatible-pointer-types-discards-qualifiers -EXTRA_LINK_FLAGS := +EXTRA_LINK_FLAGS := -lpthread BIN_ARGS := diff --git a/test/exit_group/Makefile b/test/exit_group/Makefile index 9e1b6dec..5c1ee8c1 100644 --- a/test/exit_group/Makefile +++ b/test/exit_group/Makefile @@ -1,5 +1,5 @@ include ../test_common.mk EXTRA_C_FLAGS := -EXTRA_LINK_FLAGS := +EXTRA_LINK_FLAGS := -lpthread BIN_ARGS := diff --git a/test/fcntl/main.c b/test/fcntl/main.c index e2db46d7..4374734c 100644 --- a/test/fcntl/main.c +++ b/test/fcntl/main.c @@ -1,3 +1,4 @@ +#define _LARGEFILE64_SOURCE #include #include #include "test_fs.h" @@ -5,9 +6,14 @@ // ============================================================================ // Helper macro // ============================================================================ +#ifdef __GLIBC__ +#define TMPFILE_FLAG __O_TMPFILE +#else +#define TMPFILE_FLAG O_TMPFILE +#endif #define CREATION_FLAGS_MASK (O_CLOEXEC | O_CREAT| O_DIRECTORY | O_EXCL | \ - O_NOCTTY | O_NOFOLLOW | O_TMPFILE | O_TRUNC) + O_NOCTTY | O_NOFOLLOW | TMPFILE_FLAG | O_TRUNC) // ============================================================================ // Test cases for fcntl diff --git a/test/link/main.c b/test/link/main.c index f1fbead9..b935790c 100644 --- a/test/link/main.c +++ b/test/link/main.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE #include #include #include diff --git a/test/pipe/main.c b/test/pipe/main.c index 51fa0d4a..c126e351 100644 --- a/test/pipe/main.c +++ b/test/pipe/main.c @@ -1,8 +1,10 @@ +#define _GNU_SOURCE #include #include #include #include #include +#include #include #include #include diff --git a/test/prctl/Makefile b/test/prctl/Makefile index 541855aa..482fb6f8 100644 --- a/test/prctl/Makefile +++ b/test/prctl/Makefile @@ -1,5 +1,5 @@ include ../test_common.mk EXTRA_C_FLAGS := -Wno-stringop-truncation -EXTRA_LINK_FLAGS := +EXTRA_LINK_FLAGS := -lpthread BIN_ARGS := diff --git a/test/pthread/Makefile b/test/pthread/Makefile index 9e1b6dec..5c1ee8c1 100644 --- a/test/pthread/Makefile +++ b/test/pthread/Makefile @@ -1,5 +1,5 @@ include ../test_common.mk EXTRA_C_FLAGS := -EXTRA_LINK_FLAGS := +EXTRA_LINK_FLAGS := -lpthread BIN_ARGS := diff --git a/test/readdir/main.c b/test/readdir/main.c index 7c6e6fc0..94c11b69 100644 --- a/test/readdir/main.c +++ b/test/readdir/main.c @@ -45,8 +45,10 @@ static int getdents_with_big_enough_buffer(bool use_explicit_syscall) { while (1) { if (use_explicit_syscall) { len = syscall(__NR_getdents, fd, buf, sizeof(buf)); +#ifndef __GLIBC__ } else { len = getdents(fd, (struct dirent *)buf, sizeof(buf)); +#endif } if (len < 0) { close(fd); @@ -60,10 +62,12 @@ static int getdents_with_big_enough_buffer(bool use_explicit_syscall) { return 0; } +#ifndef __GLIBC__ static int test_getdents_with_big_enough_buffer() { bool use_explicit_syscall = false; return getdents_with_big_enough_buffer(use_explicit_syscall); } +#endif static int test_getdents_via_explicit_syscall_with_big_enough_buffer() { bool use_explicit_syscall = true; @@ -80,8 +84,10 @@ static int getdents_with_too_small_buffer(bool use_explicit_syscall) { } if (use_explicit_syscall) { len = syscall(__NR_getdents, fd, buf, sizeof(buf)); +#ifndef __GLIBC__ } else { len = getdents(fd, (struct dirent *)buf, sizeof(buf)); +#endif } if (len >= 0 || errno != EINVAL) { close(fd); @@ -91,10 +97,12 @@ static int getdents_with_too_small_buffer(bool use_explicit_syscall) { return 0; } +#ifndef __GLIBC__ static int test_getdents_with_too_small_buffer() { bool use_explicit_syscall = false; return getdents_with_too_small_buffer(use_explicit_syscall); } +#endif static int test_getdents_via_explicit_syscall_with_too_small_buffer() { bool use_explicit_syscall = true; @@ -107,9 +115,13 @@ static int test_getdents_via_explicit_syscall_with_too_small_buffer() { static test_case_t test_cases[] = { TEST_CASE(test_readdir), +#ifndef __GLIBC__ TEST_CASE(test_getdents_with_big_enough_buffer), +#endif TEST_CASE(test_getdents_via_explicit_syscall_with_big_enough_buffer), +#ifndef __GLIBC__ TEST_CASE(test_getdents_with_too_small_buffer), +#endif TEST_CASE(test_getdents_via_explicit_syscall_with_too_small_buffer), }; diff --git a/test/server_epoll/main.c b/test/server_epoll/main.c index da77264b..de60f331 100644 --- a/test/server_epoll/main.c +++ b/test/server_epoll/main.c @@ -1,3 +1,4 @@ +#define _GNU_SOURCE #include #include #include diff --git a/test/signal/Makefile b/test/signal/Makefile index f5d7fb16..8740658b 100644 --- a/test/signal/Makefile +++ b/test/signal/Makefile @@ -1,5 +1,5 @@ include ../test_common.mk EXTRA_C_FLAGS := -Wno-return-stack-address -Wno-unused-but-set-variable -EXTRA_LINK_FLAGS := +EXTRA_LINK_FLAGS := -lpthread BIN_ARGS := diff --git a/test/stat/main.c b/test/stat/main.c index 7c2659ff..de21687a 100644 --- a/test/stat/main.c +++ b/test/stat/main.c @@ -1,6 +1,8 @@ +#define _GNU_SOURCE #include #include #include +#include #include "test_fs.h" // ============================================================================ diff --git a/test/symlink/main.c b/test/symlink/main.c index 4936bd5d..a5799a08 100644 --- a/test/symlink/main.c +++ b/test/symlink/main.c @@ -77,7 +77,7 @@ static int __test_readlink_from_proc_self_fd(const char *file_path) { } static int __test_realpath(const char *file_path) { - char buf[128] = { 0 }; + char buf[PATH_MAX] = { 0 }; char dir_buf[PATH_MAX] = { 0 }; char base_buf[PATH_MAX] = { 0 }; char *dir_name, *file_name, *res; diff --git a/test/test_common.mk b/test/test_common.mk index 7ad8d1b4..db7e4bc3 100644 --- a/test/test_common.mk +++ b/test/test_common.mk @@ -17,8 +17,13 @@ CXX_OBJS := $(addprefix $(BUILD_DIR)/test/obj/$(TEST_NAME)/,$(CXX_SRCS:%.cc=%.o) ALL_BUILD_SUBDIRS := $(sort $(patsubst %/,%,$(dir $(BIN) $(C_OBJS) $(CXX_OBJS)))) -CC := occlum-gcc -CXX := occlum-g++ +ifeq ($(OCCLUM_TEST_GLIBC), 1) + CC = gcc + CXX = g++ +else + CC = occlum-gcc + CXX = occlum-g++ +endif C_FLAGS = -Wall -Wno-return-local-addr -I../include -O2 -fPIC $(EXTRA_C_FLAGS) ifeq ($(SGX_MODE), SIM) diff --git a/tools/docker/Dockerfile.centos8.1 b/tools/docker/Dockerfile.centos8.1 index 46d5a31d..ad15a371 100644 --- a/tools/docker/Dockerfile.centos8.1 +++ b/tools/docker/Dockerfile.centos8.1 @@ -12,6 +12,7 @@ RUN yum install epel-release -y && \ dnf group install 'Development Tools' -y && \ dnf --enablerepo=PowerTools install -y \ astyle \ + bison \ boost-devel \ cmake \ createrepo \ @@ -19,6 +20,7 @@ RUN yum install epel-release -y && \ expect \ fuse-devel \ fuse-libs \ + gawk \ gmp-devel \ golang \ jq \ @@ -67,11 +69,16 @@ RUN curl https://sh.rustup.rs -sSf | \ cargo install sccache # Install Occlum toolchain -COPY toolchains/gcc /tmp/gcc +COPY toolchains/musl-gcc /tmp/musl-gcc WORKDIR /tmp -RUN cd gcc && ./build.sh && ./install_zlib.sh && rm -rf /tmp/gcc +RUN cd musl-gcc && ./build.sh && ./install_zlib.sh && rm -rf /tmp/musl-gcc ENV PATH="/opt/occlum/build/bin:/usr/local/occlum/bin:$PATH" +# Install glibc +COPY toolchains/glibc /tmp/glibc +WORKDIR /tmp +RUN cd glibc && ./build.sh && rm -rf /tmp/glibc + # Install Occlum Golang toolchain COPY toolchains/golang /tmp/golang WORKDIR /tmp @@ -83,7 +90,7 @@ COPY toolchains/rust /tmp/rust WORKDIR /tmp RUN cd rust && ./build.sh && rm -rf /tmp/rust ENV PATH="/opt/occlum/toolchains/rust/bin:$PATH" -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/intel/sgxsdk/sdk_libs" +ENV LD_LIBRARY_PATH="/opt/intel/sgxsdk/sdk_libs" # Install Occlum Java toolchain (JDK 11) ARG JDK11_PATH=/opt/occlum/toolchains/jvm/java-11-openjdk diff --git a/tools/docker/Dockerfile.ubuntu18.04 b/tools/docker/Dockerfile.ubuntu18.04 index 44c64e58..e6bf25d3 100644 --- a/tools/docker/Dockerfile.ubuntu18.04 +++ b/tools/docker/Dockerfile.ubuntu18.04 @@ -14,6 +14,7 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-i astyle \ autoconf \ automake \ + bison \ build-essential \ ca-certificates \ cmake \ @@ -21,6 +22,7 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y --no-i debhelper \ expect \ g++ \ + gawk \ gdb \ git-core \ golang-go \ @@ -79,11 +81,16 @@ RUN curl https://sh.rustup.rs -sSf | \ cargo install sccache # Install Occlum toolchain -COPY toolchains/gcc /tmp/gcc +COPY toolchains/musl-gcc /tmp/musl-gcc WORKDIR /tmp -RUN cd gcc && ./build.sh && ./install_zlib.sh && rm -rf /tmp/gcc +RUN cd musl-gcc && ./build.sh && ./install_zlib.sh && rm -rf /tmp/musl-gcc ENV PATH="/opt/occlum/build/bin:/usr/local/occlum/bin:$PATH" +# Install glibc +COPY toolchains/glibc /tmp/glibc +WORKDIR /tmp +RUN cd glibc && ./build.sh && rm -rf /tmp/glibc + # Install Occlum Golang toolchain COPY toolchains/golang /tmp/golang WORKDIR /tmp @@ -95,7 +102,7 @@ COPY toolchains/rust /tmp/rust WORKDIR /tmp RUN cd rust && ./build.sh && rm -rf /tmp/rust ENV PATH="/opt/occlum/toolchains/rust/bin:$PATH" -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/intel/sgxsdk/sdk_libs" +ENV LD_LIBRARY_PATH="/opt/intel/sgxsdk/sdk_libs" # Install Occlum Java toolchain (JDK 11) ARG JDK11_PATH=/opt/occlum/toolchains/jvm/java-11-openjdk diff --git a/tools/occlum b/tools/occlum index 3511cfdc..ccf81867 100755 --- a/tools/occlum +++ b/tools/occlum @@ -129,20 +129,43 @@ cmd_init() { mkdir -p image mkdir -p image/bin mkdir -p image/lib + mkdir -p image/lib64 mkdir -p image/root mkdir -p image/host mkdir -p image/tmp + local occlum_glibc_lib=/opt/occlum/glibc/lib + local cpu_lib=/sys/devices/system/cpu + if [ -d "$occlum_glibc_lib" ]; then + mkdir -p "image/$occlum_glibc_lib" + mkdir -p "image/$cpu_lib" + fi # add default /etc/hosts mkdir -p image/etc echo "127.0.0.1 localhost" > image/etc/hosts - local occlum_gcc_lib=/usr/local/occlum/x86_64-linux-musl/lib - cp -t image/lib/ \ + # add musl + local occlum_musl_lib=/usr/local/occlum/x86_64-linux-musl/lib + cp -t image/lib \ /lib/ld-musl-x86_64.so.1 \ - "$occlum_gcc_lib/libc.so" \ - "$occlum_gcc_lib/libstdc++.so.6" \ - "$occlum_gcc_lib/libgcc_s.so.1" \ - "$occlum_gcc_lib/libgomp.so.1" + "$occlum_musl_lib/libc.so" \ + "$occlum_musl_lib/libstdc++.so.6" \ + "$occlum_musl_lib/libgcc_s.so.1" \ + "$occlum_musl_lib/libgomp.so.1" + + # add glibc + if [ -d "$occlum_glibc_lib" ]; then + cp -t image/lib64 \ + "$occlum_glibc_lib/ld-linux-x86-64.so.2" + ln -sf /lib64/ld-linux-x86-64.so.2 "image/$occlum_glibc_lib/ld-linux-x86-64.so.2" + cp -t "image/$occlum_glibc_lib" \ + "$occlum_glibc_lib/libc.so.6" \ + "$occlum_glibc_lib/libpthread.so.0" \ + "$occlum_glibc_lib/libm.so.6" \ + "/usr/lib/x86_64-linux-gnu/libstdc++.so.6" \ + "/usr/lib/gcc/x86_64-linux-gnu/7/libgcc_s.so.1" + cp -t "image/$cpu_lib" \ + "$cpu_lib/online" + fi cp "$occlum_dir"/etc/template/Occlum.json "$instance_dir"/ chmod 644 "$instance_dir"/Occlum.json diff --git a/tools/toolchains/glibc/build.sh b/tools/toolchains/glibc/build.sh new file mode 100755 index 00000000..dc13e550 --- /dev/null +++ b/tools/toolchains/glibc/build.sh @@ -0,0 +1,27 @@ +#!/bin/bash +SRC_DIR=/tmp/glibc/glibc +BUILD_DIR=/tmp/glibc/glibc_build +INSTALL_DIR=/opt/occlum/glibc + +# Exit if any command fails +set -e + +# Clean previous build and installation if any +rm -rf ${SRC_DIR} +rm -rf ${BUILD_DIR} +rm -rf ${INSTALL_DIR} + +mkdir -p ${SRC_DIR} +cd ${SRC_DIR} +# Download glibc +git clone -b occlum-glibc-2.27 https://github.com/occlum/glibc . + +mkdir -p ${BUILD_DIR} +cd ${BUILD_DIR} +# Build and install glibc +unset LD_LIBRARY_PATH +CFLAGS="-O2 -g" ${SRC_DIR}/configure \ + --prefix=${INSTALL_DIR} --with-tls --without-selinux \ + --enable-stack-protector=strong --disable-nscd +make +make install diff --git a/tools/toolchains/gcc/0014-libgomp-futex-occlum.diff b/tools/toolchains/musl-gcc/0014-libgomp-futex-occlum.diff similarity index 100% rename from tools/toolchains/gcc/0014-libgomp-futex-occlum.diff rename to tools/toolchains/musl-gcc/0014-libgomp-futex-occlum.diff diff --git a/tools/toolchains/gcc/build.sh b/tools/toolchains/musl-gcc/build.sh similarity index 100% rename from tools/toolchains/gcc/build.sh rename to tools/toolchains/musl-gcc/build.sh diff --git a/tools/toolchains/gcc/install_zlib.sh b/tools/toolchains/musl-gcc/install_zlib.sh similarity index 100% rename from tools/toolchains/gcc/install_zlib.sh rename to tools/toolchains/musl-gcc/install_zlib.sh