Add gRPC glibc demo and stress test

.github/workflows/demo_test.yml

@@ -305,7 +305,7 @@ jobs:
 
 
 # Below tests needs test image to run faster
-  Grpc_test:
+  Grpc_musl_test:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v1
@@ -322,15 +322,47 @@ jobs:
       run: docker exec ${{ github.job }} bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"
 
     - name: Prepare grpc sample project
-      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc && ./prepare_client_server.sh"
+      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_musl && ./prepare_client_server.sh"
 
     - name: Run grpc server
-      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc && SGX_MODE=SIM ./run_server_on_occlum.sh" &
+      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_musl && SGX_MODE=SIM ./run_server_on_occlum.sh" &
 
     - name: Run grpc client
       run: |
         sleep ${{ env.nap_time }};
-        docker exec ${{ github.job }} bash -c "cd /root/demos/grpc && SGX_MODE=SIM ./run_client_on_occlum.sh"
+        docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_musl && SGX_MODE=SIM ./run_client_on_occlum.sh"
+
+
+  Grpc_glibc_test:
+    runs-on: ubuntu-20.04
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
+
+    - name: Create container
+      run: docker run -itd --name=${{ github.job }} -v $GITHUB_WORKSPACE:/root/occlum zhubojun/occlum:latest-ubuntu20.04-grpc
+
+    - name: Build dependencies
+      run: docker exec ${{ github.job }} bash -c "cd /root/occlum; make submodule"
+
+    - name: Make install
+      run: docker exec ${{ github.job }} bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"
+
+    - name: Prepare grpc sample project
+      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_glibc && ./prepare_client_server_glibc.sh"
+
+    - name: Run grpc server
+      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_glibc && SGX_MODE=SIM ./run_server_on_occlum_glibc.sh" &
+
+    - name: Run grpc client
+      run: |
+        sleep ${{ env.nap_time }};
+        docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_glibc && SGX_MODE=SIM ./run_client_on_occlum_glibc.sh"
+
+    - name: Run grpc stress client
+      run: docker exec ${{ github.job }} bash -c "cd /root/demos/grpc/grpc_glibc && ./run_stress_test.sh"
+
 
   Grpc_tls_test:
     runs-on: ubuntu-20.04

.github/workflows/hw_mode_test.yml

@@ -401,11 +401,11 @@ jobs:
 
     - name: Create container
       run: |
-        docker pull occlumbackup/occlum:latest-ubuntu18.04-python
+        docker pull occlumbackup/occlum:latest-ubuntu20.04-python
         if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
-          python_musl_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python);
+          python_musl_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-python);
         elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
-          python_musl_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python);
+          python_musl_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-python);
         else
           echo "Unsupported Hardware"
         fi;
@@ -465,11 +465,11 @@ jobs:
 
     - name: Create container
       run: |
-        docker pull occlumbackup/occlum:latest-ubuntu18.04-openvino
+        docker pull occlumbackup/occlum:latest-ubuntu20.04-openvino
         if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
-          openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-openvino);
+          openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-openvino);
         elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
-          openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-openvino);
+          openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-openvino);
         else
           echo "Unsupported Hardware"
         fi;
@@ -526,11 +526,11 @@ jobs:
 
     - name: Create container
       run: |
-        docker pull occlumbackup/occlum:latest-ubuntu18.04-grpc
+        docker pull occlumbackup/occlum:latest-ubuntu20.04-grpc
         if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
-          grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-grpc);
+          grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-grpc);
         elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
-          grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-grpc);
+          grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-grpc);
         else
           echo "Unsupported Hardware"
         fi;
@@ -552,16 +552,37 @@ jobs:
     - name: Make install
       run: docker exec $grpc_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"
 
-    - name: Prepare grpc sample project
-      run: docker exec $grpc_test bash -c "cd /root/demos/grpc && ./prepare_client_server.sh"
+    - name: Prepare grpc musl sample project
+      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_musl && ./prepare_client_server.sh"
 
-    - name: Run grpc server
-      run: docker exec $grpc_test bash -c "cd /root/demos/grpc && ./run_server_on_occlum.sh" &
+    - name: Run grpc musl server
+      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_musl && ./run_server_on_occlum.sh" &
 
-    - name: Run grpc client
+    - name: Run grpc musl client
       run: |
         sleep ${{ env.nap_time }};
-        docker exec $grpc_test bash -c "cd /root/demos/grpc && ./run_client_on_occlum.sh"
+        docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_musl && ./run_client_on_occlum.sh"
+
+    - name: Clean the environment
+      if: ${{ always() }}
+      run: docker stop $grpc_test
+
+    - name: Restart the docker
+      run: docker start $grpc_test
+
+    - name: Prepare grpc glibc sample project
+      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./prepare_client_server_glibc.sh"
+
+    - name: Run grpc glibc server
+      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./run_server_on_occlum_glibc.sh" &
+
+    - name: Run grpc glibc client
+      run: |
+        sleep ${{ env.nap_time }};
+        docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./run_client_on_occlum_glibc.sh"
+
+    - name: Run grpc stress test
+      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./run_stress_test.sh"
 
     - name: Clean the environment
       if: ${{ always() }}
@@ -595,11 +616,11 @@ jobs:
 
     - name: Create container
       run: |
-        docker pull occlumbackup/occlum:latest-ubuntu18.04-gvisor_test
+        docker pull occlumbackup/occlum:latest-ubuntu20.04-gvisor_test
         if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
-          gvisor_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-gvisor_test);
+          gvisor_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-gvisor_test);
         elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
-          gvisor_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-gvisor_test);
+          gvisor_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-gvisor_test);
         else
           echo "Unsupported Hardware"
         fi;
@@ -688,9 +709,9 @@ jobs:
       run: |
         cd demos/deployment
         if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
-          ./deploy_image_test.sh ubuntu18.04
+          ./deploy_image_test.sh ubuntu20.04
         elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
-          DEVICE_OPTION="-v /dev/sgx:/dev/sgx" ./deploy_image_test.sh ubuntu18.04
+          DEVICE_OPTION="-v /dev/sgx:/dev/sgx" ./deploy_image_test.sh ubuntu20.04
         else
           echo "Unsupported Hardware"
           exit 1
@@ -699,8 +720,8 @@ jobs:
     - name: Clean the environment
       if: ${{ always() }}
       run: |
-        docker stop ubuntu18.04_deploy_test
-        docker rm -f ubuntu18.04_deploy_test
+        docker stop ubuntu20.04_deploy_test
+        docker rm -f ubuntu20.04_deploy_test
 
 
 # Tensorflow_serving requires binary tensorflow_serving PIC, here we compile tensorflow_model_server before workflow
@@ -732,11 +753,11 @@ jobs:
 
     - name: Create container
       run: |
-        docker pull occlumbackup/occlum:latest-ubuntu18.04-tf_serving
+        docker pull occlumbackup/occlum:latest-ubuntu20.04-tf_serving
         if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
-          tf_serving_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-tf_serving);
+          tf_serving_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-tf_serving);
         elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
-          tf_serving_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-tf_serving);
+          tf_serving_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-tf_serving);
         else
           echo "Unsupported Hardware"
         fi;

demos/README.md

@@ -19,7 +19,7 @@ This set of demos shows how real-world apps can be easily run inside SGX enclave
 * [fish](fish/): A demo of [FISH](https://fishshell.com) shell script.
 * [flink](flink/): A demo of [Apache Flink](https://flink.apache.org).
 * [font](font/font_support_for_java): A demo of supporting font with Java.
-* [grpc](grpc/): A client and server communicating through [gRPC](https://grpc.io).
+* [grpc](grpc/): A client and server communicating through [gRPC](https://grpc.io), containing [glibc-supported demo](grpc/grpc_glibc) and [musl-supported demo](grpc/grpc_musl).
 * [https_server](https_server/): A HTTPS file server based on [Mongoose Embedded Web Server Library](https://github.com/cesanta/mongoose).
 * [openvino](openvino/) A benchmark of [OpenVINO Inference Engine](https://docs.openvinotoolkit.org/2019_R3/_docs_IE_DG_inference_engine_intro.html).
 * [pytorch](pytorch/): A demo of [PyTorch](https://pytorch.org/).

demos/grpc/grpc_glibc/.gitignore

@@ -0,0 +1,4 @@
+grpc_src/
+ghz_src/
+occlum_server_instance/
+occlum_client_instance/

demos/grpc/grpc_glibc/README.md

@@ -0,0 +1,42 @@
+# Run gRPC C++ Client/Server on Occlum
+
+## Step 1:
+Download, build and install cares, protobuf and gRPC:
+```
+./download_and_install_grpc_glibc.sh
+```
+
+## Step 2:
+Prepare the gRPC C++ Hello World sample project, which consists of a client and server:
+```
+./prepare_client_server_glibc.sh
+```
+
+## Step 3:
+Run the demo `server` which will listen on port `50051` on Occlum:
+```
+./run_server_on_occlum_glibc.sh
+```
+Then you can invoke gRPC service by running `client` in a different terminal on Occlum:
+```
+./run_client_on_occlum_glibc.sh
+```
+And you will see the "Greeter received: Hello world" in the client side output.
+
+# Run gRPC stress test by [ghz](https://ghz.sh/)
+
+## Step 1:
+Prepare the stress test tool:
+```
+./prepare_stress_test_tool.sh
+```
+
+## Step 2:
+Run the demo `server` which listen on port `50051` on Occlum:
+```
+./run_server_on_occlum_glibc.sh
+```
+Then you can run the stress test through:
+```
+./run_stress_test.sh [-n <total_requests>] [-c <concurrent_workers>]
+```

demos/grpc/grpc_glibc/download_and_install_grpc_glibc.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+set -e
+
+GRPC_SRC_DIR=$PWD/grpc_src
+
+mkdir -p $GRPC_SRC_DIR && cd $GRPC_SRC_DIR
+git clone https://github.com/grpc/grpc.git .
+git checkout tags/v1.24.3
+
+# Install c-ares
+cd $GRPC_SRC_DIR/third_party/cares/cares
+git submodule update --init .
+git checkout tags/cares-1_15_0
+mkdir -p build
+cd build
+cmake ../ \
+	-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC -pie" -DCMAKE_C_FLAGS="-fPIC -pie"
+make -j$(nproc)
+make install
+
+# Install protobuf
+cd $GRPC_SRC_DIR/third_party/protobuf
+git submodule update --init .
+git checkout tags/v3.10.0
+cd cmake
+mkdir -p build
+cd build
+cmake ../ \
+	-Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=TRUE \
+	-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC -pie" -DCMAKE_C_FLAGS="-fPIC -pie" \
+	-DCMAKE_NO_SYSTEM_FROM_IMPORTED=TRUE
+make -j$(nproc)
+make install
+
+# Install gRPC
+cd $GRPC_SRC_DIR/cmake
+mkdir -p build
+cd build
+cmake ../.. \
+	-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC -pie" -DCMAKE_C_FLAGS="-fPIC -pie" \
+	-DgRPC_INSTALL=ON -DgRPC_PROTOBUF_PROVIDER=package \
+	-DgRPC_ZLIB_PROVIDER=package -DgRPC_CARES_PROVIDER=package \
+	-DgRPC_SSL_PROVIDER=package -DCMAKE_NO_SYSTEM_FROM_IMPORTED=TRUE
+make -j$(nproc)
+make install
+
+echo "gRPC build success"

demos/grpc/grpc_glibc/grpc_client_glibc.yaml

@@ -0,0 +1,7 @@
+includes:
+  - base.yaml
+targets:
+  - target: /bin/
+    copy:
+      - files:
+        - ../grpc_src/examples/cpp/helloworld/cmake/build/greeter_client

demos/grpc/grpc_glibc/grpc_server_glibc.yaml

@@ -0,0 +1,7 @@
+includes:
+  - base.yaml
+targets:
+  - target: /bin/
+    copy:
+      - files:
+        - ../grpc_src/examples/cpp/helloworld/cmake/build/greeter_server

demos/grpc/grpc_glibc/prepare_client_server_glibc.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+set -e
+
+export LD_LIBRARY_PATH="/usr/local/lib:$LD_LIBRARY_PATH"
+
+cd grpc_src/examples/cpp/helloworld
+mkdir -p cmake/build && cd cmake/build
+cmake ../.. \
+	-DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_FLAGS="-fPIC -pie" -DCMAKE_C_FLAGS="-fPIC -pie"
+make -j$(nproc)

demos/grpc/grpc_glibc/prepare_stress_test_tool.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -e
+
+mkdir ghz_src && cd ghz_src
+git clone https://github.com/bojand/ghz .
+git checkout tags/v0.105.0
+make build

demos/grpc/grpc_glibc/run_client_on_occlum_glibc.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+if [ ! -d "occlum_client_instance" ];then
+    mkdir occlum_client_instance
+    cd occlum_client_instance
+    occlum init
+
+    rm -rf image
+    copy_bom -f ../grpc_client_glibc.yaml --root image --include-dir /opt/occlum/etc/template
+    occlum build
+else
+    cd occlum_client_instance
+fi
+
+occlum run /bin/greeter_client

demos/grpc/grpc_glibc/run_server_on_occlum_glibc.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+if [ ! -d "occlum_server_instance" ];then
+    mkdir occlum_server_instance
+    cd occlum_server_instance
+    occlum init
+
+    rm -rf image
+    copy_bom -f ../grpc_server_glibc.yaml --root image --include-dir /opt/occlum/etc/template
+    occlum build
+else
+    cd occlum_server_instance
+fi
+
+occlum run /bin/greeter_server

demos/grpc/grpc_glibc/run_stress_test.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+set -e
+
+total=200
+concurrency=50
+
+show_usage() {
+    echo ""
+    cat <<EOF
+Usage:
+    run_stress_test.sh [-n <total_requests>] [-c <concurrent_workers>]
+
+    The combination of -c and -n are critical in how the benchmarking is done.
+    It takes the -c argument and spawns that many worker goroutines. In parallel
+    these goroutines each do their share (n / c) requests.
+    For example with the default -c 50 -n 200 options we would spawn 50 goroutines
+    which in parallel each do 4 requests.
+
+EOF
+}
+
+exit_error() {
+    echo "Error: $@" >&2
+    show_usage
+    exit 1
+}
+
+while [ -n "$1" ]; do
+    case "$1" in
+    -n) [ -n "$2" ] && total=$2 ; shift 2 || exit_error "empty total number of requests to run"             ;;
+    -c) [ -n "$2" ] && concurrency=$2 ; shift 2 || exit_error "empty number of workers to run concurrently" ;;
+    *) exit_error "Unknown option: $1"                                                                      ;;
+    esac
+done
+
+# Use ghz tool to run the stress test
+./ghz_src/dist/ghz \
+    --insecure -n $total -c $concurrency \
+    --proto ./grpc_src/examples/protos/helloworld.proto \
+    --call helloworld.Greeter.SayHello \
+    -d '{"name":"World"}' localhost:50051

demos/grpc/grpc_musl/run_client_on_occlum.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 INSTALL_DIR=/usr/local/occlum/x86_64-linux-musl
 
-export PATH=$PATH:$INSTALL_DIR/bin
+export PATH=$INSTALL_DIR/bin:$PATH
 
 cd client
 

demos/grpc/grpc_musl/run_server_on_occlum.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 INSTALL_DIR=/usr/local/occlum/x86_64-linux-musl
-export PATH=$PATH:$INSTALL_DIR/bin
+export PATH=$INSTALL_DIR/bin:$PATH
 
 cd server
 

tools/docker/ci/Dockerfile.grpc

@@ -9,8 +9,12 @@ RUN rm -rf /root/demos && \
     cp -r occlum/demos /root/demos && \
     rm -rf /root/occlum
 
-WORKDIR /root/demos/grpc
+WORKDIR /root/demos/grpc/grpc_musl
 RUN bash -x download_and_install_openssl.sh && \
     bash -x download_and_install_grpc.sh
 
+WORKDIR /root/demos/grpc/grpc_glibc
+RUN bash -x download_and_install_grpc_glibc.sh && \
+    bash -x prepare_stress_test_tool.sh
+
 WORKDIR /root