occlum/.github/workflows/hw_mode_test.yml
Hui, Chunyang 92a8db7938 Enhance SGX hardware mode CI test
1. Add FISH and XGBoost demo test
2. Resolve cargo network error when there is too much concurrency
3. Add git proxy for git download
2020-12-03 10:57:02 +08:00

547 lines
20 KiB
YAML

name: SGX Hardware Mode Test
on: [push]
env:
nap_time: 60
jobs:
Make-test-on-ubuntu:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
occlum_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "occlum_test=$occlum_test" >> $GITHUB_ENV
- name: Change download source of crates.io
run: |
docker exec $occlum_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $occlum_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Build source
run: docker exec $occlum_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"
- name: Integration test
run: docker exec $occlum_test bash -c "cd /root/occlum; make test"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $occlum_test
C_cpp_rust_golang_embedded_mode_support_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
language_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "language_support_test=$language_support_test" >> $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $language_support_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $language_support_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $language_support_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: C test
run: docker exec $language_support_test bash -c "cd /root/occlum/demos/hello_c && make;
occlum new occlum_instance;
cp hello_world occlum_instance/image/bin;
cd occlum_instance && occlum build;
occlum run /bin/hello_world"
- name: C++ test
run: docker exec $language_support_test bash -c "cd /root/occlum/demos/hello_cc && make;
occlum new occlum_instance;
cp hello_world occlum_instance/image/bin;
cd occlum_instance && occlum build;
occlum run /bin/hello_world"
- name: Rust test
run: docker exec $language_support_test bash -c "cd /root/occlum/demos/rust && ./run_rust_demo_on_occlum.sh"
- name: Embedded mode test
run: docker exec $language_support_test bash -c "cd /root/occlum/demos/embedded_mode && make;
make test"
- name: Go server set up and run
run: docker exec $language_support_test bash -c "export GO111MODULE=on && export GOPROXY=https://goproxy.cn;
cd /root/occlum/demos/golang/web_server && occlum-go get -u -v github.com/gin-gonic/gin;
occlum-go build -o web_server ./web_server.go;
./run_golang_on_occlum.sh" &
- name: Curl test
run: |
sleep ${{ env.nap_time }};
docker exec $language_support_test bash -c "curl http://127.0.0.1:8090/ping"
- name: Set up Golang grpc pingpong test
run: docker exec $language_support_test bash -c "export GO111MODULE=on && export GOPROXY=https://goproxy.cn;
cd /root/occlum/demos/golang/grpc_pingpong && ./prepare_ping_pong.sh"
- name: Start Golang grpc pingpong server
run: docker exec $language_support_test bash -c "cd /root/occlum/demos/golang/grpc_pingpong && ./run_pong_on_occlum.sh" &
- name: Run Golang grpc ping test
run: |
sleep ${{ env.nap_time }};
docker exec $language_support_test bash -c "cd /root/occlum/demos/golang/grpc_pingpong && ./run_ping_on_occlum.sh"
- name: Run Golang sqlite test
run: docker exec $language_support_test bash -c "export GO111MODULE=on && export GOPROXY=https://goproxy.cn;
cd /root/occlum/demos/golang/go_sqlite/ && ./run_go_sqlite_demo.sh"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $language_support_test
Java_support_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
java_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "java_support_test=$java_support_test" >> $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $java_support_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $java_support_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $java_support_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Compile Java
run: docker exec $java_support_test bash -c "cd /root/occlum/demos/java && occlum-javac ./hello_world/Main.java"
- name: Run hello world
run: docker exec $java_support_test bash -c "cd /root/occlum/demos/java && ./run_java_on_occlum.sh hello"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $java_support_test
Bazel_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
hello_bazel_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "hello_bazel_test=$hello_bazel_test" >> $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $hello_bazel_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $hello_bazel_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $hello_bazel_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Config git proxy
run: |
docker exec $hello_bazel_test bash -c "git config --global http.proxy socks5://localhost:5432"
docker exec $hello_bazel_test bash -c "git config --global https.proxy socks5://localhost:5432"
- name: Install bazel
run: docker exec $hello_bazel_test bash -c "cd /root/occlum/demos/hello_bazel && wget https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-linux-x86_64.sh;
chmod +x bazel-3.2.0-installer-linux-x86_64.sh;
./bazel-3.2.0-installer-linux-x86_64.sh"
- name: Build bazel dependencies
run: docker exec $hello_bazel_test bash -c "cd /root/occlum/demos/hello_bazel && ./build_bazel_sample.sh"
- name: Test bazel
run: docker exec $hello_bazel_test bash -c "cd /root/occlum/demos/hello_bazel && occlum new occlum_instance;
cp examples/cpp-tutorial/stage3/bazel-bin/main/hello-world occlum_instance/image/bin;
cd occlum_instance && occlum build;
occlum run /bin/hello-world"
- name: Clean the environment
run: docker stop $hello_bazel_test
Fish_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
fish_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "fish_test=$fish_test" >> $GITHUB_ENV
cat $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $fish_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $fish_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $fish_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Config git proxy
run: |
docker exec $fish_test bash -c "git config --global http.proxy socks5://localhost:5432"
docker exec $fish_test bash -c "git config --global https.proxy socks5://localhost:5432"
- name: Build Fish dependencies
run: docker exec $fish_test bash -c "cd /root/occlum/demos/fish && ./download_and_build.sh"
- name: Run Fish test
run: docker exec $fish_test bash -c "cd /root/occlum/demos/fish && ./run_fish_test.sh"
- name: Run Fish process rlimit test
run: docker exec $fish_test bash -c "cd /root/occlum/demos/fish && ./run_per_process_config_test.sh"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $fish_test
Xgboost_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
xgboost_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "xgboost_test=$xgboost_test" >> $GITHUB_ENV
cat $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $xgboost_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $xgboost_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $xgboost_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Config git proxy
run: |
docker exec $xgboost_test bash -c "git config --global http.proxy socks5://localhost:5432"
docker exec $xgboost_test bash -c "git config --global https.proxy socks5://localhost:5432"
- name: Build xgboost dependencies
run: docker exec $xgboost_test bash -c "cd /root/occlum/demos/xgboost && ./download_and_build_xgboost.sh"
- name: Run xgboost test
run: docker exec $xgboost_test bash -c "cd /root/occlum/demos/xgboost && make test"
- name: Run xgboost cluster test
run: docker exec $xgboost_test bash -c "cd /root/occlum/demos/xgboost && make test-local-cluster"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $xgboost_test
Sqlite_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Get Occlum version
run: echo "OCCLUM_VERSION=$(grep 'Version =' src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV;
- name: Create container
run: |
sqlite_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:$OCCLUM_VERSION-ubuntu18.04)
echo "sqlite_test=$sqlite_test" >> $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $sqlite_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $sqlite_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $sqlite_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Config git proxy
run: |
docker exec $sqlite_test bash -c "git config --global http.proxy socks5://localhost:5432"
docker exec $sqlite_test bash -c "git config --global https.proxy socks5://localhost:5432"
- name: Build sqlite dependencies
run: docker exec $sqlite_test bash -c "cd /root/occlum/demos/sqlite && ./download_and_build_sqlite.sh"
- name: Run sqlite test
run: docker exec $sqlite_test bash -c "cd /root/occlum/demos/sqlite && ./run_sqlite_on_occlum.sh"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $sqlite_test
Python_support_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Create container
run: |
docker pull occlumbackup/occlum:latest-ubuntu18.04-python
python_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python)
echo "python_support_test=$python_support_test" >> $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $python_support_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'tuna'
[source.tuna]
registry = \"https://mirrors.tuna.tsinghua.edu.cn/git/crates.io-index.git\"
EOF"
- name: Build dependencies
run: docker exec $python_support_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $python_support_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Run python support test
run: docker exec $python_support_test bash -c "cd /root/occlum/demos/python; ./run_python_on_occlum.sh"
- name: Check result
run: docker exec $python_support_test bash -c "cd /root/occlum/demos/python/occlum_instance; cat smvlight.dat"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $python_support_test
Openvino_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Create container
run: |
docker pull occlumbackup/occlum:latest-ubuntu18.04-openvino
openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-openvino)
echo "openvino_test=$openvino_test" >> $GITHUB_ENV
- name: Configure dependency source
run: |
docker exec $openvino_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $openvino_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $openvino_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
- name: Run openVINO benchmark
run: docker exec $openvino_test bash -c "cd /root/demos/openvino && ./run_benchmark_on_occlum.sh"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $openvino_test
Grpc_test:
runs-on: [self-hosted, SGX1-HW]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
uses: actions/checkout@v1
with:
submodules: true
- name: Create container
run: |
docker pull occlumbackup/occlum:latest-ubuntu18.04-grpc
grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-grpc)
echo "grpc_test=$grpc_test" >> $GITHUB_ENV
- name: Change download source of crates.io
run: |
docker exec $grpc_test bash -c "cat <<- EOF >/root/.cargo/config
[source.crates-io]
registry = \"https://github.com/rust-lang/crates.io-index\"
replace-with = 'ustc'
[source.ustc]
registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
EOF"
- name: Build dependencies
run: docker exec $grpc_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"
- name: Make install
run: docker exec $grpc_test bash -c "cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"
- name: Prepare grpc sample project
run: docker exec $grpc_test bash -c "cd /root/demos/grpc && ./prepare_client_server.sh"
- name: Run grpc server
run: docker exec $grpc_test bash -c "cd /root/demos/grpc && ./run_server_on_occlum.sh" &
- name: Run grpc client
run: |
sleep ${{ env.nap_time }};
docker exec $grpc_test bash -c "cd /root/demos/grpc && ./run_client_on_occlum.sh"
- name: Clean the environment
if: ${{ always() }}
run: docker stop $grpc_test