name: SGX Hardware Mode Test

# Triggers the workflow on push and pull request labeled "SGX-hardware-test-required".
on:
  push:
  pull_request_target:
    types: labeled
  schedule:
  # Schedule to run everyday at 6PM UTC (2AM CST)
    - cron:  '0 18 * * *'

env:
  nap_time: 60
  repeat_times: 500  # Stress test repeat times

# Cancel previous running jobs on push or pull request
concurrency:
  group: ${{ github.workflow }}-${{ github.ref || github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  Make-test-on-ubuntu:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      # This step is only needed when the pull request is labeled.
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        # For pull request, we need to merge the commit from fork to the base
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Integration test
      run:  docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum; OCCLUM_LOG_LEVEL=trace make test"

    - name: Integration test with Glibc
      run:  docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum; OCCLUM_LOG_LEVEL=trace make test-glibc"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  C_cpp_rust_golang_embedded_mode_support_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: C test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/hello_c && make;
            occlum new occlum_instance;
            cp hello_world occlum_instance/image/bin;
            cd occlum_instance && occlum build;
            occlum run /bin/hello_world"

    - name: C with encrypted image test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/hello_c && make;
            rm -rf occlum_instance && occlum new occlum_instance;
            occlum gen-image-key occlum_instance/image_key;
            cp hello_world occlum_instance/image/bin;
            cd occlum_instance && occlum build --image-key ./image_key --buildin-image-key;
            occlum run /bin/hello_world"

    - name: C++ test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/hello_cc && make;
            occlum new occlum_instance;
            cp hello_world occlum_instance/image/bin;
            cd occlum_instance && occlum build;
            occlum run /bin/hello_world"

    - name: Rust test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/rust && ./run_rust_demo_on_occlum.sh"

    - name: Embedded mode test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/embedded_mode && make;
            make test"

    - name: Run Golang sqlite test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "export GO111MODULE=on;
            cd /root/occlum/demos/golang/go_sqlite/ && ./run_go_sqlite_demo.sh"

    - name: Go server set up and run
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "export GO111MODULE=on;
            cd /root/occlum/demos/golang/web_server && occlum-go mod init web_server && occlum-go get -u -v github.com/gin-gonic/gin;
            occlum-go build -o web_server ./web_server.go;
            ./run_golang_on_occlum.sh" &

    - name: Set up Golang grpc pingpong test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "export GO111MODULE=on;
            cd /root/occlum/demos/golang/grpc_pingpong && ./prepare_ping_pong.sh"

    - name: Start Golang grpc pingpong server
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/golang/grpc_pingpong && ./run_pong_on_occlum.sh" &

    - name: Run Golang grpc ping test
      run: |
        sleep ${{ env.nap_time }};
        docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/golang/grpc_pingpong && ./run_ping_on_occlum.sh"

    # Sleeps longer to make sure the server is up.
    - name: Curl test
      run: |
        sleep ${{ env.nap_time }};
        docker exec ${{ env.CONTAINER_NAME }} bash -c "curl http://127.0.0.1:8090/ping"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  Java_support_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Compile Java
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/java && occlum-javac ./hello_world/Main.java"

    - name: Run hello world
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/java && ./run_java_on_occlum.sh hello"

    - name: Compile processBuilder demo
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/java && occlum-javac ./processBuilder/processBuilder.java"

    - name: Run processBuilder
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/java && ./run_java_on_occlum.sh processBuilder"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  Bazel_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Install bazel
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/hello_bazel && wget https://github.com/bazelbuild/bazel/releases/download/3.2.0/bazel-3.2.0-installer-linux-x86_64.sh;
              chmod +x bazel-3.2.0-installer-linux-x86_64.sh;
              ./bazel-3.2.0-installer-linux-x86_64.sh"

    - name: Build bazel dependencies
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/hello_bazel && ./build_bazel_sample.sh"

    - name: Test bazel
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/hello_bazel && occlum new occlum_instance;
            cp examples/cpp-tutorial/stage3/bazel-bin/main/hello-world occlum_instance/image/bin;
            cd occlum_instance && occlum build;
            occlum run /bin/hello-world"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  Fish_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Build Fish dependencies
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/fish && ./download_and_build.sh"

    - name: Run Fish test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/fish && ./run_fish_test.sh"

    - name: Run Fish process rlimit test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/fish && ./run_per_process_config_test.sh"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  Xgboost_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Build xgboost dependencies
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/xgboost && ./download_and_build_xgboost.sh"

    - name: Run xgboost test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/xgboost && make test"

    - name: Run xgboost cluster test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/xgboost && make test-local-cluster"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  Sqlite_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Build sqlite dependencies
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/sqlite && ./download_and_build_sqlite.sh"

    - name: Run sqlite test
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/sqlite && ./run_sqlite_on_occlum.sh"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}


  Python_musl_support_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - name: Create container
      run: |
        docker pull occlumbackup/occlum:latest-ubuntu20.04-python
        if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
          python_musl_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-python);
        elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
          python_musl_support_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-python);
        else
          echo "Unsupported Hardware"
        fi;
        echo "python_musl_support_test=$python_musl_support_test" >> $GITHUB_ENV

    - uses: ./.github/workflows/composite_action/prebuild
      with:
        container-name: $python_musl_support_test

    # - name: Configure dependency source
    #   run: |
    #     docker exec $python_musl_support_test bash -c "cat <<- EOF >/root/.cargo/config
    #     [source.crates-io]
    #     registry = \"https://github.com/rust-lang/crates.io-index\"
    #     replace-with = 'tuna'
    #     [source.tuna]
    #     registry = \"https://mirrors.tuna.tsinghua.edu.cn/git/crates.io-index.git\"
    #     EOF"

    - name: Build dependencies
      run: docker exec $python_musl_support_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"

    - name: Make install
      run: docker exec $python_musl_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"

    - name: Run python musl support test
      run: docker exec $python_musl_support_test bash -c "cd /root/occlum/demos/python/python_musl; ./run_python_on_occlum.sh"

    - name: Check result
      run: docker exec $python_musl_support_test bash -c "cd /root/occlum/demos/python/python_musl/occlum_instance; cat smvlight.dat"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop $python_musl_support_test


  Openvino_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - name: Create container
      run: |
        docker pull occlumbackup/occlum:latest-ubuntu20.04-openvino
        if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
          openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-openvino);
        elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
          openvino_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-openvino);
        else
          echo "Unsupported Hardware"
        fi;
        echo "openvino_test=$openvino_test" >> $GITHUB_ENV

    - uses: ./.github/workflows/composite_action/prebuild
      with:
        container-name: $openvino_test

    # - name: Configure dependency source
    #   run: |
    #     docker exec $openvino_test bash -c "cat <<- EOF >/root/.cargo/config
    #     [source.crates-io]
    #     registry = \"https://github.com/rust-lang/crates.io-index\"
    #     replace-with = 'ustc'
    #     [source.ustc]
    #     registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
    #     EOF"

    - name: Build dependencies
      run: docker exec $openvino_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"

    - name: Make install
      run: docker exec $openvino_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"

    - name: Run openVINO benchmark
      run: docker exec $openvino_test bash -c "cd /root/demos/openvino && ./run_benchmark_on_occlum.sh"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop $openvino_test


  Grpc_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - name: Create container
      run: |
        docker pull occlumbackup/occlum:latest-ubuntu20.04-grpc
        if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
          grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-grpc);
        elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
          grpc_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-grpc);
        else
          echo "Unsupported Hardware"
        fi;
        echo "grpc_test=$grpc_test" >> $GITHUB_ENV

    - uses: ./.github/workflows/composite_action/prebuild
      with:
        container-name: $grpc_test

    # - name: Change download source of crates.io
    #   run: |
    #     docker exec $grpc_test bash -c "cat <<- EOF >/root/.cargo/config
    #     [source.crates-io]
    #     registry = \"https://github.com/rust-lang/crates.io-index\"
    #     replace-with = 'ustc'
    #     [source.ustc]
    #     registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
    #     EOF"

    - name: Build dependencies
      run: docker exec $grpc_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"

    - name: Make install
      run: docker exec $grpc_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"

    - name: Prepare grpc musl sample project
      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_musl && ./prepare_client_server.sh"

    - name: Run grpc musl server
      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_musl && ./run_server_on_occlum.sh" &

    - name: Run grpc musl client
      run: |
        sleep ${{ env.nap_time }};
        docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_musl && ./run_client_on_occlum.sh"

    - name: Prepare grpc glibc sample project
      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./prepare_client_server_glibc.sh"

    - name: Run grpc glibc server
      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./run_server_on_occlum_glibc.sh" &

    - name: Run grpc glibc client
      run: |
        sleep ${{ env.nap_time }};
        docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./run_client_on_occlum_glibc.sh"

    - name: Run grpc stress test
      run: docker exec $grpc_test bash -c "cd /root/demos/grpc/grpc_glibc && ./run_stress_test.sh"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop $grpc_test


  Gvisor_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - name: Create container
      run: |
        docker pull occlumbackup/occlum:latest-ubuntu20.04-gvisor_test
        if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
          gvisor_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-gvisor_test);
        elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
          gvisor_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-gvisor_test);
        else
          echo "Unsupported Hardware"
        fi;
        echo "gvisor_test=$gvisor_test" >> $GITHUB_ENV

    - uses: ./.github/workflows/composite_action/prebuild
      with:
        container-name: $gvisor_test

    # - name: Change download source of crates.io
    #   run: |
    #     docker exec $gvisor_test bash -c "cat <<- EOF >/root/.cargo/config
    #     [source.crates-io]
    #     registry = \"https://github.com/rust-lang/crates.io-index\"
    #     replace-with = 'ustc'
    #     [source.ustc]
    #     registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
    #     EOF"

    - name: Build dependencies
      run: docker exec $gvisor_test bash -c "cargo uninstall sccache || true; cd /root/occlum; make submodule"

    - name: Make install
      run: docker exec $gvisor_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=y make install"

    - name: Clone gvisor code
      run: docker exec $gvisor_test bash -c "git clone https://github.com/occlum/gvisor.git"

    - name: Run gvisor syscall test
      run: docker exec $gvisor_test bash -c "cd /root/gvisor/occlum; ./run_occlum_passed_tests.sh"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop $gvisor_test


  # Test_rpm_deploy:
  #   if: github.event_name == 'push'
  #   runs-on: ${{ matrix.self_runner }}
  #   strategy:
  #     matrix:
  #       self_runner: [[self-hosted, SGX1-HW], [self-hosted, SGX2-HW]]

  #   steps:
  #   - name: Clean before running
  #     run: |
  #       sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

  #   - name: Checkout code
  #     if: github.event_name == 'push'
  #     uses: actions/checkout@v2
  #     with:
  #       submodules: false

  #   - name: Test deployment
  #     run: |
  #       cd demos/deployment
  #       if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
  #         ./deploy_image_test.sh centos8.2
  #       elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
  #         DEVICE_OPTION="-v /dev/sgx:/dev/sgx" ./deploy_image_test.sh centos8.2
  #       else
  #         echo "Unsupported Hardware"
  #         exit 1
  #       fi;

  #   - name: Clean the environment
  #     if: ${{ always() }}
  #     run: |
  #       docker stop centos8.2_deploy_test
  #       docker rm -f centos8.2_deploy_test


  Test_deb_deploy:
    timeout-minutes: 180
    if: github.event_name == 'push'
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: false

    - name: Test deployment
      run: |
        cd demos/deployment
        if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
          ./deploy_image_test.sh ubuntu20.04
        elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
          DEVICE_OPTION="-v /dev/sgx:/dev/sgx" ./deploy_image_test.sh ubuntu20.04
        else
          echo "Unsupported Hardware"
          exit 1
        fi;

    - name: Clean the environment
      if: ${{ always() }}
      run: |
        docker stop ubuntu20.04_deploy_test
        docker rm -f ubuntu20.04_deploy_test


# Tensorflow_serving requires binary tensorflow_serving PIC, here we compile tensorflow_model_server before workflow
  Tensorflow_serving_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        # Tensorflow serving test requires AVX512 instruction support. Only the SGX2-HW machine has support for that.
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - name: Create container
      run: |
        docker pull occlumbackup/occlum:latest-ubuntu20.04-tf_serving
        if [[ "${{ matrix.self_runner[1] }}" == "SGX1-HW" ]]; then
          tf_serving_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host --device /dev/isgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-tf_serving);
        elif [[ "${{ matrix.self_runner[1] }}" == "SGX2-HW" ]]; then
          tf_serving_test=$(docker run -itd --privileged --rm --env CARGO_HTTP_MULTIPLEXING=false --net host -v /dev/sgx:/dev/sgx -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu20.04-tf_serving);
        else
          echo "Unsupported Hardware"
        fi;
        echo "tf_serving_test=$tf_serving_test" >> $GITHUB_ENV

    - uses: ./.github/workflows/composite_action/prebuild
      with:
        container-name: $tf_serving_test

    # - name: Change download source of crates.io
    #   run: |
    #     docker exec $tf_serving_test bash -c "cat <<- EOF >/root/.cargo/config
    #     [source.crates-io]
    #     registry = \"https://github.com/rust-lang/crates.io-index\"
    #     replace-with = 'ustc'
    #     [source.ustc]
    #     registry = \"git://mirrors.ustc.edu.cn/crates.io-index\"
    #     EOF"

    - name: Build dependencies
      run: docker exec $tf_serving_test bash -c "cd /root/occlum; make submodule"

    - name: Make install
      run: docker exec $tf_serving_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"

    - name: Move dependencies to current demo
      run: docker exec $tf_serving_test bash -c "mv /root/tensorflow_model_server /root/occlum/demos/tensorflow/tensorflow_serving; mv /root/resnet50-v15-fp32 /root/occlum/demos/tensorflow/tensorflow_serving"

    - name: Generate SSL
      run: docker exec $tf_serving_test bash -c "cd /root/occlum/demos/tensorflow/tensorflow_serving; ./generate_ssl_config.sh localhost"

    - name: Run tf_serving server
      run: docker exec $tf_serving_test bash -c "cd /root/occlum/demos/tensorflow/tensorflow_serving; ./run_occlum_tf_serving.sh"

    - name: Run tf_serving client
      run: |
        sleep 120;
        docker exec $tf_serving_test bash -c "cd /root/occlum/demos/tensorflow/tensorflow_serving/client; ./benchmark.sh python3 localhost:8500 ../ssl_configure/server.crt"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop $tf_serving_test


  Remote_attestation_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Build and run remote attestation demo
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/dcap; ./run_dcap_quote_on_occlum.sh"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}

  Init_RA_test:
    timeout-minutes: 180
    if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      if: github.event_name == 'push'
      uses: actions/checkout@v2
      with:
        submodules: true

    - name: Checkout code from fork
      if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
      uses: actions/checkout@v2
      with:
        ref: refs/pull/${{ github.event.pull_request.number }}/merge
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Install dependencies
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "apt update && apt install -y netcat"

    - name: Build the init-ra all content
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum/demos/remote_attestation/init_ra_flow; ./build_content.sh"

    - name: Run init-ra server and client(flask-tls) on backgroud tiil ready
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow; ./run_till_ready.sh"

    - name: Test PUT data with certificate
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow;
            curl --cacert flask.crt -X PUT https://localhost:4996/customer/1 -d "data=Tom""

    - name: Test GET data with certificate
      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow;
            curl --cacert flask.crt -X GET https://localhost:4996/customer/1"

    - name: Clean the environment
      if: ${{ always() }}
      run: docker stop ${{ env.CONTAINER_NAME }}

  Stress_test_with_musl:
    timeout-minutes: 360
    if: github.event_name == 'schedule'
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      uses: actions/checkout@v2
      with:
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Stress test with musl
      run:  docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum; make test times=${{ env.repeat_times }}"

    - name: Show failed cases and clean the environment
      if: ${{ always() }}
      run: |
        docker exec ${{ env.CONTAINER_NAME }} bash -c "cat /root/occlum/build/test/.fail"
        docker stop ${{ env.CONTAINER_NAME }}


  Stress_test_with_glibc:
    timeout-minutes: 360
    if: github.event_name == 'schedule'
    runs-on: ${{ matrix.self_runner }}
    strategy:
      matrix:
        self_runner: [[self-hosted, SGX2-HW]]

    steps:
    - name: Clean before running
      run: |
        sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"

    - name: Checkout code
      uses: actions/checkout@v2
      with:
        submodules: true

    - uses: ./.github/workflows/composite_action/hw
      with:
        container-name: ${{ github.job }}
        build-envs: 'OCCLUM_RELEASE_BUILD=1'

    - name: Stress test with Glibc
      run:  docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum; make test-glibc times=${{ env.repeat_times }}"

    - name: Show failed cases and clean the environment
      if: ${{ always() }}
      run: |
        docker exec ${{ env.CONTAINER_NAME }} bash -c "cat /root/occlum/build/test/.fail"
        docker stop ${{ env.CONTAINER_NAME }}