From 5a03deb6b9b5ad1c0270a001eb59f6fb64f0abec Mon Sep 17 00:00:00 2001
From: Shaowei Song <1498430017@qq.com>
Date: Thu, 23 Mar 2023 15:28:26 +0800
Subject: [PATCH] [demos] Improve steps of dependency preinstall

---
 .github/workflows/demo_test.yml             | 20 +++++++-------
 .github/workflows/hw_mode_test.yml          | 10 +++----
 demos/flink/README.md                       | 26 +++++++++++++-----
 demos/flink/download_flink.sh               | 10 -------
 demos/flink/preinstall_deps.sh              | 14 ++++++++++
 demos/mysql/README.md                       |  8 +++++-
 demos/mysql/dl_and_build_mysql.sh           | 19 --------------
 demos/mysql/preinstall_deps.sh              | 29 +++++++++++++++++++++
 demos/mysql/run_benchmarks.sh               | 11 +++-----
 demos/netty_ut/README                       | 12 ++++++---
 demos/netty_ut/build.sh                     |  6 ++---
 demos/netty_ut/preinstall_deps.sh           | 13 +++++++++
 demos/xgboost/README.md                     | 14 +++++++---
 demos/xgboost/download_and_build_xgboost.sh | 12 ---------
 demos/xgboost/install_cmake.sh              |  2 ++
 demos/xgboost/preinstall_deps.sh            | 19 ++++++++++++++
 16 files changed, 144 insertions(+), 81 deletions(-)
 create mode 100755 demos/flink/preinstall_deps.sh
 create mode 100755 demos/mysql/preinstall_deps.sh
 create mode 100755 demos/netty_ut/preinstall_deps.sh
 create mode 100755 demos/xgboost/preinstall_deps.sh

diff --git a/.github/workflows/demo_test.yml b/.github/workflows/demo_test.yml
index 206f93c7..463204af 100644
--- a/.github/workflows/demo_test.yml
+++ b/.github/workflows/demo_test.yml
@@ -231,8 +231,8 @@ jobs:
         container-name: ${{ github.job }}
         build-envs: 'OCCLUM_RELEASE_BUILD=1'
 
-    - name: Build xgboost dependencies
-      run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/xgboost && ./download_and_build_xgboost.sh"
+    - name: Preinstall dependencies and build xgboost
+      run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/xgboost && ./preinstall_deps.sh && ./download_and_build_xgboost.sh"
 
     - name: Run xgboost test
       run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/xgboost && SGX_MODE=SIM make test"
@@ -525,7 +525,7 @@ jobs:
     - name: Run redis benchmark
       run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/redis; SGX_MODE=SIM ./benchmark_glibc.sh"
 
-  flink_test:
+  Flink_test:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v1
@@ -537,8 +537,8 @@ jobs:
         container-name: ${{ github.job }}
         build-envs: 'OCCLUM_RELEASE_BUILD=1'
 
-    - name: Download flink
-      run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/flink && ./download_flink.sh"
+    - name: Preinstall dependencies and download flink
+      run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/flink && ./preinstall_deps.sh && ./download_flink.sh"
 
     - name: Run jobmanager on host
       run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/flink && SGX_MODE=SIM ./run_flink_jobmanager_on_host.sh"
@@ -573,7 +573,7 @@ jobs:
     - name: Run cluster serving test
       run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/cluster_serving; source ./environment.sh; SGX_MODE=SIM ./start-all.sh; ./push-image.sh"
 
-  enclave_ra_tls_test:
+  Enclave_RA_TLS_test:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v1
@@ -597,7 +597,7 @@ jobs:
         sleep ${{ env.nap_time }};
         docker exec ${{ github.job }} bash -c "/usr/share/enclave-tls/samples/enclave-tls-client" || true
 
-  vault_test:
+  Vault_test:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v1
@@ -620,7 +620,7 @@ jobs:
         sleep ${{ env.nap_time }};
         docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/golang/vault && ./run_occlum_vault_test.sh"
 
-  sofaboot_test:
+  Sofaboot_test:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v1
@@ -659,7 +659,7 @@ jobs:
         build-envs: 'OCCLUM_RELEASE_BUILD=1'
 
     - name: Install OpenJDK 11 (Glibc)
-      run: docker exec ${{ github.job }} bash -c "apt update && apt install -y openjdk-11-jdk"
+      run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/netty_ut && ./preinstall_deps.sh"
 
     - name: Download and compile netty unit test demos
       run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/netty_ut && ./build.sh"
@@ -767,7 +767,7 @@ jobs:
       run: docker exec ${{ github.job }} bash -c "cd /root/occlum/demos/python/flask;
             curl --cacert flask.crt -X GET https://localhost:4996/customer/1"
 
-  iperf2_test:
+  Iperf2_test:
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v1
diff --git a/.github/workflows/hw_mode_test.yml b/.github/workflows/hw_mode_test.yml
index b6136e3d..b63ebf08 100644
--- a/.github/workflows/hw_mode_test.yml
+++ b/.github/workflows/hw_mode_test.yml
@@ -342,8 +342,8 @@ jobs:
         container-name: ${{ github.job }}
         build-envs: 'OCCLUM_RELEASE_BUILD=1'
 
-    - name: Build xgboost dependencies
-      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/xgboost && ./download_and_build_xgboost.sh"
+    - name: Preinstall dependencies and build xgboost
+      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/xgboost && ./preinstall_deps.sh && ./download_and_build_xgboost.sh"
 
     - name: Run xgboost test
       run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/xgboost && make test"
@@ -985,8 +985,8 @@ jobs:
         container-name: ${{ github.job }}
         build-envs: 'OCCLUM_RELEASE_BUILD=1'
 
-    - name: Download and build mysql
-      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/mysql && ./dl_and_build_mysql.sh"
+    - name: Preinstall dependencies and build mysql
+      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/mysql && ./preinstall_deps.sh && ./dl_and_build_mysql.sh"
 
     - name: Run mysql server
       run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/mysql && ./run_mysql_server.sh" &
@@ -1034,7 +1034,7 @@ jobs:
         build-envs: 'OCCLUM_RELEASE_BUILD=1'
 
     - name: Install OpenJDK 11 (Glibc)
-      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "apt update && apt install -y openjdk-11-jdk"
+      run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/netty_ut && ./preinstall_deps.sh"
 
     - name: Download and compile netty unit test demos
       run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/netty_ut && ./build.sh"
diff --git a/demos/flink/README.md b/demos/flink/README.md
index d5e375cf..122d1e26 100644
--- a/demos/flink/README.md
+++ b/demos/flink/README.md
@@ -1,11 +1,25 @@
-1. Run the flink jobmanager
-    ./run_flink_jobmanager_on_host.sh
+# Run Flink on Occlum
 
-2. Run the taskManager
-    ./run_flink_on_occlum_glibc.sh tm
+### Preinstall dependencies
+Related dependencies: openjdk-11
+```
+./preinstall_deps.sh
+```
 
-3. Run flink jobs example
-    ./run_flink_on_occlum_glibc.sh task
+### Run the flink jobmanager
+```
+./run_flink_jobmanager_on_host.sh
+```
+
+### Run the taskManager
+```
+./run_flink_on_occlum_glibc.sh tm
+```
+
+### Run flink jobs example
+```
+./run_flink_on_occlum_glibc.sh task
+```
 
 **Note:**  
 1. If running the jobmanager in docker, please export the port 8081 and 6123
diff --git a/demos/flink/download_flink.sh b/demos/flink/download_flink.sh
index 1b7f50ab..15c27006 100755
--- a/demos/flink/download_flink.sh
+++ b/demos/flink/download_flink.sh
@@ -1,16 +1,6 @@
 #!/bin/bash
 set -e
 
-OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
-if [ "$OS" == "\"Ubuntu\"" ]; then
-  apt-get update -y && apt-get install -y openjdk-11-jre
-  # The openjdk has a broken symlink, remove it as a workaround
-  rm -f /usr/lib/jvm/java-11-openjdk-amd64/lib/security/blacklisted.certs
-else
-  echo "Unsupported OS: $OS"
-  exit 1
-fi
-
 rm -rf flink-1.10.1*
 wget https://archive.apache.org/dist/flink/flink-1.10.1/flink-1.10.1-bin-scala_2.11.tgz
 tar -xvzf flink-1.10.1-bin-scala_2.11.tgz
diff --git a/demos/flink/preinstall_deps.sh b/demos/flink/preinstall_deps.sh
new file mode 100755
index 00000000..7f0ec281
--- /dev/null
+++ b/demos/flink/preinstall_deps.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+set -e
+
+OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
+if [ "$OS" == "\"Ubuntu\"" ]; then
+  apt-get update -y && apt-get install -y openjdk-11-jre
+  # The openjdk has a broken symlink, remove it as a workaround
+  rm -f /usr/lib/jvm/java-11-openjdk-amd64/lib/security/blacklisted.certs
+else
+  echo "Unsupported OS: $OS"
+  exit 1
+fi
+
+echo "Install dependencies success"
diff --git a/demos/mysql/README.md b/demos/mysql/README.md
index 9e1599fd..03270887 100644
--- a/demos/mysql/README.md
+++ b/demos/mysql/README.md
@@ -2,7 +2,13 @@
 
 [`MySQL`](https://www.mysql.com/) is a widely used open-source relational database management system (RDBMS).
 
-### Build and install
+### Preinstall dependencies
+Related dependencies: libnuma-dev libboost-all-dev boost-1.77.0
+```
+./preinstall_deps.sh
+```
+
+### Download and build MySQL
 ```
 ./dl_and_build_mysql.sh
 ```
diff --git a/demos/mysql/dl_and_build_mysql.sh b/demos/mysql/dl_and_build_mysql.sh
index e58f8c27..62ff71b3 100755
--- a/demos/mysql/dl_and_build_mysql.sh
+++ b/demos/mysql/dl_and_build_mysql.sh
@@ -3,25 +3,6 @@ set -e
 
 BLUE='\033[1;34m'
 NC='\033[0m'
-echo -e "${BLUE}Start installing dependencies.${NC}"
-
-# Prepare environment
-DEPS="libnuma-dev libboost-all-dev"
-
-apt-get update
-apt-get install -y ${DEPS}
-
-BOOST="boost_1_77_0"
-wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/${BOOST}.tar.bz2
-tar --bzip2 -xf ${BOOST}.tar.bz2
-pushd ${BOOST}
-./bootstrap.sh --prefix=/usr --with-python=python3 &&
-./b2 stage -j4 threading=multi link=shared
-./b2 install threading=multi link=shared
-popd
-
-echo -e "${BLUE}Finish installing dependencies.${NC}"
-
 echo -e "${BLUE}Start building mysql from src.${NC}"
 
 # Download released tarball
diff --git a/demos/mysql/preinstall_deps.sh b/demos/mysql/preinstall_deps.sh
new file mode 100755
index 00000000..000d838c
--- /dev/null
+++ b/demos/mysql/preinstall_deps.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+
+BLUE='\033[1;34m'
+NC='\033[0m'
+echo -e "${BLUE}Start installing dependencies.${NC}"
+
+DEPS="libnuma-dev libboost-all-dev"
+
+OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
+if [ "$OS" == "\"Ubuntu\"" ]; then
+  apt-get update -y && apt-get install -y ${DEPS}
+  # Install sysbench for benchmarking purpose
+  apt-get install -y sysbench
+else
+  echo "Unsupported OS: $OS"
+  exit 1
+fi
+
+BOOST="boost_1_77_0"
+wget https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/${BOOST}.tar.bz2
+tar --bzip2 -xf ${BOOST}.tar.bz2
+pushd ${BOOST}
+./bootstrap.sh --prefix=/usr --with-python=python3 &&
+./b2 stage -j4 threading=multi link=shared
+./b2 install threading=multi link=shared
+popd
+
+echo -e "${BLUE}Finish installing dependencies.${NC}"
diff --git a/demos/mysql/run_benchmarks.sh b/demos/mysql/run_benchmarks.sh
index a48645ce..30492968 100755
--- a/demos/mysql/run_benchmarks.sh
+++ b/demos/mysql/run_benchmarks.sh
@@ -4,17 +4,12 @@ set -e
 GREEN='\033[1;32m'
 NC='\033[0m'
 
-echo -e "${GREEN}Install sysbench first${NC}"
-
-# We use sysbench to test mysql, need to install it first
-apt-get install -y sysbench
-
 function run_benchmarks()
 {
     WORKLOADS=("oltp_point_select" "oltp_write_only" "oltp_read_write")
     for item in ${WORKLOADS[@]}
     do
-        echo "start to prepare for $item"
+        echo "${GREEN}start to prepare for $item${NC}"
         sleep 3
         sysbench /usr/share/sysbench/$item.lua\
             --mysql-host='127.0.0.1'\
@@ -26,7 +21,7 @@ function run_benchmarks()
             --rand-type=pareto\
             prepare
 
-        echo "start to run $item"
+        echo "${GREEN}start to run $item${NC}"
         sleep 3
         sysbench /usr/share/sysbench/$item.lua\
             --mysql-host='127.0.0.1'\
@@ -40,7 +35,7 @@ function run_benchmarks()
             --report-interval=10\
             run
 
-        echo "start to cleanup $item"
+        echo "${GREEN}start to cleanup $item${NC}"
         sleep 3
         sysbench /usr/share/sysbench/$item.lua\
             --mysql-host='127.0.0.1'\
diff --git a/demos/netty_ut/README b/demos/netty_ut/README
index 72e1d8e1..a04754f6 100644
--- a/demos/netty_ut/README
+++ b/demos/netty_ut/README
@@ -1,13 +1,19 @@
 # A Simple netty unit test usage demo
 
-This project demonstrates how to verify the completeness of Netty on Occlum.
+This demo demonstrates how to verify the completeness of Netty on Occlum.
 
-1. Download junit and related netty jars
+### 1. Preinstall dependencies
+Related dependencies: OpenJDK 11 (Glibc)
+```
+./preinstall_deps.sh
+```
+
+### 2. Download junit and related netty jars
 ```
 ./build.sh
 ```
 
-2. Run `netty unit test ` on Occlum
+### 3. Run `netty unit test ` on Occlum
 ```
 ./run_netty_ut_jdk11.sh
 ```
diff --git a/demos/netty_ut/build.sh b/demos/netty_ut/build.sh
index b6cc24e6..0b01f153 100755
--- a/demos/netty_ut/build.sh
+++ b/demos/netty_ut/build.sh
@@ -1,10 +1,10 @@
 #!/bin/bash
-sudo apt-get update
-sudo apt-get install openjdk-11-jdk
-rm -rf /usr/lib/jvm/java-11-openjdk-amd64/lib/security/blacklisted.certs
+set -e
 
 # Download netty testsuite, junit platform and related dependencies
 
 if [ ! -d "netty" ]; then
     wget -i ./ut-jar.url -P ./netty
 fi
+
+echo "Build Netty unit test success"
diff --git a/demos/netty_ut/preinstall_deps.sh b/demos/netty_ut/preinstall_deps.sh
new file mode 100755
index 00000000..f949f8c9
--- /dev/null
+++ b/demos/netty_ut/preinstall_deps.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e
+
+OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
+if [ "$OS" == "\"Ubuntu\"" ]; then
+  apt-get update -y && apt-get install -y openjdk-11-jdk
+  rm -rf /usr/lib/jvm/java-11-openjdk-amd64/lib/security/blacklisted.certs
+else
+  echo "Unsupported OS: $OS"
+  exit 1
+fi
+
+echo "Install dependencies success"
diff --git a/demos/xgboost/README.md b/demos/xgboost/README.md
index 8447a89d..7f40570e 100644
--- a/demos/xgboost/README.md
+++ b/demos/xgboost/README.md
@@ -1,22 +1,28 @@
 # Use XGBoost in SGX with Occlum
 
-Step 1: Download XGBoost and its dependencies, and then build XGBoost
+### Step 1: Preinstall dependencies
+Related dependencies: python3-pip python3-setuptools kubernetes cmake
+```
+./preinstall_deps.sh
+```
+
+### Step 2: Download and build XGBoost
 ```
 ./download_and_build_xgboost.sh
 ```
 When completed, the resulting XGBoost can be found in `xgboost_src` directory.
 
-Step 2: To train data with XGBoost in a single process, run
+### Step 3: To train data with XGBoost in a single process, run
 ```
 make test
 ```
 
-Step 3: To train data with a two-node XGBoost cluster, run
+### Step 4: To train data with a two-node XGBoost cluster, run
 ```
 make test-local-cluster
 ```
 
-Step 4 (Optional): To train data with XGBoost in a single process in Linux, run
+### Step 5 (Optional): To train data with XGBoost in a single process in Linux, run
 ```
 make test-native
 ```
diff --git a/demos/xgboost/download_and_build_xgboost.sh b/demos/xgboost/download_and_build_xgboost.sh
index ab6d8786..b52484ef 100755
--- a/demos/xgboost/download_and_build_xgboost.sh
+++ b/demos/xgboost/download_and_build_xgboost.sh
@@ -4,18 +4,6 @@ set -e
 # Tell CMake to search for packages in Occlum toolchain's directory only
 export PKG_CONFIG_LIBDIR=/usr/local/occlum/x86_64-linux-musl/lib
 
-# Install the dependencies
-OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
-if [ "$OS" == "\"Ubuntu\"" ]; then
-  apt-get update -y && apt-get install -y python3-pip python3-setuptools
-else
-  yum install -y python3-pip python3-setuptools
-fi
-pip3 install kubernetes
-
-#install the cmake
-./install_cmake.sh
-
 # Download and build XGBoost
 rm -rf xgboost_src && mkdir xgboost_src
 pushd xgboost_src
diff --git a/demos/xgboost/install_cmake.sh b/demos/xgboost/install_cmake.sh
index c6c223b8..b5a88c03 100755
--- a/demos/xgboost/install_cmake.sh
+++ b/demos/xgboost/install_cmake.sh
@@ -12,3 +12,5 @@ cd cmake-3.15.5
 ./bootstrap
 make -j$(nproc)
 sudo make install
+
+echo "Install CMake success"
diff --git a/demos/xgboost/preinstall_deps.sh b/demos/xgboost/preinstall_deps.sh
new file mode 100755
index 00000000..ab2b4dac
--- /dev/null
+++ b/demos/xgboost/preinstall_deps.sh
@@ -0,0 +1,19 @@
+#!/bin/bash
+set -e
+
+# Tell CMake to search for packages in Occlum toolchain's directory only
+export PKG_CONFIG_LIBDIR=/usr/local/occlum/x86_64-linux-musl/lib
+
+# Install dependencies
+OS=`awk -F= '/^NAME/{print $2}' /etc/os-release`
+if [ "$OS" == "\"Ubuntu\"" ]; then
+  apt-get update -y && apt-get install -y python3-pip python3-setuptools
+else
+  yum install -y python3-pip python3-setuptools
+fi
+pip3 install kubernetes
+
+echo "Install dependencies success"
+
+# Install CMake
+./install_cmake.sh