Add Bash demo

.github/workflows/demo_test.yml

@@ -666,3 +666,28 @@ jobs:
       run: |
         sleep ${{ env.nap_time }};
         docker exec sofaboot_test bash -c "curl http://localhost:8080/actuator/versions"
+
+  Bash_test:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
+
+    - name: Get occlum version
+      run: echo "OCCLUM_VERSION=$(grep "Version =" src/pal/include/occlum_version.h |  awk '{print $4}')" >> $GITHUB_ENV
+
+    - name: Create container
+      run: docker run -itd --name=bash_test -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04
+
+    - name: Build dependencies
+      run: docker exec bash_test bash -c "cd /root/occlum; make submodule"
+
+    - name: Make install
+      run: docker exec bash_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
+
+    - name: Build Bash dependencies
+      run: docker exec bash_test bash -c "cd /root/occlum/demos/bash && ./prepare_bash_demo.sh"
+
+    - name: Run Bash test
+      run: docker exec bash_test bash -c "cd /root/occlum/demos/bash && SGX_MODE=SIM ./run_bash_demo.sh"

demos/README.md

@@ -14,6 +14,7 @@ This set of demos shows how the Occlum toolchain can be used with different buil
 
 This set of demos shows how real-world apps can be easily run inside SGX enclaves with Occlum.
 
+* [bash](bash/): A demo of [Bash](https://www.gnu.org/software/bash/) shell script.
 * [cluster_serving](cluster_serving/): A demo of [Analytics Zoo Cluster Serving](https://analytics-zoo.github.io/master/#ClusterServingGuide/ProgrammingGuide/) inference solution.
 * [fish](fish/): A demo of [FISH](https://fishshell.com) shell script.
 * [flink](flink/): A demo of [Apache Flink](https://flink.apache.org).

demos/bash/.gitignore

@@ -0,0 +1,3 @@
+bash_for_occlum/
+occlum_instance/
+busybox/

demos/bash/Occlum.json

@@ -0,0 +1,65 @@
+{
+    "resource_limits": {
+        "user_space_size": "600MB",
+        "kernel_space_heap_size": "32MB",
+        "kernel_space_stack_size": "1MB",
+        "max_num_of_threads": 16
+    },
+    "process": {
+        "default_stack_size": "2MB",
+        "default_heap_size": "16MB",
+        "default_mmap_size": "80MB"
+    },
+    "entry_points": [
+        "/root/bin"
+    ],
+    "env": {
+        "default": [
+            "OCCLUM=yes",
+            "HOME=/root"
+        ],
+        "untrusted": [
+        ]
+    },
+    "metadata": {
+        "product_id": 0,
+        "version_number": 0,
+        "debuggable": true
+    },
+    "mount": [
+        {
+            "target": "/",
+            "type": "unionfs",
+            "options": {
+                "layers": [
+                    {
+                        "target": "/",
+                        "type": "sefs",
+                        "source": "./build/mount/__ROOT",
+                        "options": {
+                            "MAC": ""
+                        }
+                    },
+                    {
+                        "target": "/",
+                        "type": "sefs",
+                        "source": "./run/mount/__ROOT"
+                    }
+                ]
+            }
+        },
+        {
+            "target": "/host",
+            "type": "hostfs",
+            "source": "."
+        },
+        {
+            "target": "/proc",
+            "type": "procfs"
+        },
+        {
+            "target": "/dev",
+            "type": "devfs"
+        }
+    ]
+}

demos/bash/README.md

@@ -0,0 +1,19 @@
+# Run Bash script on Occlum
+
+In this demo, we will show how to run a Bash script inside Occlum.
+
+Bash is the most widely used shell implementation around the world. Previously, we didn't support Bash because of too many technical challenges, such as compilation, lack of fork and execve system calls, etc.
+
+Now, Bash is finally supported with modification to the source code of Bash. We have evaluated and all commands defined in `occlum_bash_test.sh` are all supported.
+
+Please follow below steps to run this demo:
+
+1. Download and build Busybox and Occlum-version Bash
+```
+./prepare_bash_demo.sh
+```
+
+2. Run Bash script in Occlum
+```
+./run_bash_demo.sh
+```

demos/bash/occlum_bash_test.sh

@@ -0,0 +1,80 @@
+#! /bin/bash
+
+# Exit when error
+set -xe
+
+# Test pipe
+busybox echo -e "Hello-world-from-bash\n test" | busybox awk '$1=$1' FS="-" OFS=" " > /root/output.txt
+busybox cat /root/output.txt
+busybox rm /root/output.txt
+busybox ls -l /root/output.txt || true
+
+# Test command substitution
+DATE=$(busybox date)
+busybox echo $DATE
+TEST=$(busybox echo $(busybox date))
+busybox echo $TEST
+
+# Test command subsitution and pipe
+busybox echo $(busybox echo -e "Hello-world-from-bash\n test" | busybox awk '$1=$1' FS="-" OFS=" ")
+
+# Test multiple redirection
+busybox ls . *.blah > log 2>&1 || true
+busybox echo "start log:"
+busybox cat log
+busybox rm log
+
+# Test subshell
+SCRIPT_ENV="this is script env"
+(
+    busybox echo "in subshell:"
+    busybox echo $SCRIPT_ENV
+    SUBSHELL_ENV="this is subshell env"
+    SCRIPT_ENV="this is script env in subshell"
+    busybox echo $SUBSHELL_ENV | busybox awk '{print $3}'
+    busybox echo $SCRIPT_ENV
+)
+busybox echo "out subshell:"
+busybox echo $SCRIPT_ENV
+if [ "$SCRIPT_ENV" != "this is script env" ]; then
+    busybox echo "env wrong"
+    exit 1
+fi
+
+busybox echo $SUBSHELL_ENV
+if [ ! -z "$SUBSHELL_ENV" ]; then
+    busybox echo "env wrong"
+    exit 1
+fi
+
+#TEST exec in subshell
+(
+    exec busybox date
+    # This shouldn't be reached
+    exit 1
+)
+
+# Test unrecognized commands
+fake_inst || true
+
+# Test builtin command
+cd /opt/occlum/glibc/lib
+pwd
+cd -
+
+# Test ulimit defined below will overide configuration in Occlum.json
+ulimit -Ss 10240 # stack size 10M
+ulimit -Sd 40960 # heap size 40M
+ulimit -Sv 122880 # virtual memory size 120M (including heap, stack, mmap size)
+
+echo "ulimit result:"
+ulimit -a
+
+# Test background process
+busybox sleep 1000 &
+sleep_pid=$!
+kill $sleep_pid
+
+# TODO: Support process substitution
+
+busybox echo "Test is done"

demos/bash/prepare_bash_demo.sh

@@ -0,0 +1,35 @@
+#! /bin/bash
+set -e
+
+current=$(pwd)
+bash_src="$current/bash_for_occlum"
+bash_git="https://github.com/occlum/bash.git"
+bash_branch="bash_5.1_for_occlum"
+busybox_src="$current/busybox"
+
+if [ ! -d "$bash_src" ]; then
+    # Download and configure Bash
+    cd $current
+    git clone -b $bash_branch $bash_git bash_for_occlum
+    cd bash_for_occlum && git checkout $bash_branch
+    if [ "$DEBUG" == "1" ]; then
+        CFLAGS="-D DEBUG=1 -g -O0" ./configure --enable-debugger
+    else
+        ./configure
+    fi
+
+    # Build
+    make -j$(nproc)
+    echo "Bash is ready."
+fi
+
+if [ ! -d "$busybox_src" ]; then
+    cd $current
+    busybox_config="$current/../fish/.config"
+    git clone -b 1_31_1 --depth 1 https://github.com/mirror/busybox.git
+    cd busybox
+    # CROSS_COMPILE=/opt/occlum/toolchains/gcc/bin/occlum-
+    make defconfig
+    cp $busybox_config .
+    make -j$(nproc)
+fi

demos/bash/run_bash_demo.sh

@@ -0,0 +1,29 @@
+#! /bin/bash
+set -e
+
+current=$(pwd)
+bash_src="$current/bash_for_occlum"
+busybox_src="$current/busybox"
+occlum_instance="$current/occlum_instance"
+occlum_glibc_path=$occlum_instance/image/opt/occlum/glibc/lib
+# Executable path in Occlum
+exec_path=/root/bin
+
+rm -rf occlum_instance
+occlum new occlum_instance
+
+cd occlum_instance
+
+cp $bash_src/bash ./image/bin/
+cp /lib/x86_64-linux-gnu/libtinfo.so.5 $occlum_glibc_path
+cp /lib/x86_64-linux-gnu/libdl.so.2 $occlum_glibc_path
+cp $busybox_src/busybox image/bin
+cp /lib/x86_64-linux-gnu/libm.so.6 $occlum_glibc_path
+cp /lib/x86_64-linux-gnu/libresolv.so.2 $occlum_glibc_path
+
+mkdir -p "$occlum_instance/image/$exec_path"
+cp "$current/occlum_bash_test.sh" "$occlum_instance/image/$exec_path"
+cp "$current/Occlum.json" "$occlum_instance"
+
+occlum build
+occlum run /root/bin/occlum_bash_test.sh