Add glibc-supported python demo

.github/workflows/demo_test.yml

@@ -359,26 +359,55 @@ jobs:
 
 
   # Python test also needs its own image because in Alpine environment, modules are built locally and consumes a lot of time.
-  Python_support_test:
+  Python_musl_support_test:
     runs-on: ubuntu-18.04
     steps:
     - uses: actions/checkout@v1
       with:
         submodules: true
     - name: Create container
-      run: docker run -itd --name=python_support_test -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python
+      run: docker run -itd --name=python_musl_support_test -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python
 
     - name: Build dependencies
-      run: docker exec python_support_test bash -c "cd /root/occlum; make submodule"
+      run: docker exec python_musl_support_test bash -c "cd /root/occlum; make submodule"
 
     - name: Make install
-      run: docker exec python_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
+      run: docker exec python_musl_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
 
     - name: Run python support test
-      run: docker exec python_support_test bash -c "cd /root/occlum/demos/python; SGX_MODE=SIM ./run_python_on_occlum.sh"
+      run: docker exec python_musl_support_test bash -c "cd /root/occlum/demos/python/python_musl; SGX_MODE=SIM ./run_python_on_occlum.sh"
 
     - name: Check result
-      run: docker exec python_support_test bash -c "cd /root/occlum/demos/python/occlum_instance; cat smvlight.dat"
+      run: docker exec python_musl_support_test bash -c "cd /root/occlum/demos/python/python_musl/occlum_instance; cat smvlight.dat"
+
+  # Python glibc support test
+  Python_glibc_support_test:
+    runs-on: ubuntu-18.04
+    steps:
+    - uses: actions/checkout@v1
+      with:
+        submodules: true
+
+    - name: Get occlum version
+      run: echo "OCCLUM_VERSION=$(grep "Version =" src/pal/include/occlum_version.h |  awk '{print $4}')" >> $GITHUB_ENV
+
+    - name: Create container
+      run: docker run -itd --name=python_glibc_support_test -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04
+
+    - name: Build dependencies
+      run: docker exec python_glibc_support_test bash -c "cd /root/occlum; make submodule"
+
+    - name: Make install
+      run: docker exec python_glibc_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install"
+
+    - name: download conda and build python
+      run: docker exec python_glibc_support_test bash -c "cd /root/occlum/demos/python/python_glibc; ./install_python_with_conda.sh"
+
+    - name: Run python glibc support test
+      run: docker exec python_glibc_support_test bash -c "cd /root/occlum/demos/python/python_glibc; SGX_MODE=SIM ./run_python_on_occlum.sh"
+
+    - name: Check result
+      run: docker exec python_glibc_support_test bash -c "cd /root/occlum/demos/python/python_glibc/occlum_instance; cat smvlight.dat"
 
   # Redis test
   Redis_support_test:

demos/README.md

@@ -31,7 +31,7 @@ This set of demos shows how apps written with popular programming languages can
 
 * [golang](golang/): A collection of [Golang](https://golang.org) program demos.
 * [java](java/): A demo of [Java](https://openjdk.java.net) program.
-* [python](python/) A demo of [Python](https://www.python.org) program.
+* [python](python/) A collection of [Python](https://www.python.org) program demos, contain [glibc-supported python](python/python_glibc) demo and [musl-supported python](python/python_musl) demo.
 * [rust](rust/) A demo of [Rust](https://www.rust-lang.org) program.
 
 ## Other demos

demos/python/python_glibc/README.md

@@ -0,0 +1,32 @@
+# Use Python with Occlum
+
+This project demonstrates how Occlum enables _unmodified_ [Python](https://www.python.org) programs running in SGX enclaves, which is based on glibc.
+
+## Sample Code: CSV Processing in Python
+
+To make the sample code more realistic, we choose to write a Python program that processes CSV data files using [NumPy](https://numpy.org), [pandas](https://pandas.pydata.org), and [scikit-learn](https://scikit-learn.org). The sample code can be found [here](demo.py).
+
+## How to Run
+
+This tutorial is written under the assumption that you have Docker installed and use Occlum in a Docker container.
+
+Occlum is compatible with glibc-supported Python, we employ miniconda as python installation tool. You can import any python dependencies using conda. Here, miniconda is automatically installed by install_python_with_conda.sh script, the required python and related dependencies for this project are also loaded by this script. Here, we take occlum/occlum:0.22.0-ubuntu18.04 as example.
+
+Step 1 (on the host): Start an Occlum container
+```
+docker pull occlum/occlum:0.22.0-ubuntu18.04
+docker run -it --name=pythonDemo --device /dev/sgx/enclave occlum/occlum:0.22.0-ubuntu18.04 bash
+```
+
+Step 2 (on the host): Download miniconda and install python to prefix position.
+```
+cd /root/occlum/demos/python/python_glibc
+bash ./install_python_with_conda.sh
+```
+
+Step 3 (on the host): Run the sample code on Occlum
+```
+cd /root/occlum/demos/python/python_glibc
+bash ./run_python_on_occlum.sh
+```
+It will process CSV data files and generate a file (`smvlight.dat`) in `./occlum_instance`.

demos/python/python_glibc/install_python_with_conda.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -e
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}"  )" >/dev/null 2>&1 && pwd )"
+
+# 1. Init occlum workspace
+[ -d occlum_instance ] || occlum new occlum_instance
+
+# 2. Install python and dependencies to specified position
+[ -d Miniconda3-latest-Linux-x86_64.sh ] || wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
+[ -d miniconda ] || bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $script_dir/miniconda
+$script_dir/miniconda/bin/conda create --prefix $script_dir/occlum_instance/image/opt/python-occlum -y python=3.7 numpy pandas scipy=1.3.1 Cython scikit-learn=0.21.1
+

demos/python/python_glibc/run_python_on_occlum.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+set -e
+
+BLUE='\033[1;34m'
+NC='\033[0m'
+
+script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}"  )" >/dev/null 2>&1 && pwd )"
+python_dir="$script_dir/occlum_instance/image/opt/python-occlum"
+
+if [ ! -d $python_dir ];then
+    echo "Error: cannot stat '$python_dir' directory"
+    exit 1
+fi
+
+cd occlum_instance
+# Copy files into Occlum Workspace and build
+if [ ! -d "image/lib/python3" ];then
+    ln -s /opt/python-occlum/bin/python3 image/bin/python3
+    cp -f /opt/occlum/glibc/lib/libdl.so.2 image/opt/occlum/glibc/lib/
+    cp -f /opt/occlum/glibc/lib/libutil.so.1 image/opt/occlum/glibc/lib/
+    cp -f /opt/occlum/glibc/lib/librt.so.1 image/opt/occlum/glibc/lib/
+    cp -rf ../dataset image
+    cp -f ../demo.py image
+    new_json="$(jq '.resource_limits.user_space_size = "320MB" |
+                    .resource_limits.kernel_space_heap_size = "256MB" |
+                    .process.default_mmap_size = "256MB" |
+                    .env.default += ["PYTHONHOME=/opt/python-occlum"]' Occlum.json)" && \
+    echo "${new_json}" > Occlum.json
+    occlum build
+fi
+
+# Run the python demo
+echo -e "${BLUE}occlum run /bin/python3 demo.py${NC}"
+occlum run /bin/python3 demo.py

demos/python/python_musl/dataset/input.csv

@@ -0,0 +1,5 @@
+id,fea_1,fea_2
+1001,5.8,1
+1002,9.2,0
+1003,7.3,0
+1004,6.6,0

demos/python/python_musl/dataset/input_label.csv

@@ -0,0 +1,5 @@
+label,id
+0,1001
+0,1002
+1,1003
+1,1004

demos/python/python_musl/demo.py

@@ -0,0 +1,12 @@
+import pandas as pd
+import numpy as np
+from sklearn.datasets import dump_svmlight_file
+
+df1 = pd.read_csv("./dataset/input_label.csv")
+df2 = pd.read_csv("./dataset/input.csv")
+res = pd.merge(df1, df2, how='left', left_on='id', right_on='id')
+
+X = res[np.setdiff1d(res.columns,['label','id'])]
+y = res.label
+
+dump_svmlight_file(X,y,'/host/smvlight.dat',zero_based=True,multilabel=False)