Rewrite Python demo with NumPy, scikit-learn, and pandas

demos/golang/run_golang_on_occlum.sh

@@ -16,8 +16,8 @@ fi
 rm -rf occlum_context && mkdir occlum_context
 cd occlum_context
 occlum init
-jq '.resource_limits.user_space_size = "380MB"' Occlum.json > temp_Occlum.json
+new_json="$(jq '.resource_limits.user_space_size = "380MB" | .process.default_mmap_size = "300MB"' Occlum.json)" && \
-jq '.process.default_mmap_size = "300MB"' temp_Occlum.json > Occlum.json
+echo "${new_json}" > Occlum.json
 
 # 2. Copy program into Occlum Workspace and build
 cp ../web_server image/bin

demos/openvino/run_benchmark_on_occlum.sh

@@ -9,8 +9,8 @@ rm -rf occlum_context
 mkdir occlum_context
 cd occlum_context
 occlum init
-jq '.resource_limits.user_space_size = "320MB"' Occlum.json > temp_Occlum.json
+new_json="$(jq '.resource_limits.user_space_size = "320MB" | .process.default_mmap_size = "256MB"' Occlum.json)" && \
-jq '.process.default_mmap_size = "256MB"' temp_Occlum.json > Occlum.json
+echo "${new_json}" > Occlum.json
 
 # 2. Copy files into Occlum Workspace and Build
 cp ../$inference_bin/$benchmark image/bin

demos/python/README.md

@@ -1,21 +1,41 @@
 # Use Python with Occlum
 
-This project demonstrates how Occlum enables [Python](https://www.python.org) programs running in SGX enclaves.
+This project demonstrates how Occlum enables _unmodified_ [Python](https://www.python.org) programs running in SGX enclaves.
 
-Occlum is compatible with native binaries from Alpine Linux, so we can copy the Python from Alpine Linux and run it directly on Occlum.
+## Sample Code: CSV Processing in Python
 
-Step 1: Copy the `import_alpine_python.sh` script from an Occlum container to host
+To make the sample code more realistic, we choose to write a Python program that processes CSV data files using [NumPy](https://numpy.org), [pandas](https://pandas.pydata.org), and [scikit-learn](https://scikit-learn.org). The sample code can be found [here](demo.py).
-```
-docker cp <occlum_container>:/root/demos/python/import_alpine_python.sh <host_dir>
-```
-The script downloads a Docker image of Alpine Linux with Python preinstalled and imports the rootfs of the image into an Occlum container so that later we can copy the Alpine's Python libraries into an Occlum secure FS image.
 
-Step 2: Import the rootfs of Alpine Linux's Python Docker image from host to the Occlum container (`/root/alpine_python`)
+## How to Run
+
+This tutorial is written under the assumption that you have Docker installed and use Occlum in a Docker container.
+
+Occlum is compatible with native binaries from Alpine Linux, so we can prepare an Alpine Python Docker image and copy the rootfs of it, then run Python code directly on Occlum.
+
+Step 1 (on the host): Start an Alpine Linux container with Python preinstalled
 ```
-./<host_dir>/import_alpine_python.sh <occlum_container>
+docker pull python:3.7-alpine3.10
+docker run -it --entrypoint /bin/sh --name "<alpine_container_name>" python:3.7-alpine3.10
 ```
 
-Step 3: You can attach to the Occlum container and run a `hello.py` sample on Occlum via
+Step 2 (in the Alpine container): Install the required Python modules
+```
+apk add g++ lapack-dev gfortran
+pip3 install numpy pandas scipy==1.3.1 Cython scikit-learn==0.21.1
+```
+Now that we have installed the required Python libraries in the Alpine Docker image, we can copy the content of the Alpine Docker image into the Occlum container so that we can build a trusted Occlum FS image with Alpine's Python installation inside.
+
+Step 3 (on the host): Copy the `import_alpine_python.sh` script from Occlum container to host,
+```
+docker cp "<occlum_container_name>":/root/demos/python/import_alpine_python.sh <host_dir>
+```
+and import the rootfs of Alpine Linux Docker image to the Occlum container (`/root/alpine_python`)
+```
+./<host_dir>/import_alpine_python.sh "<alpine_container_name>" "<occlum_container_name>"
+```
+
+Step 4 (in the Occlum container): Run the sample code on Occlum via
 ```
 ./run_python_on_occlum.sh
 ```
+It will process CSV data files and generate a file (`smvlight.dat`) in `./occlum_context`.

demos/python/dataset/input.csv

@@ -0,0 +1,5 @@
+id,fea_1,fea_2
+1001,5.8,1
+1002,9.2,0
+1003,7.3,0
+1004,6.6,0

demos/python/dataset/input_label.csv

@@ -0,0 +1,5 @@
+label,id
+0,1001
+0,1002
+1,1003
+1,1004

demos/python/demo.py

@@ -0,0 +1,12 @@
+import pandas as pd
+import numpy as np
+from sklearn.datasets import dump_svmlight_file
+
+df1 = pd.read_csv("./dataset/input_label.csv")
+df2 = pd.read_csv("./dataset/input.csv")
+res = pd.merge(df1, df2, how='left', left_on='id', right_on='id')
+
+X = res[np.setdiff1d(res.columns,['label','id'])]
+y = res.label
+
+dump_svmlight_file(X,y,'/host/smvlight.dat',zero_based=True,multilabel=False)

demos/python/hello.py

@@ -1,3 +0,0 @@
-# Sample code
-
-print('Hello World, Python!')

demos/python/import_alpine_python.sh

@@ -2,14 +2,18 @@
 set -e
 script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}"  )" >/dev/null 2>&1 && pwd )"
 
-target_container=$1
+alpine_container=$1
+target_container=$2
 
-if [ "$target_container" == "" ];then
+if [ "$alpine_container" == "" -o "$target_container" == "" ];then
 cat <<EOF
 Import the rootfs of Alpine Linux's Python Docker image into a target Occlum container (/root/alpine_python)
 
 USAGE:
-    ./import_alpine_python.sh <target_container>
+    ./import_alpine_python.sh <alpine_container> <target_container>
+
+<alpine_container>:
+    The id or name of Alpine Linux Docker container.
 
 <target_container>:
     The id or name of Docker container that you want to copy to.
@@ -17,15 +21,11 @@ EOF
     exit 1
 fi
 
-alpine_python_container="alpine_python_docker"
 alpine_python_tar="$script_dir/alpine_python.tar"
 alpine_python="$script_dir/alpine_python"
 
 # Export the rootfs from Alpine's Docker image
-docker pull python:3.7-alpine3.10
+docker export -o $alpine_python_tar $alpine_container
-docker create --name $alpine_python_container python:3.7-alpine3.10
-docker export -o $alpine_python_tar $alpine_python_container
-docker rm $alpine_python_container
 
 # Copy the exported rootfs to the Occlum container
 rm -rf $alpine_python && mkdir -p $alpine_python

demos/python/run_python_on_occlum.sh

@@ -8,23 +8,36 @@ alpine_fs="/root/alpine_python"
 
 if [ ! -d $alpine_fs ];then
     echo "Error: cannot stat '$alpine_fs' directory"
-    echo "Please see README and import the rootfs of Alpine Linux's Python Docker image"
     exit 1
 fi
 
 # 1. Init Occlum Workspace
-rm -rf occlum_context && mkdir occlum_context
+[ -d occlum_context ] || mkdir occlum_context
 cd occlum_context
-occlum init
+[ -d image ] || occlum init
 
 # 2. Copy files into Occlum Workspace and build
-cp $alpine_fs/usr/local/bin/python3.7 image/bin
+if [ ! -d "image/lib/python3.7" ];then
-cp $alpine_fs/usr/local/lib/libpython3.7m.so.1.0 image/lib
+    cp -f $alpine_fs/usr/local/bin/python3.7 image/bin
-cp $alpine_fs/usr/local/lib/libpython3.so image/lib
+    cp -f $alpine_fs/usr/local/lib/libpython3.7m.so.1.0 image/lib
-cp -r $alpine_fs/usr/local/lib/python3.7 image/lib
+    cp -f $alpine_fs/usr/local/lib/libpython3.so image/lib
-cp ../hello.py image
+    cp -rf $alpine_fs/usr/local/lib/python3.7 image/lib
+    cp -f $alpine_fs/usr/lib/libblas.so.3 image/lib
+    cp -f $alpine_fs/usr/lib/libcblas.so.3 image/lib
+    cp -f $alpine_fs/usr/lib/libbz2.so.1 image/lib
+    cp -f $alpine_fs/usr/lib/libffi.so.6 image/lib
+    cp -f $alpine_fs/usr/lib/libgcc_s.so.1 image/lib
+    cp -f $alpine_fs/usr/lib/libgfortran.so.5 image/lib
+    cp -f $alpine_fs/usr/lib/liblapack.so.3 image/lib
+    cp -f $alpine_fs/usr/lib/liblzma.so.5 image/lib
+    cp -f $alpine_fs/usr/lib/libquadmath.so.0 image/lib
+    cp -f $alpine_fs/lib/libz.so.1 image/lib
+    cp -rf ../dataset image
+    cp -f ../demo.py image
+    new_json="$(jq '.resource_limits.user_space_size = "320MB" | .resource_limits.kernel_space_heap_size = "144MB" | .process.default_mmap_size = "256MB"' Occlum.json)" && echo "${new_json}" > Occlum.json
     occlum build
+fi
 
 # 3. Run the hello world sample
-echo -e "${BLUE}occlum run /bin/python3.7 hello.py${NC}"
+echo -e "${BLUE}occlum run /bin/python3.7 demo.py${NC}"
-occlum run /bin/python3.7 hello.py
+occlum run /bin/python3.7 demo.py