diff --git a/demos/golang/run_golang_on_occlum.sh b/demos/golang/run_golang_on_occlum.sh index 2552c10e..3920c88f 100755 --- a/demos/golang/run_golang_on_occlum.sh +++ b/demos/golang/run_golang_on_occlum.sh @@ -16,8 +16,8 @@ fi rm -rf occlum_context && mkdir occlum_context cd occlum_context occlum init -jq '.resource_limits.user_space_size = "380MB"' Occlum.json > temp_Occlum.json -jq '.process.default_mmap_size = "300MB"' temp_Occlum.json > Occlum.json +new_json="$(jq '.resource_limits.user_space_size = "380MB" | .process.default_mmap_size = "300MB"' Occlum.json)" && \ +echo "${new_json}" > Occlum.json # 2. Copy program into Occlum Workspace and build cp ../web_server image/bin diff --git a/demos/openvino/run_benchmark_on_occlum.sh b/demos/openvino/run_benchmark_on_occlum.sh index 4f17a702..eb1a65f0 100755 --- a/demos/openvino/run_benchmark_on_occlum.sh +++ b/demos/openvino/run_benchmark_on_occlum.sh @@ -9,8 +9,8 @@ rm -rf occlum_context mkdir occlum_context cd occlum_context occlum init -jq '.resource_limits.user_space_size = "320MB"' Occlum.json > temp_Occlum.json -jq '.process.default_mmap_size = "256MB"' temp_Occlum.json > Occlum.json +new_json="$(jq '.resource_limits.user_space_size = "320MB" | .process.default_mmap_size = "256MB"' Occlum.json)" && \ +echo "${new_json}" > Occlum.json # 2. Copy files into Occlum Workspace and Build cp ../$inference_bin/$benchmark image/bin diff --git a/demos/python/README.md b/demos/python/README.md index 8f166e10..aa0a568b 100644 --- a/demos/python/README.md +++ b/demos/python/README.md @@ -1,21 +1,41 @@ # Use Python with Occlum -This project demonstrates how Occlum enables [Python](https://www.python.org) programs running in SGX enclaves. +This project demonstrates how Occlum enables _unmodified_ [Python](https://www.python.org) programs running in SGX enclaves. -Occlum is compatible with native binaries from Alpine Linux, so we can copy the Python from Alpine Linux and run it directly on Occlum. +## Sample Code: CSV Processing in Python -Step 1: Copy the `import_alpine_python.sh` script from an Occlum container to host -``` -docker cp :/root/demos/python/import_alpine_python.sh -``` -The script downloads a Docker image of Alpine Linux with Python preinstalled and imports the rootfs of the image into an Occlum container so that later we can copy the Alpine's Python libraries into an Occlum secure FS image. +To make the sample code more realistic, we choose to write a Python program that processes CSV data files using [NumPy](https://numpy.org), [pandas](https://pandas.pydata.org), and [scikit-learn](https://scikit-learn.org). The sample code can be found [here](demo.py). -Step 2: Import the rootfs of Alpine Linux's Python Docker image from host to the Occlum container (`/root/alpine_python`) +## How to Run + +This tutorial is written under the assumption that you have Docker installed and use Occlum in a Docker container. + +Occlum is compatible with native binaries from Alpine Linux, so we can prepare an Alpine Python Docker image and copy the rootfs of it, then run Python code directly on Occlum. + +Step 1 (on the host): Start an Alpine Linux container with Python preinstalled ``` -.//import_alpine_python.sh +docker pull python:3.7-alpine3.10 +docker run -it --entrypoint /bin/sh --name "" python:3.7-alpine3.10 ``` -Step 3: You can attach to the Occlum container and run a `hello.py` sample on Occlum via +Step 2 (in the Alpine container): Install the required Python modules +``` +apk add g++ lapack-dev gfortran +pip3 install numpy pandas scipy==1.3.1 Cython scikit-learn==0.21.1 +``` +Now that we have installed the required Python libraries in the Alpine Docker image, we can copy the content of the Alpine Docker image into the Occlum container so that we can build a trusted Occlum FS image with Alpine's Python installation inside. + +Step 3 (on the host): Copy the `import_alpine_python.sh` script from Occlum container to host, +``` +docker cp "":/root/demos/python/import_alpine_python.sh +``` +and import the rootfs of Alpine Linux Docker image to the Occlum container (`/root/alpine_python`) +``` +.//import_alpine_python.sh "" "" +``` + +Step 4 (in the Occlum container): Run the sample code on Occlum via ``` ./run_python_on_occlum.sh ``` +It will process CSV data files and generate a file (`smvlight.dat`) in `./occlum_context`. diff --git a/demos/python/dataset/input.csv b/demos/python/dataset/input.csv new file mode 100644 index 00000000..44a77986 --- /dev/null +++ b/demos/python/dataset/input.csv @@ -0,0 +1,5 @@ +id,fea_1,fea_2 +1001,5.8,1 +1002,9.2,0 +1003,7.3,0 +1004,6.6,0 \ No newline at end of file diff --git a/demos/python/dataset/input_label.csv b/demos/python/dataset/input_label.csv new file mode 100644 index 00000000..eff42898 --- /dev/null +++ b/demos/python/dataset/input_label.csv @@ -0,0 +1,5 @@ +label,id +0,1001 +0,1002 +1,1003 +1,1004 \ No newline at end of file diff --git a/demos/python/demo.py b/demos/python/demo.py new file mode 100644 index 00000000..f734e7ac --- /dev/null +++ b/demos/python/demo.py @@ -0,0 +1,12 @@ +import pandas as pd +import numpy as np +from sklearn.datasets import dump_svmlight_file + +df1 = pd.read_csv("./dataset/input_label.csv") +df2 = pd.read_csv("./dataset/input.csv") +res = pd.merge(df1, df2, how='left', left_on='id', right_on='id') + +X = res[np.setdiff1d(res.columns,['label','id'])] +y = res.label + +dump_svmlight_file(X,y,'/host/smvlight.dat',zero_based=True,multilabel=False) diff --git a/demos/python/hello.py b/demos/python/hello.py deleted file mode 100644 index 35306e02..00000000 --- a/demos/python/hello.py +++ /dev/null @@ -1,3 +0,0 @@ -# Sample code - -print('Hello World, Python!') diff --git a/demos/python/import_alpine_python.sh b/demos/python/import_alpine_python.sh index b3b35ea1..082e9e61 100755 --- a/demos/python/import_alpine_python.sh +++ b/demos/python/import_alpine_python.sh @@ -2,14 +2,18 @@ set -e script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -target_container=$1 +alpine_container=$1 +target_container=$2 -if [ "$target_container" == "" ];then +if [ "$alpine_container" == "" -o "$target_container" == "" ];then cat < + ./import_alpine_python.sh + +: + The id or name of Alpine Linux Docker container. : The id or name of Docker container that you want to copy to. @@ -17,15 +21,11 @@ EOF exit 1 fi -alpine_python_container="alpine_python_docker" alpine_python_tar="$script_dir/alpine_python.tar" alpine_python="$script_dir/alpine_python" # Export the rootfs from Alpine's Docker image -docker pull python:3.7-alpine3.10 -docker create --name $alpine_python_container python:3.7-alpine3.10 -docker export -o $alpine_python_tar $alpine_python_container -docker rm $alpine_python_container +docker export -o $alpine_python_tar $alpine_container # Copy the exported rootfs to the Occlum container rm -rf $alpine_python && mkdir -p $alpine_python diff --git a/demos/python/run_python_on_occlum.sh b/demos/python/run_python_on_occlum.sh index 45d9cee2..1211b728 100755 --- a/demos/python/run_python_on_occlum.sh +++ b/demos/python/run_python_on_occlum.sh @@ -8,23 +8,36 @@ alpine_fs="/root/alpine_python" if [ ! -d $alpine_fs ];then echo "Error: cannot stat '$alpine_fs' directory" - echo "Please see README and import the rootfs of Alpine Linux's Python Docker image" exit 1 fi # 1. Init Occlum Workspace -rm -rf occlum_context && mkdir occlum_context +[ -d occlum_context ] || mkdir occlum_context cd occlum_context -occlum init +[ -d image ] || occlum init # 2. Copy files into Occlum Workspace and build -cp $alpine_fs/usr/local/bin/python3.7 image/bin -cp $alpine_fs/usr/local/lib/libpython3.7m.so.1.0 image/lib -cp $alpine_fs/usr/local/lib/libpython3.so image/lib -cp -r $alpine_fs/usr/local/lib/python3.7 image/lib -cp ../hello.py image -occlum build +if [ ! -d "image/lib/python3.7" ];then + cp -f $alpine_fs/usr/local/bin/python3.7 image/bin + cp -f $alpine_fs/usr/local/lib/libpython3.7m.so.1.0 image/lib + cp -f $alpine_fs/usr/local/lib/libpython3.so image/lib + cp -rf $alpine_fs/usr/local/lib/python3.7 image/lib + cp -f $alpine_fs/usr/lib/libblas.so.3 image/lib + cp -f $alpine_fs/usr/lib/libcblas.so.3 image/lib + cp -f $alpine_fs/usr/lib/libbz2.so.1 image/lib + cp -f $alpine_fs/usr/lib/libffi.so.6 image/lib + cp -f $alpine_fs/usr/lib/libgcc_s.so.1 image/lib + cp -f $alpine_fs/usr/lib/libgfortran.so.5 image/lib + cp -f $alpine_fs/usr/lib/liblapack.so.3 image/lib + cp -f $alpine_fs/usr/lib/liblzma.so.5 image/lib + cp -f $alpine_fs/usr/lib/libquadmath.so.0 image/lib + cp -f $alpine_fs/lib/libz.so.1 image/lib + cp -rf ../dataset image + cp -f ../demo.py image + new_json="$(jq '.resource_limits.user_space_size = "320MB" | .resource_limits.kernel_space_heap_size = "144MB" | .process.default_mmap_size = "256MB"' Occlum.json)" && echo "${new_json}" > Occlum.json + occlum build +fi # 3. Run the hello world sample -echo -e "${BLUE}occlum run /bin/python3.7 hello.py${NC}" -occlum run /bin/python3.7 hello.py +echo -e "${BLUE}occlum run /bin/python3.7 demo.py${NC}" +occlum run /bin/python3.7 demo.py