From 53658e865b3860fbc7646375b4f05e738ad1863c Mon Sep 17 00:00:00 2001 From: ClawSeven Date: Mon, 7 Jun 2021 17:09:20 +0800 Subject: [PATCH] Add glibc-supported python demo --- .github/workflows/demo_test.yml | 41 ++++++++++++++++--- demos/README.md | 2 +- demos/python/python_glibc/README.md | 32 +++++++++++++++ .../{ => python_glibc}/dataset/input.csv | 0 .../dataset/input_label.csv | 0 demos/python/{ => python_glibc}/demo.py | 0 .../python_glibc/install_python_with_conda.sh | 12 ++++++ .../python_glibc/run_python_on_occlum.sh | 34 +++++++++++++++ demos/python/{ => python_musl}/README.md | 0 demos/python/python_musl/dataset/input.csv | 5 +++ .../python_musl/dataset/input_label.csv | 5 +++ demos/python/python_musl/demo.py | 12 ++++++ .../{ => python_musl}/import_alpine_python.sh | 0 .../{ => python_musl}/run_python_on_occlum.sh | 0 14 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 demos/python/python_glibc/README.md rename demos/python/{ => python_glibc}/dataset/input.csv (100%) rename demos/python/{ => python_glibc}/dataset/input_label.csv (100%) rename demos/python/{ => python_glibc}/demo.py (100%) create mode 100755 demos/python/python_glibc/install_python_with_conda.sh create mode 100755 demos/python/python_glibc/run_python_on_occlum.sh rename demos/python/{ => python_musl}/README.md (100%) create mode 100644 demos/python/python_musl/dataset/input.csv create mode 100644 demos/python/python_musl/dataset/input_label.csv create mode 100644 demos/python/python_musl/demo.py rename demos/python/{ => python_musl}/import_alpine_python.sh (100%) rename demos/python/{ => python_musl}/run_python_on_occlum.sh (100%) diff --git a/.github/workflows/demo_test.yml b/.github/workflows/demo_test.yml index ac865195..775e05dd 100644 --- a/.github/workflows/demo_test.yml +++ b/.github/workflows/demo_test.yml @@ -359,26 +359,55 @@ jobs: # Python test also needs its own image because in Alpine environment, modules are built locally and consumes a lot of time. - Python_support_test: + Python_musl_support_test: runs-on: ubuntu-18.04 steps: - uses: actions/checkout@v1 with: submodules: true - name: Create container - run: docker run -itd --name=python_support_test -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python + run: docker run -itd --name=python_musl_support_test -v $GITHUB_WORKSPACE:/root/occlum occlumbackup/occlum:latest-ubuntu18.04-python - name: Build dependencies - run: docker exec python_support_test bash -c "cd /root/occlum; make submodule" + run: docker exec python_musl_support_test bash -c "cd /root/occlum; make submodule" - name: Make install - run: docker exec python_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install" + run: docker exec python_musl_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install" - name: Run python support test - run: docker exec python_support_test bash -c "cd /root/occlum/demos/python; SGX_MODE=SIM ./run_python_on_occlum.sh" + run: docker exec python_musl_support_test bash -c "cd /root/occlum/demos/python/python_musl; SGX_MODE=SIM ./run_python_on_occlum.sh" - name: Check result - run: docker exec python_support_test bash -c "cd /root/occlum/demos/python/occlum_instance; cat smvlight.dat" + run: docker exec python_musl_support_test bash -c "cd /root/occlum/demos/python/python_musl/occlum_instance; cat smvlight.dat" + + # Python glibc support test + Python_glibc_support_test: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v1 + with: + submodules: true + + - name: Get occlum version + run: echo "OCCLUM_VERSION=$(grep "Version =" src/pal/include/occlum_version.h | awk '{print $4}')" >> $GITHUB_ENV + + - name: Create container + run: docker run -itd --name=python_glibc_support_test -v $GITHUB_WORKSPACE:/root/occlum occlum/occlum:${{ env.OCCLUM_VERSION }}-ubuntu18.04 + + - name: Build dependencies + run: docker exec python_glibc_support_test bash -c "cd /root/occlum; make submodule" + + - name: Make install + run: docker exec python_glibc_support_test bash -c "source /opt/intel/sgxsdk/environment; cd /root/occlum; OCCLUM_RELEASE_BUILD=1 make install" + + - name: download conda and build python + run: docker exec python_glibc_support_test bash -c "cd /root/occlum/demos/python/python_glibc; ./install_python_with_conda.sh" + + - name: Run python glibc support test + run: docker exec python_glibc_support_test bash -c "cd /root/occlum/demos/python/python_glibc; SGX_MODE=SIM ./run_python_on_occlum.sh" + + - name: Check result + run: docker exec python_glibc_support_test bash -c "cd /root/occlum/demos/python/python_glibc/occlum_instance; cat smvlight.dat" # Redis test Redis_support_test: diff --git a/demos/README.md b/demos/README.md index 9899c824..27932b38 100644 --- a/demos/README.md +++ b/demos/README.md @@ -31,7 +31,7 @@ This set of demos shows how apps written with popular programming languages can * [golang](golang/): A collection of [Golang](https://golang.org) program demos. * [java](java/): A demo of [Java](https://openjdk.java.net) program. -* [python](python/) A demo of [Python](https://www.python.org) program. +* [python](python/) A collection of [Python](https://www.python.org) program demos, contain [glibc-supported python](python/python_glibc) demo and [musl-supported python](python/python_musl) demo. * [rust](rust/) A demo of [Rust](https://www.rust-lang.org) program. ## Other demos diff --git a/demos/python/python_glibc/README.md b/demos/python/python_glibc/README.md new file mode 100644 index 00000000..b8373f53 --- /dev/null +++ b/demos/python/python_glibc/README.md @@ -0,0 +1,32 @@ +# Use Python with Occlum + +This project demonstrates how Occlum enables _unmodified_ [Python](https://www.python.org) programs running in SGX enclaves, which is based on glibc. + +## Sample Code: CSV Processing in Python + +To make the sample code more realistic, we choose to write a Python program that processes CSV data files using [NumPy](https://numpy.org), [pandas](https://pandas.pydata.org), and [scikit-learn](https://scikit-learn.org). The sample code can be found [here](demo.py). + +## How to Run + +This tutorial is written under the assumption that you have Docker installed and use Occlum in a Docker container. + +Occlum is compatible with glibc-supported Python, we employ miniconda as python installation tool. You can import any python dependencies using conda. Here, miniconda is automatically installed by install_python_with_conda.sh script, the required python and related dependencies for this project are also loaded by this script. Here, we take occlum/occlum:0.22.0-ubuntu18.04 as example. + +Step 1 (on the host): Start an Occlum container +``` +docker pull occlum/occlum:0.22.0-ubuntu18.04 +docker run -it --name=pythonDemo --device /dev/sgx/enclave occlum/occlum:0.22.0-ubuntu18.04 bash +``` + +Step 2 (on the host): Download miniconda and install python to prefix position. +``` +cd /root/occlum/demos/python/python_glibc +bash ./install_python_with_conda.sh +``` + +Step 3 (on the host): Run the sample code on Occlum +``` +cd /root/occlum/demos/python/python_glibc +bash ./run_python_on_occlum.sh +``` +It will process CSV data files and generate a file (`smvlight.dat`) in `./occlum_instance`. diff --git a/demos/python/dataset/input.csv b/demos/python/python_glibc/dataset/input.csv similarity index 100% rename from demos/python/dataset/input.csv rename to demos/python/python_glibc/dataset/input.csv diff --git a/demos/python/dataset/input_label.csv b/demos/python/python_glibc/dataset/input_label.csv similarity index 100% rename from demos/python/dataset/input_label.csv rename to demos/python/python_glibc/dataset/input_label.csv diff --git a/demos/python/demo.py b/demos/python/python_glibc/demo.py similarity index 100% rename from demos/python/demo.py rename to demos/python/python_glibc/demo.py diff --git a/demos/python/python_glibc/install_python_with_conda.sh b/demos/python/python_glibc/install_python_with_conda.sh new file mode 100755 index 00000000..00dba6c9 --- /dev/null +++ b/demos/python/python_glibc/install_python_with_conda.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +# 1. Init occlum workspace +[ -d occlum_instance ] || occlum new occlum_instance + +# 2. Install python and dependencies to specified position +[ -d Miniconda3-latest-Linux-x86_64.sh ] || wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh +[ -d miniconda ] || bash ./Miniconda3-latest-Linux-x86_64.sh -b -p $script_dir/miniconda +$script_dir/miniconda/bin/conda create --prefix $script_dir/occlum_instance/image/opt/python-occlum -y python=3.7 numpy pandas scipy=1.3.1 Cython scikit-learn=0.21.1 + diff --git a/demos/python/python_glibc/run_python_on_occlum.sh b/demos/python/python_glibc/run_python_on_occlum.sh new file mode 100755 index 00000000..6e7ff867 --- /dev/null +++ b/demos/python/python_glibc/run_python_on_occlum.sh @@ -0,0 +1,34 @@ +#!/bin/bash +set -e + +BLUE='\033[1;34m' +NC='\033[0m' + +script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +python_dir="$script_dir/occlum_instance/image/opt/python-occlum" + +if [ ! -d $python_dir ];then + echo "Error: cannot stat '$python_dir' directory" + exit 1 +fi + +cd occlum_instance +# Copy files into Occlum Workspace and build +if [ ! -d "image/lib/python3" ];then + ln -s /opt/python-occlum/bin/python3 image/bin/python3 + cp -f /opt/occlum/glibc/lib/libdl.so.2 image/opt/occlum/glibc/lib/ + cp -f /opt/occlum/glibc/lib/libutil.so.1 image/opt/occlum/glibc/lib/ + cp -f /opt/occlum/glibc/lib/librt.so.1 image/opt/occlum/glibc/lib/ + cp -rf ../dataset image + cp -f ../demo.py image + new_json="$(jq '.resource_limits.user_space_size = "320MB" | + .resource_limits.kernel_space_heap_size = "256MB" | + .process.default_mmap_size = "256MB" | + .env.default += ["PYTHONHOME=/opt/python-occlum"]' Occlum.json)" && \ + echo "${new_json}" > Occlum.json + occlum build +fi + +# Run the python demo +echo -e "${BLUE}occlum run /bin/python3 demo.py${NC}" +occlum run /bin/python3 demo.py diff --git a/demos/python/README.md b/demos/python/python_musl/README.md similarity index 100% rename from demos/python/README.md rename to demos/python/python_musl/README.md diff --git a/demos/python/python_musl/dataset/input.csv b/demos/python/python_musl/dataset/input.csv new file mode 100644 index 00000000..44a77986 --- /dev/null +++ b/demos/python/python_musl/dataset/input.csv @@ -0,0 +1,5 @@ +id,fea_1,fea_2 +1001,5.8,1 +1002,9.2,0 +1003,7.3,0 +1004,6.6,0 \ No newline at end of file diff --git a/demos/python/python_musl/dataset/input_label.csv b/demos/python/python_musl/dataset/input_label.csv new file mode 100644 index 00000000..eff42898 --- /dev/null +++ b/demos/python/python_musl/dataset/input_label.csv @@ -0,0 +1,5 @@ +label,id +0,1001 +0,1002 +1,1003 +1,1004 \ No newline at end of file diff --git a/demos/python/python_musl/demo.py b/demos/python/python_musl/demo.py new file mode 100644 index 00000000..f734e7ac --- /dev/null +++ b/demos/python/python_musl/demo.py @@ -0,0 +1,12 @@ +import pandas as pd +import numpy as np +from sklearn.datasets import dump_svmlight_file + +df1 = pd.read_csv("./dataset/input_label.csv") +df2 = pd.read_csv("./dataset/input.csv") +res = pd.merge(df1, df2, how='left', left_on='id', right_on='id') + +X = res[np.setdiff1d(res.columns,['label','id'])] +y = res.label + +dump_svmlight_file(X,y,'/host/smvlight.dat',zero_based=True,multilabel=False) diff --git a/demos/python/import_alpine_python.sh b/demos/python/python_musl/import_alpine_python.sh similarity index 100% rename from demos/python/import_alpine_python.sh rename to demos/python/python_musl/import_alpine_python.sh diff --git a/demos/python/run_python_on_occlum.sh b/demos/python/python_musl/run_python_on_occlum.sh similarity index 100% rename from demos/python/run_python_on_occlum.sh rename to demos/python/python_musl/run_python_on_occlum.sh