Add occlum-ra-flow demo

This commit is contained in:
Zheng, Qi 2022-02-23 15:01:38 +08:00 committed by Zongmin.Gu
parent f4665dac11
commit b7edb11a88
14 changed files with 663 additions and 1 deletions

@ -880,12 +880,62 @@ jobs:
- name: Run gRPC client - name: Run gRPC client
run: | run: |
sleep ${{ env.nap_time }}; sleep ${{ env.nap_time }};
docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/ra_tls; ./run.sh client" docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/ra_tls; ./run.sh client cert"
- name: Clean the environment - name: Clean the environment
if: ${{ always() }} if: ${{ always() }}
run: docker stop ${{ env.CONTAINER_NAME }} run: docker stop ${{ env.CONTAINER_NAME }}
Init_RA_test:
if: github.event_name == 'push' || ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
runs-on: ${{ matrix.self_runner }}
strategy:
matrix:
self_runner: [[self-hosted, SGX2-HW]]
steps:
- name: Clean before running
run: |
sudo chown -R ${{ secrets.CI_ADMIN }} "${{ github.workspace }}"
- name: Checkout code
if: github.event_name == 'push'
uses: actions/checkout@v2
with:
submodules: true
- name: Checkout code from fork
if: ${{ contains(github.event.pull_request.labels.*.name, 'SGX-hardware-test-required') }}
uses: actions/checkout@v2
with:
ref: refs/pull/${{ github.event.pull_request.number }}/merge
submodules: true
- uses: ./.github/workflows/composite_action/hw
with:
container-name: ${{ github.job }}
build-envs: 'OCCLUM_RELEASE_BUILD=1'
- name: Build the init-ra all content
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow; ./build_content.sh"
- name: Run init-ra server and client(flask-tls) on backgroud
run: docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow; ./run.sh"
- name: Test PUT data with certificate
run: |
docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow;
curl --cacert flask.crt -X PUT https://localhost:4996/customer/1 -d "data=Tom"
- name: Test GET data with certificate
run: |
docker exec ${{ env.CONTAINER_NAME }} bash -c "cd /root/occlum/demos/remote_attestation/init_ra_flow;
curl --cacert flask.crt -X GET https://localhost:4996/customer/1
- name: Clean the environment
if: ${{ always() }}
run: docker stop ${{ env.CONTAINER_NAME }}
Stress_test_with_musl: Stress_test_with_musl:
if: github.event_name == 'schedule' if: github.event_name == 'schedule'
runs-on: ${{ matrix.self_runner }} runs-on: ${{ matrix.self_runner }}

@ -0,0 +1,92 @@
# Occlum RA Flow in a real demo
## Overview
Remote attestation is a key part in the confidential computing. Occlum provides a [`DCAP Library`](../../../tools/toolchains/dcap_lib/) to ease the RA application. A brief Occlum RA [`introduction`](../../../docs/remote_attestation.md) is a good entry point. But still, utilizing the RA in a real application is complicated for general developers.
Occlum provides a `Init RA` way to seperate the RA operation and the actual application. With this way, the APP developers don't need know too much about the RA and the application doesn't need to be modified for RA.
This demo shows the `Init RA` way with a sample [`Flask TLS web application`](../../python/flask/), based on [`GRPC-RATLS`](../../ra_tls/) server/client implementation and a modified [`init`](./init_ra/) for Occlum InitFS.
![Arch Overview](./arch.png)
The GRPC-RATLS server holds some sensitive data thus it is usually deploed on secure environment. The application consuming the sensitive data could be deployed on general environment, such as Cloud service vendor provided SGX instance. For this demo, all are running on one SGX machine.
## Flow
* Starts the GRPC-RATLS server. It holds `RA Verify Config` JSON and `Secrets` JSON files. The `RA Verify Config` JSON records which SGX quote part should be verified. The template is [`ra_config_template.json`](./ra_config_template.json), all supported `verify_xxx` are on in default.
```
{
"verify_mr_enclave" : "on",
"verify_mr_signer" : "on",
"verify_isv_prod_id" : "on",
"verify_isv_svn" : "on",
"verify_enclave_debuggable" : "on",
"sgx_mrs": [
{
"mr_enclave" : "",
"mr_signer" : "",
"isv_prod_id" : "0",
"isv_svn" : "0",
"debuggable" : false
}
],
"other" : []
}
```
Users need decide which `verify_xxx` are taking effect.
1. if yes, fill in the measures data under `sgx_mrs`.
2. if no, set `verify_xxx` to `off`.
Details could refer to the `build_server_instance` in script [`build_content.sh`](./build_content.sh).
The `RA Verify Config` JSON records the secrets. Each secret has a name and its base64 encoded string value, such as
```
{
"flask_cert" : "dGVzdCBzYW1wbGUgY2VydGlmaWNhdGVzCg==",
"flask_key" : "dGVzdCBzYW1wbGUga2V5Cg=="
"image_key" : "YTUtNmQtN2YtY2YtYWUtOTMtZTItMWYtNWItOGEtODMtM2YtNzktNzgtMjktZmYK"
}
```
`flask_cert` and `flask_key` are generated by `openssl` by script `gen-cert.sh`. They are used for the Flask-TLS restful sever set-up.
`image_key` is used to encrypt the Occlum APP RootFS image which is `Flask-TLS` in this demo. It is generated by command `occlum gen-image-key image_key`. The image encryption could be done by `occlum build --image-key image-key`. With this encryption, anything saved in the RootFS has a good protection.
* Starts the Flask-TLS-Infer demo. For every Occlum built application, it starts `init` process first, then starts the real application in RootFS. The default [`init`](../../../tools/init/) process just run RootFS integrity check and then load the RootFS where the real application is located.
For this demo, a modified [`init`](./init_ra/) is used. Besides the general `init` operation, it embeds the `GRPC-RATLS` client API `gr_client_get_secret`, gets the secrets(base64 encoded) from the `GRPC-RATLS server`, does base64 decoding, acquires the real secrets. The `image_key` is used to decrypt the RootFS image. The other two are saved to RootFS. In this example, they are `/etc/flask.crt` and `/etc/flask.key`. Finally, when the Flask-TLS app is running, all secrets are securely obtained already in `init` thus the app runs successfully without RA involvement in this stage.
## How-to build the demo
* Just run `build_content.sh` which builds everything.
Once successful, two Occlum instances are created.
```
occlum_client
occlum_server
```
## How-to run
* Starts the GRPC-RATLS server in background.
```
cd occlum_server
occlum run /bin/server &
```
* Starts the Flask-TLS web portal in backgroud.
```
cd occlum_client
occlum run /bin/rest_api.py &
```
Above two could be executed in one script [`run.sh`](./run.sh).
* Access the Flask-TLS web portal with valid certificate.
```
curl --cacert flask.crt -X PUT https://localhost:4996/customer/1 -d "data=Tom"
curl --cacert flask.crt -X PUT https://localhost:4996/customer/2 -d "data=Jerry"
curl --cacert flask.crt -X GET https://localhost:4996/customer/1
curl --cacert flask.crt -X GET https://localhost:4996/customer/2
```

Binary file not shown.

After

Width:  |  Height:  |  Size: 157 KiB

@ -0,0 +1,135 @@
#!/bin/bash
set -e
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
export DEP_LIBS_DIR="${script_dir}/dep_libs"
export INITRA_DIR="${script_dir}/init_ra"
export FLASK_DIR="${script_dir}/../../python/flask"
export RATLS_DIR="${script_dir}/../../ra_tls"
function build_ratls()
{
rm -rf ${DEP_LIBS_DIR} && mkdir ${DEP_LIBS_DIR}
pushd ${RATLS_DIR}
./download_and_prepare.sh
./build_and_install.sh musl
./build_occlum_instance.sh musl
cp ./grpc-src/examples/cpp/ratls/build/libgrpc_ratls_client.so ${DEP_LIBS_DIR}/
cp ./grpc-src/examples/cpp/ratls/build/libhw_grpc_proto.so ${DEP_LIBS_DIR}/
popd
}
function build_flask()
{
pushd ${FLASK_DIR}
${FLASK_DIR}/install_python_with_conda.sh
popd
}
function build_init_ra()
{
pushd ${INITRA_DIR}
occlum-cargo clean
occlum-cargo build --release
popd
}
function build_client_instance()
{
# generate client image key
occlum gen-image-key image_key
rm -rf occlum_client && occlum new occlum_client
pushd occlum_client
# prepare flask content
rm -rf image
copy_bom -f ../flask.yaml --root image --include-dir /opt/occlum/etc/template
new_json="$(jq '.resource_limits.user_space_size = "600MB" |
.resource_limits.kernel_space_heap_size = "128MB" |
.resource_limits.max_num_of_threads = 32 |
.metadata.debuggable = false |
.env.default += ["PYTHONHOME=/opt/python-occlum"]' Occlum.json)" && \
echo "${new_json}" > Occlum.json
occlum build --image-key ../image_key
# Get server mrsigner.
# Here client and server use the same signer-key thus using client mrsigner directly.
jq ' .verify_mr_enclave = "off" |
.verify_mr_signer = "off" |
.verify_isv_prod_id = "off" |
.verify_isv_svn = "off" |
.verify_enclave_debuggable = "on" |
.sgx_mrs[0].mr_signer = ''"'`get_mr client mr_signer`'" |
.sgx_mrs[0].debuggable = false ' ../ra_config_template.json > dynamic_config.json
# prepare init-ra content
rm -rf initfs
copy_bom -f ../init_ra_client.yaml --root initfs --include-dir /opt/occlum/etc/template
occlum build -f --image-key ../image_key
popd
}
function get_mr() {
sgx_sign dump -enclave ${script_dir}/occlum_$1/build/lib/libocclum-libos.signed.so -dumpfile ../metadata_info_$1.txt
if [ "$2" == "mr_enclave" ]; then
sed -n -e '/enclave_hash.m/,/metadata->enclave_css.body.isv_prod_id/p' ../metadata_info_$1.txt |head -3|tail -2|xargs|sed 's/0x//g'|sed 's/ //g'
elif [ "$2" == "mr_signer" ]; then
tail -2 ../metadata_info_$1.txt |xargs|sed 's/0x//g'|sed 's/ //g'
fi
}
function gen_secret_json() {
# First generate cert/key by openssl
./gen-cert.sh
# Then do base64 encode
cert=$(base64 -w 0 flask.crt)
key=$(base64 -w 0 flask.key)
image_key=$(base64 -w 0 image_key)
# Then generate secret json
jq -n --arg cert "$cert" --arg key "$key" --arg image_key "$image_key" \
'{"flask_cert": $cert, "flask_key": $key, "image_key": $image_key}' > secret_config.json
}
function build_server_instance()
{
gen_secret_json
rm -rf occlum_server && occlum new occlum_server
pushd occlum_server
jq '.verify_mr_enclave = "on" |
.verify_mr_signer = "on" |
.verify_isv_prod_id = "off" |
.verify_isv_svn = "off" |
.verify_enclave_debuggable = "on" |
.sgx_mrs[0].mr_enclave = ''"'`get_mr client mr_enclave`'" |
.sgx_mrs[0].mr_signer = ''"'`get_mr client mr_signer`'" |
.sgx_mrs[0].debuggable = false ' ../ra_config_template.json > dynamic_config.json
new_json="$(jq '.resource_limits.user_space_size = "500MB" |
.metadata.debuggable = false ' Occlum.json)" && \
echo "${new_json}" > Occlum.json
rm -rf image
copy_bom -f ../ra_server.yaml --root image --include-dir /opt/occlum/etc/template
occlum build
popd
}
build_ratls
build_flask
build_init_ra
build_client_instance
build_server_instance

@ -0,0 +1,17 @@
includes:
- base.yaml
targets:
- target: /usr/bin
createlinks:
- src: /opt/python-occlum/bin/python3
linkname: python3
# python packages
- target: /opt
copy:
- dirs:
- ${FLASK_DIR}/python-occlum
# below are python code and data
- target: /bin
copy:
- files:
- ${FLASK_DIR}/rest_api.py

@ -0,0 +1,14 @@
#!/bin/bash
# Users should run this script in secure environment and keep
# the geneated key/cert proctected.
pushd ~
openssl rand -writerand .rnd
popd
# Geneate self-signed key/cert
# Generate valid Flask server Key/Cert
openssl genrsa -out flask.key 2048
openssl req -nodes -new -key flask.key -subj "/CN=localhost" -out flask.csr
openssl x509 -req -sha256 -days 365 -in flask.csr -signkey flask.key -out flask.crt

@ -0,0 +1,94 @@
# This file is automatically @generated by Cargo.
# It is not intended for manual editing.
[[package]]
name = "init"
version = "0.0.1"
dependencies = [
"libc",
"serde",
"serde_json",
]
[[package]]
name = "itoa"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"
[[package]]
name = "libc"
version = "0.2.84"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff"
[[package]]
name = "proc-macro2"
version = "1.0.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71"
dependencies = [
"unicode-xid",
]
[[package]]
name = "quote"
version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7"
dependencies = [
"proc-macro2",
]
[[package]]
name = "ryu"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "serde"
version = "1.0.123"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae"
dependencies = [
"serde_derive",
]
[[package]]
name = "serde_derive"
version = "1.0.123"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "serde_json"
version = "1.0.62"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486"
dependencies = [
"itoa",
"ryu",
"serde",
]
[[package]]
name = "syn"
version = "1.0.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081"
dependencies = [
"proc-macro2",
"quote",
"unicode-xid",
]
[[package]]
name = "unicode-xid"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"

@ -0,0 +1,11 @@
[package]
name = "init"
version = "0.0.1"
build = "build.rs"
authors = ["LI Qing geding.lq@antgroup.com"]
edition = "2018"
[dependencies]
libc = "0.2.84"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"

@ -0,0 +1,5 @@
fn main() {
println!("cargo:rustc-link-search=native=../dep_libs");
println!("cargo:rustc-link-lib=dylib=grpc_ratls_client");
println!("cargo:rustc-link-lib=dylib=hw_grpc_proto");
}

@ -0,0 +1,177 @@
extern crate libc;
extern crate serde;
extern crate serde_json;
use libc::syscall;
use serde::Deserialize;
use std::error::Error;
use std::fs;
use std::fs::File;
use std::io::{ErrorKind, Read};
use std::ffi::CString;
use std::os::raw::{c_int, c_char};
#[link(name = "grpc_ratls_client")]
extern "C" {
fn gr_client_get_secret(
server_addr: *const c_char, // grpc server address+port, such as "localhost:50051"
config_json: *const c_char, // ratls handshake config json file
name: *const c_char, // secret name to be requested
secret_file: *const c_char // secret file to be saved
) -> c_int;
}
fn main() -> Result<(), Box<dyn Error>> {
// Load the configuration from initfs
const IMAGE_CONFIG_FILE: &str = "/etc/image_config.json";
let image_config = load_config(IMAGE_CONFIG_FILE)?;
// Get the MAC of Occlum.json.protected file
let occlum_json_mac = {
let mut mac: sgx_aes_gcm_128bit_tag_t = Default::default();
parse_str_to_bytes(&image_config.occlum_json_mac, &mut mac)?;
mac
};
let occlum_json_mac_ptr = &occlum_json_mac as *const sgx_aes_gcm_128bit_tag_t;
// Get client secrets through grpc-ratls
let server_addr = CString::new("localhost:50051").unwrap();
let config_json = CString::new("dynamic_config.json").unwrap();
// Get the key of FS image if needed
let key = match &image_config.image_type[..] {
"encrypted" => {
// Get the image encrypted key through RA
let secret = CString::new("image_key").unwrap();
let filename = CString::new("/etc/image_key").unwrap();
let ret = unsafe {
gr_client_get_secret(
server_addr.as_ptr(),
config_json.as_ptr(),
secret.as_ptr(),
filename.as_ptr())
};
if ret != 0 {
println!("gr_client_get_secret failed return {}", ret);
return Err(Box::new(std::io::Error::last_os_error()));
}
const IMAGE_KEY_FILE: &str = "/etc/image_key";
let key_str = load_key(IMAGE_KEY_FILE)?;
let mut key: sgx_key_128bit_t = Default::default();
parse_str_to_bytes(&key_str, &mut key)?;
Some(key)
}
"integrity-only" => None,
_ => unreachable!(),
};
let key_ptr = key
.as_ref()
.map(|key| key as *const sgx_key_128bit_t)
.unwrap_or(std::ptr::null());
// Get certificate
let secret = CString::new("flask_cert").unwrap();
let filename = CString::new("cert_file").unwrap();
let ret = unsafe {
gr_client_get_secret(
server_addr.as_ptr(),
config_json.as_ptr(),
secret.as_ptr(),
filename.as_ptr())
};
if ret != 0 {
println!("gr_client_get_secret failed return {}", ret);
return Err(Box::new(std::io::Error::last_os_error()));
}
let cert_secret = fs::read_to_string(filename.into_string().unwrap())
.expect("Something went wrong reading the file");
// Get key
let secret = CString::new("flask_key").unwrap();
let filename = CString::new("key_file").unwrap();
let ret = unsafe {
gr_client_get_secret(
server_addr.as_ptr(),
config_json.as_ptr(),
secret.as_ptr(),
filename.as_ptr())
};
if ret != 0 {
println!("gr_client_get_secret failed return {}", ret);
return Err(Box::new(std::io::Error::last_os_error()));
}
let key_secret = fs::read_to_string(filename.into_string().unwrap())
.expect("Something went wrong reading the file");
// Mount the image
const SYS_MOUNT_FS: i64 = 363;
let ret = unsafe { syscall(SYS_MOUNT_FS, key_ptr, occlum_json_mac_ptr) };
if ret < 0 {
return Err(Box::new(std::io::Error::last_os_error()));
}
// Write the secrets to rootfs
fs::write("/etc/flask.crt", cert_secret.into_bytes())?;
fs::write("/etc/flask.key", key_secret.into_bytes())?;
Ok(())
}
#[allow(non_camel_case_types)]
type sgx_key_128bit_t = [u8; 16];
#[allow(non_camel_case_types)]
type sgx_aes_gcm_128bit_tag_t = [u8; 16];
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
struct ImageConfig {
occlum_json_mac: String,
image_type: String,
}
fn load_config(config_path: &str) -> Result<ImageConfig, Box<dyn Error>> {
let mut config_file = File::open(config_path)?;
let config_json = {
let mut config_json = String::new();
config_file.read_to_string(&mut config_json)?;
config_json
};
let config: ImageConfig = serde_json::from_str(&config_json)?;
Ok(config)
}
fn load_key(key_path: &str) -> Result<String, Box<dyn Error>> {
let mut key_file = File::open(key_path)?;
let mut key = String::new();
key_file.read_to_string(&mut key)?;
Ok(key.trim_end_matches(|c| c == '\r' || c == '\n').to_string())
}
fn parse_str_to_bytes(arg_str: &str, bytes: &mut [u8]) -> Result<(), Box<dyn Error>> {
let bytes_str_vec = {
let bytes_str_vec: Vec<&str> = arg_str.split('-').collect();
if bytes_str_vec.len() != bytes.len() {
return Err(Box::new(std::io::Error::new(
ErrorKind::InvalidData,
"The length or format of Key/MAC string is invalid",
)));
}
bytes_str_vec
};
for (byte_i, byte_str) in bytes_str_vec.iter().enumerate() {
bytes[byte_i] = u8::from_str_radix(byte_str, 16)?;
}
Ok(())
}

@ -0,0 +1,19 @@
includes:
- base.yaml
targets:
- target: /bin/
copy:
- files:
- ${INITRA_DIR}/target/x86_64-unknown-linux-musl/release/init
- target: /lib/
copy:
- files:
- ${DEP_LIBS_DIR}/libgrpc_ratls_client.so
- target: /
copy:
- files:
- dynamic_config.json
- target: /usr/share/grpc/
copy:
- files:
- ${RATLS_DIR}/grpc-src/etc/roots.pem

@ -0,0 +1,16 @@
{
"verify_mr_enclave" : "on",
"verify_mr_signer" : "on",
"verify_isv_prod_id" : "on",
"verify_isv_svn" : "on",
"verify_enclave_debuggable" : "on",
"sgx_mrs": [
{
"mr_enclave" : "",
"mr_signer" : "",
"isv_prod_id" : "0",
"isv_svn" : "0",
"debuggable" : false
}
]
}

@ -0,0 +1,16 @@
includes:
- base.yaml
targets:
- target: /bin/
copy:
- files:
- ${RATLS_DIR}/grpc-src/examples/cpp/ratls/build/server
- target: /
copy:
- files:
- dynamic_config.json
- ../secret_config.json
- target: /usr/share/grpc/
copy:
- files:
- ${RATLS_DIR}/grpc-src/etc/roots.pem

@ -0,0 +1,16 @@
#!/bin/bash
set -e
echo "Start GRPC server on backgound ..."
pushd occlum_server
occlum run /bin/server &
popd
sleep 3
echo "Start Flask-TLS restful web portal on backgound ..."
pushd occlum_client
occlum run /bin/rest_api.py &
popd