From b43cd4233a248f242f536c970f9ad1009eddc968 Mon Sep 17 00:00:00 2001 From: "Zheng, Qi" Date: Thu, 29 Jun 2023 16:19:34 +0800 Subject: [PATCH] [example] Update example with init-ra way --- example/README.md | 8 +- example/build_content.sh | 63 ++++---- .../Dockerfile_occlum_instance.ubuntu20.04 | 2 +- example/init_ra/Cargo.lock | 94 ----------- example/init_ra/Cargo.toml | 11 -- example/init_ra/build.rs | 7 - example/init_ra/src/main.rs | 151 ------------------ example/init_ra_client.yaml | 21 --- example/kubernetes/README.md | 16 +- example/kubernetes/occlum_tf_demo.yaml | 4 +- example/run_container.sh | 2 +- 11 files changed, 42 insertions(+), 337 deletions(-) delete mode 100644 example/init_ra/Cargo.lock delete mode 100644 example/init_ra/Cargo.toml delete mode 100644 example/init_ra/build.rs delete mode 100644 example/init_ra/src/main.rs delete mode 100644 example/init_ra_client.yaml diff --git a/example/README.md b/example/README.md index 40427903..93adc80e 100644 --- a/example/README.md +++ b/example/README.md @@ -37,7 +37,7 @@ It is used to encrypt/decrypt the Occlum application RootFS image which is Tenso #### Step 2 -Application starts. First it starts the `init` process. This customized [`init`](./init_ra/) requests `ssl_config` and `image_key` from GRPC-RATLS server through a secure GRPC RATLS connection. Then it uses the `image_key` to decrypt the RootFS where the real application is located, mount the RootFS, save the `ssl_config` to RootFS `/etc/tf_ssl.cfg`. +Application starts. First it starts the `init` process. This customized [`init`](../tools/init_grpc_ratls/) requests `ssl_config` and `image_key` from GRPC-RATLS server through a secure GRPC RATLS connection. Then it uses the `image_key` to decrypt the RootFS where the real application is located, mount the RootFS, save the `ssl_config` to RootFS `/etc/tf_ssl.cfg`. Detail description of the above two steps Init-RA operation could refer to [`Init-RA`](../demos/remote_attestation/init_ra_flow/). @@ -55,9 +55,9 @@ Now users could send inference request with server certificates (`server.crt`). There are prebuilt docker images could be used for the examples, either in the following docker way or [`kubernates`](./kubernetes/) way. Users could pull them directly and try the example. ``` -docker pull occlum/init_ra_server:0.29.5-ubuntu20.04 -docker pull occlum/tf_demo:0.29.5-ubuntu20.04 -docker pull occlum/tf_demo_client:0.29.5-ubuntu20.04 +docker pull occlum/init_ra_server:0.29.6-ubuntu20.04 +docker pull occlum/tf_demo:0.29.6-ubuntu20.04 +docker pull occlum/tf_demo_client:0.29.6-ubuntu20.04 ``` If users want to build or customize the images, please check below part. diff --git a/example/build_content.sh b/example/build_content.sh index bba80f8f..6d7952e3 100755 --- a/example/build_content.sh +++ b/example/build_content.sh @@ -2,17 +2,8 @@ set -e script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -export INITRA_DIR="${script_dir}/init_ra" -export RATLS_DIR="${script_dir}/../tools/toolchains/grpc_ratls" export TF_DIR="${script_dir}/tf_serving" -function build_ratls() -{ - pushd ${RATLS_DIR} - ./build.sh - popd -} function build_tf_serving() { @@ -27,12 +18,22 @@ function build_tf_serving() popd } -function build_init_ra() +function update_client_init_ra_conf() { - pushd ${INITRA_DIR} - occlum-cargo clean - occlum-cargo build --release - popd + # Fill in the keys + new_json="$(jq '.kms_keys = [ {"key": "ssl_config", "path": "/etc/tf_ssl.cfg"}]' init_ra_conf.json)" && \ + echo "${new_json}" > init_ra_conf.json + + # Fill in the KMS server measurements. + new_json="$(jq ' .ra_config.verify_mr_enclave = "off" | + .ra_config.verify_mr_signer = "on" | + .ra_config.verify_isv_prod_id = "off" | + .ra_config.verify_isv_svn = "off" | + .ra_config.verify_config_svn = "off" | + .ra_config.verify_enclave_debuggable = "on" | + .ra_config.sgx_mrs[0].mr_signer = ''"'`get_mr tf mrsigner`'" | + .ra_config.sgx_mrs[0].debuggable = false ' init_ra_conf.json)" && \ + echo "${new_json}" > init_ra_conf.json } function build_tf_instance() @@ -40,40 +41,32 @@ function build_tf_instance() # generate tf image key occlum gen-image-key image_key - rm -rf occlum_tf && occlum new occlum_tf + rm -rf occlum_tf + # choose grpc_ratls as init ra kms client + occlum new occlum_tf --init-ra grpc_ratls pushd occlum_tf # prepare tf_serving content rm -rf image copy_bom -f ../tf_serving.yaml --root image --include-dir /opt/occlum/etc/template + # Try build first to get mrsigner + # In our case, client and server use the same sign-key thus also the same mrsigner + occlum build + new_json="$(jq '.resource_limits.user_space_size = "7000MB" | .resource_limits.kernel_space_heap_size="384MB" | .process.default_heap_size = "128MB" | .resource_limits.max_num_of_threads = 64 | .metadata.debuggable = false | - .env.default += ["GRPC_SERVER=localhost:50051"] | - .env.untrusted += ["GRPC_SERVER"]' Occlum.json)" && \ + .env.default += ["OCCLUM_INIT_RA_KMS_SERVER=localhost:50051"] | + .env.untrusted += ["OCCLUM_INIT_RA_KMS_SERVER"]' Occlum.json)" && \ echo "${new_json}" > Occlum.json + # Update init_ra_conf json file accordingly before occlum build + update_client_init_ra_conf + occlum build --image-key ../image_key - - # Get server mrsigner. - # Here client and server use the same signer-key thus using client mrsigner directly. - jq ' .verify_mr_enclave = "off" | - .verify_mr_signer = "on" | - .verify_isv_prod_id = "off" | - .verify_isv_svn = "off" | - .verify_config_svn = "off" | - .verify_enclave_debuggable = "on" | - .sgx_mrs[0].mr_signer = ''"'`get_mr tf mrsigner`'" | - .sgx_mrs[0].debuggable = false ' ../ra_config_template.json > dynamic_config.json - - # prepare init-ra content - rm -rf initfs - copy_bom -f ../init_ra_client.yaml --root initfs --include-dir /opt/occlum/etc/template - - occlum build -f --image-key ../image_key occlum package occlum_instance popd @@ -125,9 +118,7 @@ function build_server_instance() popd } -build_ratls build_tf_serving -build_init_ra build_tf_instance build_server_instance diff --git a/example/container/Dockerfile_occlum_instance.ubuntu20.04 b/example/container/Dockerfile_occlum_instance.ubuntu20.04 index cd6a6aca..cc84dd54 100644 --- a/example/container/Dockerfile_occlum_instance.ubuntu20.04 +++ b/example/container/Dockerfile_occlum_instance.ubuntu20.04 @@ -5,7 +5,7 @@ LABEL maintainer="Qi Zheng " ENV APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=1 ARG PSW_VERSION=2.17.100.3 ARG DCAP_VERSION=1.14.100.3 -ARG OCCLUM_VERSION=0.29.5-1 +ARG OCCLUM_VERSION=0.29.6-1 RUN apt update && DEBIAN_FRONTEND="noninteractive" apt install -y --no-install-recommends gnupg wget ca-certificates jq && \ echo 'deb [arch=amd64] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main' | tee /etc/apt/sources.list.d/intel-sgx.list && \ wget -qO - https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key | apt-key add - && \ diff --git a/example/init_ra/Cargo.lock b/example/init_ra/Cargo.lock deleted file mode 100644 index 95122653..00000000 --- a/example/init_ra/Cargo.lock +++ /dev/null @@ -1,94 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -[[package]] -name = "init" -version = "0.0.1" -dependencies = [ - "libc", - "serde", - "serde_json", -] - -[[package]] -name = "itoa" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" - -[[package]] -name = "libc" -version = "0.2.84" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cca32fa0182e8c0989459524dc356b8f2b5c10f1b9eb521b7d182c03cf8c5ff" - -[[package]] -name = "proc-macro2" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" -dependencies = [ - "unicode-xid", -] - -[[package]] -name = "quote" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "ryu" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" - -[[package]] -name = "serde" -version = "1.0.123" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.123" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.62" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486" -dependencies = [ - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "syn" -version = "1.0.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" -dependencies = [ - "proc-macro2", - "quote", - "unicode-xid", -] - -[[package]] -name = "unicode-xid" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" diff --git a/example/init_ra/Cargo.toml b/example/init_ra/Cargo.toml deleted file mode 100644 index 8cc375ec..00000000 --- a/example/init_ra/Cargo.toml +++ /dev/null @@ -1,11 +0,0 @@ -[package] -name = "init" -version = "0.0.1" -build = "build.rs" -authors = ["LI Qing geding.lq@antgroup.com"] -edition = "2021" - -[dependencies] -libc = "0.2.84" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" diff --git a/example/init_ra/build.rs b/example/init_ra/build.rs deleted file mode 100644 index f3d489f5..00000000 --- a/example/init_ra/build.rs +++ /dev/null @@ -1,7 +0,0 @@ -fn main() { - println!("cargo:rustc-link-search=native=/opt/occlum/toolchains/dcap_lib/musl"); - println!("cargo:rustc-link-search=native=/opt/occlum/toolchains/grpc_ratls/musl"); - println!("cargo:rustc-link-lib=dylib=grpc_ratls_client"); - println!("cargo:rustc-link-lib=dylib=hw_grpc_proto"); - println!("cargo:rustc-link-lib=dylib=occlum_dcap") -} \ No newline at end of file diff --git a/example/init_ra/src/main.rs b/example/init_ra/src/main.rs deleted file mode 100644 index 1e609879..00000000 --- a/example/init_ra/src/main.rs +++ /dev/null @@ -1,151 +0,0 @@ -extern crate libc; -extern crate serde; -extern crate serde_json; - -use libc::syscall; -use serde::Deserialize; - -use std::env; -use std::error::Error; -use std::fs; -use std::fs::File; -use std::io::{ErrorKind, Read}; - -use std::ffi::CString; -use std::os::raw::{c_int, c_char}; - -#[link(name = "grpc_ratls_client")] -extern "C" { - fn grpc_ratls_get_secret( - server_addr: *const c_char, // grpc server address+port, such as "localhost:50051" - config_json: *const c_char, // ratls handshake config json file - name: *const c_char, // secret name to be requested - secret_file: *const c_char // secret file to be saved - ) -> c_int; -} - -fn main() -> Result<(), Box> { - // Load the configuration from initfs - const IMAGE_CONFIG_FILE: &str = "/etc/image_config.json"; - let image_config = load_config(IMAGE_CONFIG_FILE)?; - - // Get grpc server address from environment GRPC_SERVER - let server_addr = CString::new( - env::var("GRPC_SERVER").unwrap_or("localhost:50051".to_string())) - .unwrap(); - let config_json = CString::new("dynamic_config.json").unwrap(); - - // Get the key of FS image if needed - let key = match &image_config.image_type[..] { - "encrypted" => { - // Get the image encrypted key through RA - let secret = CString::new("image_key").unwrap(); - let filename = CString::new("/etc/image_key").unwrap(); - - let ret = unsafe { - grpc_ratls_get_secret( - server_addr.as_ptr(), - config_json.as_ptr(), - secret.as_ptr(), - filename.as_ptr()) - }; - - if ret != 0 { - println!("grpc_ratls_get_secret failed return {}", ret); - return Err(Box::new(std::io::Error::last_os_error())); - } - - const IMAGE_KEY_FILE: &str = "/etc/image_key"; - let key_str = load_key(IMAGE_KEY_FILE)?; - let mut key: sgx_key_128bit_t = Default::default(); - parse_str_to_bytes(&key_str, &mut key)?; - Some(key) - } - "integrity-only" => None, - _ => unreachable!(), - }; - let key_ptr = key - .as_ref() - .map(|key| key as *const sgx_key_128bit_t) - .unwrap_or(std::ptr::null()); - - // Get certificate - let secret = CString::new("ssl_config").unwrap(); - let filename = CString::new("ssl_file").unwrap(); - - let ret = unsafe { - grpc_ratls_get_secret( - server_addr.as_ptr(), - config_json.as_ptr(), - secret.as_ptr(), - filename.as_ptr()) - }; - - if ret != 0 { - println!("grpc_ratls_get_secret failed return {}", ret); - return Err(Box::new(std::io::Error::last_os_error())); - } - - let ssl_secret = fs::read_to_string(filename.into_string().unwrap()) - .expect("Something went wrong reading the file"); - - // Mount the image - const SYS_MOUNT_FS: i64 = 363; - // User can provide valid path for runtime mount and boot - // Otherwise, just pass null pointer to do general mount and boot - let rootfs_config: *const i8 = std::ptr::null(); - let ret = unsafe { syscall(SYS_MOUNT_FS, key_ptr, rootfs_config) }; - if ret < 0 { - return Err(Box::new(std::io::Error::last_os_error())); - } - - // Write the secrets to rootfs - fs::write("/etc/tf_ssl.cfg", ssl_secret.into_bytes())?; - - Ok(()) -} - -#[allow(non_camel_case_types)] -type sgx_key_128bit_t = [u8; 16]; - -#[derive(Deserialize, Debug)] -#[serde(deny_unknown_fields)] -struct ImageConfig { - image_type: String, -} - -fn load_config(config_path: &str) -> Result> { - let mut config_file = File::open(config_path)?; - let config_json = { - let mut config_json = String::new(); - config_file.read_to_string(&mut config_json)?; - config_json - }; - let config: ImageConfig = serde_json::from_str(&config_json)?; - Ok(config) -} - -fn load_key(key_path: &str) -> Result> { - let mut key_file = File::open(key_path)?; - let mut key = String::new(); - key_file.read_to_string(&mut key)?; - Ok(key.trim_end_matches(|c| c == '\r' || c == '\n').to_string()) -} - -fn parse_str_to_bytes(arg_str: &str, bytes: &mut [u8]) -> Result<(), Box> { - let bytes_str_vec = { - let bytes_str_vec: Vec<&str> = arg_str.split('-').collect(); - if bytes_str_vec.len() != bytes.len() { - return Err(Box::new(std::io::Error::new( - ErrorKind::InvalidData, - "The length or format of Key/MAC string is invalid", - ))); - } - bytes_str_vec - }; - - for (byte_i, byte_str) in bytes_str_vec.iter().enumerate() { - bytes[byte_i] = u8::from_str_radix(byte_str, 16)?; - } - Ok(()) -} diff --git a/example/init_ra_client.yaml b/example/init_ra_client.yaml deleted file mode 100644 index 47fe9553..00000000 --- a/example/init_ra_client.yaml +++ /dev/null @@ -1,21 +0,0 @@ -includes: - - base.yaml -targets: - - target: /bin/ - copy: - - files: - - ${INITRA_DIR}/target/x86_64-unknown-linux-musl/release/init - - target: /lib/ - copy: - - files: - - /opt/occlum/toolchains/grpc_ratls/musl/libgrpc_ratls_client.so - - /opt/occlum/toolchains/grpc_ratls/musl/libhw_grpc_proto.so - - /opt/occlum/toolchains/dcap_lib/musl/libocclum_dcap.so.0.1.0 - - target: / - copy: - - files: - - dynamic_config.json - - target: /etc - copy: - - dirs: - - /etc/ssl diff --git a/example/kubernetes/README.md b/example/kubernetes/README.md index a9c73a20..7bd302f5 100644 --- a/example/kubernetes/README.md +++ b/example/kubernetes/README.md @@ -50,18 +50,16 @@ Build Occlum TF examples container images for k8s deployment. usage: build.sh [OPTION]... -r the container image registry -g container image tag - -d GPRC RA server domain - -p GPRC RA server port ``` For example, below command generates three container images. ``` -# ./build.sh -r demo -g 0.29.5 +# ./build.sh -r demo -g 0.29.6 ``` -* **`demo/init_ra_server:0.29.5`** acts as key broker pod. -* **`demo/tf_demo:0.29.5`** acts as tensorflow serving pod. -* **`demo/tf_demo_client:0.29.5`** acts as client. +* **`demo/init_ra_server:0.29.6`** acts as key broker pod. +* **`demo/tf_demo:0.29.6`** acts as tensorflow serving pod. +* **`demo/tf_demo_client:0.29.6`** acts as client. ## How to test @@ -80,7 +78,7 @@ For example, below command generates three container images. ``` In this case, for inference, "8000MB" SGX EPC memory size is used because Occlum `user_space_size` is set to "7000MB" in building stage. cpu limits "1000m" here is to limit the CPU usage for each Occlum inference pod with the purpose to show the performance gain by scalability in `benchmark`. -* Args `"taskset -c 2,3,4,5"` is necessary till Occlum v1.0 release. The purpose is to limit the CPU cores used in tensorflow serving which makes the SGX thread number used won't exceed the `max_num_of_threads` defined in building stage. +* Args `"taskset -c 0-3"` is necessary till Occlum v1.0 release. The purpose is to limit the CPU cores used in tensorflow serving which makes the SGX thread number used won't exceed the `max_num_of_threads` defined in building stage. ### Start the key broker service @@ -110,7 +108,7 @@ In default, only one replica for the tensorflow serving pod. ### Try the inference request ``` -$ docker run --rm --network host demo/tf_demo_client:0.29.5 python3 resnet_client_grpc.py --server=localhost:31001 --crt server.crt --image cat.jpg +$ docker run --rm --network host demo/tf_demo_client:0.29.6 python3 resnet_client_grpc.py --server=localhost:31001 --crt server.crt --image cat.jpg ``` If successful, it prints the classification results. @@ -120,7 +118,7 @@ If successful, it prints the classification results. Below command can do benchmark test for the tensorflow serving service running in Occlum. ``` -$ docker run --rm --network host demo/tf_demo_client:0.29.5 python3 benchmark.py --server localhost:31001 --crt server.crt --cnum 4 --loop 10 --image cat.jpg +$ docker run --rm --network host demo/tf_demo_client:0.29.6 python3 benchmark.py --server localhost:31001 --crt server.crt --cnum 4 --loop 10 --image cat.jpg ``` Try scale up the tensorflow serving pods number, better `tps` can be achieved. diff --git a/example/kubernetes/occlum_tf_demo.yaml b/example/kubernetes/occlum_tf_demo.yaml index 4997eda2..00ae0885 100644 --- a/example/kubernetes/occlum_tf_demo.yaml +++ b/example/kubernetes/occlum_tf_demo.yaml @@ -20,12 +20,12 @@ spec: env: - name: PCCS_URL value: https://sgx-dcap-server.cn-shanghai.aliyuncs.com/sgx/certification/v3/ - - name: GRPC_SERVER + - name: OCCLUM_INIT_RA_KMS_SERVER value: init-ra-server-svc:5000 args: - taskset - -c - - 2,3,4,5 + - 0-3 - occlum - run - /bin/tensorflow_model_server diff --git a/example/run_container.sh b/example/run_container.sh index d62662b2..98996f92 100755 --- a/example/run_container.sh +++ b/example/run_container.sh @@ -54,7 +54,7 @@ GRPC_SERVER="${grpc_domain}:${grpc_port}" docker run --network host \ --device /dev/sgx/enclave --device /dev/sgx/provision \ --env PCCS_URL=${pccs_url} \ - --env GRPC_SERVER="${GRPC_SERVER}" \ + --env OCCLUM_INIT_RA_KMS_SERVER="${GRPC_SERVER}" \ ${registry}/tf_demo:${tag} \ taskset -c 0,1 occlum run /bin/tensorflow_model_server \ --model_name=resnet --model_base_path=/models/resnet \