general/occlum
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add new fields for config file to enable EDMM

Browse Source
This commit is contained in:
Hui, Chunyang 2023-09-18 11:19:30 +00:00 committed by volcano
parent cb75897085
commit 28c29c8896
4 changed files with 323 additions and 84 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
src/libos/src/config.rs
View File

@ -108,7 +108,8 @@ pub struct Config {
#[derive(Debug)]
pub struct ConfigResourceLimits {
pub user_space_size: usize,
pub user_space_init_size: usize,
pub user_space_max_size: usize,
}
#[derive(Debug)]
@ -213,8 +214,12 @@ impl Config {
impl ConfigResourceLimits {
fn from_input(input: &InputConfigResourceLimits) -> Result<ConfigResourceLimits> {
let user_space_size = parse_memory_size(&input.user_space_size)?;
Ok(ConfigResourceLimits { user_space_size })
let user_space_init_size = parse_memory_size(&input.user_space_init_size)?;
let user_space_max_size = parse_memory_size(&input.user_space_max_size)?;
Ok(ConfigResourceLimits {
user_space_init_size,
user_space_max_size,
})
}
}
@ -369,12 +374,19 @@ struct InputConfig {
#[derive(Deserialize, Debug)]
#[serde(deny_unknown_fields)]
struct InputConfigResourceLimits {
#[serde(default = "InputConfigResourceLimits::get_user_space_size")]
pub user_space_size: String,
#[serde(default = "InputConfigResourceLimits::get_user_space_init_size")]
pub user_space_init_size: String,
#[serde(default = "InputConfigResourceLimits::get_user_space_max_size")]
pub user_space_max_size: String,
}
impl InputConfigResourceLimits {
fn get_user_space_size() -> String {
fn get_user_space_init_size() -> String {
"128MB".to_string()
}
// For default, just make it equal with the init size
fn get_user_space_max_size() -> String {
"128MB".to_string()
}
}
@ -382,7 +394,8 @@ impl InputConfigResourceLimits {
impl Default for InputConfigResourceLimits {
fn default() -> InputConfigResourceLimits {
InputConfigResourceLimits {
user_space_size: InputConfigResourceLimits::get_user_space_size(),
user_space_init_size: InputConfigResourceLimits::get_user_space_init_size(),
user_space_max_size: InputConfigResourceLimits::get_user_space_max_size(),
}
}
}

10
test/Occlum.json
View File

@ -1,10 +1,12 @@
{
"resource_limits": {
"kernel_space_heap_size": "40MB",
"kernel_space_heap_max_size": "512MB",
"kernel_space_stack_size": "1MB",
"user_space_size": "600MB",
"max_num_of_threads": 32
"kernel_space_heap_size": "1MB",
"kernel_space_heap_max_size": "40MB",
"user_space_size": "1MB",
"user_space_max_size": "600MB",
"init_num_of_threads": 2,
"max_num_of_threads": 64
},
"process": {
"default_stack_size": "4MB",

348
tools/gen_internal_conf/src/main.rs
View File

@ -20,28 +20,28 @@ lazy_static! {
static ref DEFAULT_CONFIG: DefaultConfig = DefaultConfig::init();
}
const MISC_FOR_EDMM_PLATFORM: (&str, &str) = ("1", "0xFFFFFFFF");
const MISC_FOR_NON_EDMM_PLATFORM: (&str, &str) = ("0", "0");
struct DefaultConfig {
// Corresponds to HeapMaxSize in Enclave.xml
kernel_heap_max_size: &'static str,
user_space_max_size: &'static str,
tcs_init_num: u32,
// Corresponds to TCSMaxNum in Enclave.xml
num_of_tcs_max: u32,
// Corresponds to MiscSelect in Enclave.xml
misc_select: &'static str,
// Corresponds to MiscMask in Enclave.xml
misc_mask: &'static str,
tcs_max_num: u32,
// Extra user region memory for SDK
extra_user_region_for_sdk: &'static str,
}
impl DefaultConfig {
fn init() -> Self {
Self {
kernel_heap_max_size: "1024MB",
num_of_tcs_max: 4096,
// If UserRegionSize is not configured, but the heap, stack and thread related
// configurations have dynamic part, set MiscSelect[0] = 1 and MiscMask[0] = 0,
// the enclave can be loaded on SGX 1.0 and 2.0 platform, and on SGX 2.0 platform,
// it can utilize the dynamic components.
misc_select: "1",
misc_mask: "0x0",
user_space_max_size: "16GB",
tcs_init_num: 16,
tcs_max_num: 4096,
extra_user_region_for_sdk: "1GB",
}
}
}
@ -49,6 +49,13 @@ impl DefaultConfig {
fn main() {
env_logger::init();
let instance_is_for_edmm_platform = {
match std::env::var("INSTANCE_IS_FOR_EDMM_PLATFORM") {
Ok(val) => val == "YES",
_ => unreachable!(),
}
};
let matches = App::new("gen_internal_conf")
.version("0.2.0")
// Input: JSON file which users may change
@ -144,10 +151,56 @@ fn main() {
enclave_config_file_path
);
// get the number of TCS
let tcs_num = occlum_config.resource_limits.max_num_of_threads;
let tcs_min_pool = tcs_num;
let tcs_max_num = std::cmp::max(tcs_num, DEFAULT_CONFIG.num_of_tcs_max);
debug!(
"Build on platform {} EDMM support",
if instance_is_for_edmm_platform {
"WITH"
} else {
"WITHOUT"
}
);
debug!(
"user config init num of threads = {:?}",
occlum_config.resource_limits.init_num_of_threads
);
// For init TCS number, try to use the values provided by users. If not provided, use the default value
let (tcs_init_num, tcs_min_pool, tcs_max_num) = {
if instance_is_for_edmm_platform {
let tcs_init_num = if let Some(ref init_num_of_threads) =
occlum_config.resource_limits.init_num_of_threads
{
*init_num_of_threads
} else {
// The user doesn't provide a value
std::cmp::min(DEFAULT_CONFIG.tcs_init_num,occlum_config.resource_limits.max_num_of_threads )
};
// For platforms with EDMM support, use the max value
let tcs_max_num = std::cmp::max(
occlum_config.resource_limits.max_num_of_threads,
DEFAULT_CONFIG.tcs_max_num,
);
(tcs_init_num, tcs_init_num, tcs_max_num)
} else {
// For platforms without EDMM support (including SIM mode), use the "max_num_of_threads" provided by user
let tcs_max_num = occlum_config.resource_limits.max_num_of_threads;
(tcs_max_num, tcs_max_num, tcs_max_num)
}
};
debug!(
"tcs init num: {}, tcs_min_pool: {}, tcs_max_num: {}",
tcs_init_num, tcs_min_pool, tcs_max_num
);
if tcs_init_num > tcs_max_num {
println!(
"init_num_of_threads: {:?}, max_num_of_threads: {:?}, wrong configuration",
occlum_config.resource_limits.init_num_of_threads, occlum_config.resource_limits.max_num_of_threads,
);
return;
}
// get the kernel stack size
let stack_max_size =
@ -160,48 +213,189 @@ fn main() {
return;
}
// get the kernel heap size
let heap_init_size =
parse_memory_size(&occlum_config.resource_limits.kernel_space_heap_size);
if heap_init_size.is_err() {
println!(
"The kernel_space_heap_size \"{}\" is not correct.",
occlum_config.resource_limits.kernel_space_heap_size
);
return;
}
// For max heap size, try to use the values provided by users. If not provided, use the default value
let heap_max_size = {
if let Some(ref kernel_space_heap_max_size) =
occlum_config.resource_limits.kernel_space_heap_max_size
{
let heap_max_size = parse_memory_size(&kernel_space_heap_max_size);
if heap_max_size.is_err() {
let (kernel_heap_init_size, kernel_heap_max_size) = {
let heap_init_size = {
let heap_init_size =
parse_memory_size(&occlum_config.resource_limits.kernel_space_heap_size);
if heap_init_size.is_err() {
println!(
"The kernel_space_heap_max_size \"{}\" is not correct.",
kernel_space_heap_max_size
"The kernel_space_heap_size \"{}\" is not correct.",
occlum_config.resource_limits.kernel_space_heap_size
);
return;
}
heap_max_size
} else {
// If the user doesn't provide a value, use the max value as heap_max_size.
std::cmp::max(
heap_init_size,
parse_memory_size(DEFAULT_CONFIG.kernel_heap_max_size),
)
heap_init_size.unwrap()
};
let optional_config_heap_max_size = {
if let Some(ref heap_max_size) =
occlum_config.resource_limits.kernel_space_heap_max_size
{
let config_kernel_heap_max_size = parse_memory_size(&heap_max_size);
if config_kernel_heap_max_size.is_err() {
println!(
"The kernel_space_heap_max_size \"{}\" is not correct.",
heap_max_size
);
return;
}
config_kernel_heap_max_size.ok()
} else {
None
}
};
debug!(
"optional_config_heap_max_size = {:?}",
optional_config_heap_max_size
);
match optional_config_heap_max_size {
Some(heap_max_size) => {
if instance_is_for_edmm_platform {
let heap_max_size = std::cmp::max(
heap_max_size,
parse_memory_size(DEFAULT_CONFIG.kernel_heap_max_size).unwrap(),
);
(heap_init_size, heap_max_size)
} else {
// User specified heap_max but no EDMM support, use user specified as the heap value
(heap_max_size, heap_max_size)
}
}
None => {
if instance_is_for_edmm_platform {
let heap_max_size = std::cmp::max(
heap_init_size,
parse_memory_size(DEFAULT_CONFIG.kernel_heap_max_size).unwrap(),
);
(heap_init_size, heap_max_size)
} else {
(heap_init_size, heap_init_size)
}
}
}
};
// get the user space size
let user_space_size = parse_memory_size(&occlum_config.resource_limits.user_space_size);
if user_space_size.is_err() {
if kernel_heap_init_size > kernel_heap_max_size {
println!(
"The user_space_size \"{}\" is not correct.",
occlum_config.resource_limits.user_space_size
"kernel_space_heap_size: {:?}, kernel_space_heap_max_size: {:?}, wrong configuration",
occlum_config.resource_limits.kernel_space_heap_size, occlum_config.resource_limits.kernel_space_heap_max_size,
);
return;
}
debug!(
"kernel heap init size = {}, kernel heap max size = {}",
kernel_heap_init_size, kernel_heap_max_size
);
assert!(kernel_heap_max_size >= kernel_heap_init_size);
let (config_user_space_init_size, config_user_space_max_size) = {
let user_space_init_size = {
let user_space_init_size =
parse_memory_size(&occlum_config.resource_limits.user_space_size);
if user_space_init_size.is_err() {
println!(
"The user_space_size \"{}\" is not correct.",
occlum_config.resource_limits.user_space_size
);
return;
}
user_space_init_size.unwrap()
};
let optional_config_user_space_max_size = {
if let Some(ref user_space_max_size) =
occlum_config.resource_limits.user_space_max_size
{
let config_user_space_max_size = parse_memory_size(&user_space_max_size);
if config_user_space_max_size.is_err() {
println!(
"The kernel_space_heap_max_size \"{}\" is not correct.",
user_space_max_size
);
return;
}
config_user_space_max_size.ok()
} else {
None
}
};
debug!(
"optional_config_user_space_max_size = {:?}",
optional_config_user_space_max_size
);
let user_space_max_size = match optional_config_user_space_max_size {
Some(user_space_max_size) => {
if instance_is_for_edmm_platform {
std::cmp::max(
user_space_max_size,
parse_memory_size(DEFAULT_CONFIG.user_space_max_size).unwrap(),
)
} else {
// Without EDMM support, just use user-provided value
user_space_max_size
}
}
None => {
if instance_is_for_edmm_platform {
std::cmp::max(
user_space_init_size,
parse_memory_size(DEFAULT_CONFIG.user_space_max_size).unwrap(),
)
} else {
user_space_init_size
}
}
};
(user_space_init_size, user_space_max_size)
};
if config_user_space_init_size > config_user_space_max_size {
println!(
"user_space_size: {:?}, user_space_max_size: {:?}, wrong configuration",
occlum_config.resource_limits.user_space_size, occlum_config.resource_limits.user_space_max_size,
);
return;
}
debug!(
"config user space init size = {},config user space max size = {}",
config_user_space_init_size, config_user_space_max_size
);
assert!(config_user_space_init_size <= config_user_space_max_size);
// Calculate the actual memory size for different regions
let (reserved_mem_size, user_region_mem_size) = {
if instance_is_for_edmm_platform {
// For platforms with EDMM support, we need extra memory for SDK usage. This might be fixed by SGX SDK in the future.
let extra_user_region = parse_memory_size(DEFAULT_CONFIG.extra_user_region_for_sdk);
if extra_user_region.is_err() {
println!("The extra_user_region_for_sdk in default config is not correct.");
return;
}
let user_region_mem_size = if config_user_space_max_size == config_user_space_init_size {
// SDK still need user region to track the EMA.
config_user_space_max_size
} else {
config_user_space_max_size + extra_user_region.unwrap()
};
(
config_user_space_init_size as u64,
Some(user_region_mem_size as u64),
)
} else {
// For platforms without EDMM support, use the max value for the user space
let reserved_mem_size = config_user_space_max_size;
(reserved_mem_size as u64, None)
}
};
debug!(
"reserved memory size = {:?}, user_region_memory size = {:?}",
reserved_mem_size, user_region_mem_size
);
#[cfg(feature = "ms_buffer")]
let marshal_buffer_size = if occlum_config.resource_limits.marshal_buffer_size.is_some() {
let marshal_buffer_size = parse_memory_size(
@ -225,16 +419,26 @@ fn main() {
let kss_tuple = parse_kss_conf(&occlum_config);
let (misc_select, misc_mask) = if instance_is_for_edmm_platform {
MISC_FOR_EDMM_PLATFORM
} else {
MISC_FOR_NON_EDMM_PLATFORM
};
debug!(
"misc_select = {:?}, misc_mask = {:?}",
misc_select, misc_mask
);
// Generate the enclave configuration
let sgx_enclave_configuration = EnclaveConfiguration {
ProdID: occlum_config.metadata.product_id,
ISVSVN: occlum_config.metadata.version_number,
StackMaxSize: stack_max_size.unwrap() as u64,
StackMinSize: stack_max_size.unwrap() as u64, // just use the same size as max size
HeapInitSize: heap_init_size.unwrap() as u64,
HeapMaxSize: heap_max_size.unwrap() as u64,
HeapMinSize: heap_init_size.unwrap() as u64,
TCSNum: tcs_num,
HeapInitSize: kernel_heap_init_size as u64,
HeapMaxSize: kernel_heap_max_size as u64,
HeapMinSize: kernel_heap_init_size as u64,
TCSNum: tcs_init_num,
TCSMinPool: tcs_min_pool,
TCSMaxNum: tcs_max_num,
TCSPolicy: 1,
@ -242,12 +446,13 @@ fn main() {
true => 0,
false => 1,
},
MiscSelect: DEFAULT_CONFIG.misc_select.to_string(),
MiscMask: DEFAULT_CONFIG.misc_mask.to_string(),
ReservedMemMaxSize: user_space_size.unwrap() as u64,
ReservedMemMinSize: user_space_size.unwrap() as u64,
ReservedMemInitSize: user_space_size.unwrap() as u64,
MiscSelect: misc_select.to_string(),
MiscMask: misc_mask.to_string(),
ReservedMemMaxSize: reserved_mem_size,
ReservedMemMinSize: reserved_mem_size,
ReservedMemInitSize: reserved_mem_size,
ReservedMemExecutable: 1,
UserRegionSize: user_region_mem_size,
#[cfg(feature = "ms_buffer")]
MarshalBufferSize: marshal_buffer_size as u64,
EnableKSS: kss_tuple.0,
@ -262,13 +467,12 @@ fn main() {
// Generate app config, including "init" and user app
let app_config = {
let app_config =
gen_app_config(
occlum_config.entry_points,
occlum_config.mount,
occlum_conf_user_fs_mac.to_string(),
occlum_conf_init_fs_mac.to_string(),
);
let app_config = gen_app_config(
occlum_config.entry_points,
occlum_config.mount,
occlum_conf_user_fs_mac.to_string(),
occlum_conf_init_fs_mac.to_string(),
);
if app_config.is_err() {
println!("Mount configuration invalid: {:?}", app_config);
return;
@ -278,7 +482,8 @@ fn main() {
let occlum_json_config = InternalOcclumJson {
resource_limits: InternalResourceLimits {
user_space_size: occlum_config.resource_limits.user_space_size.to_string(),
user_space_init_size: config_user_space_init_size.to_string() + "B",
user_space_max_size: config_user_space_max_size.to_string() + "B",
},
process: OcclumProcess {
default_stack_size: occlum_config.process.default_stack_size,
@ -429,9 +634,7 @@ fn gen_app_config(
.unwrap() = serde_json::Value::String(occlum_conf_init_fs_mac);
// Update app entry points
*app_config
.pointer_mut("/app/1/entry_points")
.unwrap() = entry_points;
*app_config.pointer_mut("/app/1/entry_points").unwrap() = entry_points;
debug!("User provided root mount config: {:?}", mount_conf);
let mut root_mount_config = mount_conf;
@ -484,12 +687,16 @@ struct OcclumConfiguration {
#[derive(Debug, PartialEq, Deserialize)]
struct OcclumResourceLimits {
#[serde(default)]
init_num_of_threads: Option<u32>,
max_num_of_threads: u32,
kernel_space_heap_size: String,
#[serde(default)]
kernel_space_heap_max_size: Option<String>,
kernel_space_stack_size: String,
user_space_size: String,
#[serde(default)]
user_space_max_size: Option<String>,
#[cfg(feature = "ms_buffer")]
marshal_buffer_size: Option<String>,
}
@ -575,6 +782,8 @@ struct EnclaveConfiguration {
ReservedMemMinSize: u64,
ReservedMemInitSize: u64,
ReservedMemExecutable: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
UserRegionSize: Option<u64>,
#[cfg(feature = "ms_buffer")]
MarshalBufferSize: u64,
EnableKSS: u32,
@ -587,7 +796,8 @@ struct EnclaveConfiguration {
#[derive(Debug, PartialEq, Clone, Serialize)]
struct InternalResourceLimits {
user_space_size: String,
user_space_init_size: String,
user_space_max_size: String,
}
#[derive(Debug, PartialEq, Clone, Serialize)]

22
tools/occlum
View File

@ -64,7 +64,7 @@ Usage:
Initialize a directory as the Occlum instance.
If flag --init-ra specifies, generate initfs with RA KMS client function.
occlum build [--sign-key <key_path>] [--sign-tool <tool_path>] [--image-key <key_path>] [--buildin-image-key] [-f/--force]
occlum build [--sign-key <key_path>] [--sign-tool <tool_path>] [--image-key <key_path>] [--buildin-image-key] [-f/--force] [--enable-edmm <Y/N>]
Build and sign an Occlum SGX enclave (.so) and generate its associated secure
FS image according to the user-provided image directory and Occlum.json config file.
The whole building process is incremental: the building artifacts are built only
@ -375,7 +375,6 @@ cmd_init() {
cmd_build() {
check_has_init
pal_lib=libocclum-pal.so
libos_lib=libocclum-libos.so
BUILDIN_IMAGE_KEY=false
@ -385,10 +384,11 @@ cmd_build() {
--sign-key) [ -n "$2" ] && ENCLAVE_SIGN_KEY=$2 ; shift 2 || exit_error "empty signing key path" ;;
--sign-tool) [ -n "$2" ] && ENCLAVE_SIGN_TOOL=$2 ; shift 2 || exit_error "empty signing tool path" ;;
--sgx-mode) [[ -n "$2" && "$2" != "HW" ]] && export SGX_MODE=$2 ; shift 2 || exit_error "empty sgx mode";;
--enable-edmm) [[ -n "$2" && "$2" != "N" ]] && export ENABLE_EDMM=Y ; shift 2 || exit_error "empty edmm mode";;
--image-key) [ -n "$2" ] && SECURE_IMAGE_KEY=$2 ; shift 2 || exit_error "empty secure image key path" ;;
--buildin-image-key) BUILDIN_IMAGE_KEY=true ; shift ;;
--force | -f) MAKE_OPTION="clean" ; shift ;;
*) exit_error "Unknown option: $1" ;;
*) exit_error "Unknown option: $1" ;;
esac
done
[ -e "$ENCLAVE_SIGN_KEY" ] || exit_error "invalid signing key path: $ENCLAVE_SIGN_KEY"
@ -410,8 +410,22 @@ cmd_build() {
libos_lib=libocclum-libos_hyper.so
fi
echo "SGX mode: $SGX_MODE"
echo "Enable EDMM: No"
INSTANCE_IS_FOR_EDMM_PLATFORM="NO"
else
echo "SGX mode: HW"
# ENABLE_EDMM is only valid for HW mode and is not enabled by default
case "$ENABLE_EDMM" in
"Y" | "YES" | "Yes" | "yes" | "True" | "true" | "1")
INSTANCE_IS_FOR_EDMM_PLATFORM="YES"
echo "Enable EDMM: Yes"
;;
*)
INSTANCE_IS_FOR_EDMM_PLATFORM="NO"
echo "Enable EDMM: No"
;;
esac
fi
if [[ -f "$instance_dir/init_ra_conf.json" ]]; then
@ -436,7 +450,7 @@ cmd_build() {
make -f $build_makefile $MAKE_OPTION
fi
occlum_dir=$occlum_dir instance_dir=$instance_dir pal_lib=$pal_lib major_ver=$major_ver \
occlum_version=$occlum_version libos_lib=$libos_lib \
occlum_version=$occlum_version libos_lib=$libos_lib INSTANCE_IS_FOR_EDMM_PLATFORM=$INSTANCE_IS_FOR_EDMM_PLATFORM \
ENCLAVE_SIGN_KEY=$ENCLAVE_SIGN_KEY ENCLAVE_SIGN_TOOL=$ENCLAVE_SIGN_TOOL \
SECURE_IMAGE_KEY=$SECURE_IMAGE_KEY BUILDIN_IMAGE_KEY=$BUILDIN_IMAGE_KEY \
make -f $build_makefile --no-builtin-rules