From eb8cac48f2a8d9f8cc2cfcc8773d4d57b92e87f8 Mon Sep 17 00:00:00 2001 From: Noor Date: Sat, 28 Jun 2025 00:41:54 +0530 Subject: [PATCH] Wip on app resource ratio and sloat system complete refactor of app deployment disabled app deployment from yaml --- Cargo.lock | 12 +++- Cargo.toml | 1 + src/bin/detee-cli.rs | 5 +- src/sgx/cli_handler.rs | 107 ++++++++++------------------------ src/sgx/deploy.rs | 127 +++++++++++++++++++++++++++++++++++++++++ src/sgx/grpc_brain.rs | 20 +------ src/sgx/grpc_dtpm.rs | 29 ++++++---- src/sgx/mod.rs | 33 ++++------- src/sgx/utils.rs | 69 ++-------------------- 9 files changed, 207 insertions(+), 196 deletions(-) create mode 100644 src/sgx/deploy.rs diff --git a/Cargo.lock b/Cargo.lock index 16824cc..202837c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1132,6 +1132,7 @@ dependencies = [ "reqwest", "rustls", "serde", + "serde_default_utils", "serde_json", "serde_yaml", "shadow-rs", @@ -3062,7 +3063,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.9.4", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3239,6 +3240,15 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde_default_utils" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61460b1489ce48857e7eee87aa4fde5cbe4e9efc29b6f07a35df9ec82379b74b" +dependencies = [ + "paste", +] + [[package]] name = "serde_derive" version = "1.0.216" diff --git a/Cargo.toml b/Cargo.toml index bc1f895..17cbd36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ openssl = { version = "0.10.71", features = ["vendored"] } tokio-retry = "0.3.0" detee-sgx = { git = "ssh://git@gitea.detee.cloud/testnet/detee-sgx.git", branch = "hratls", features=["hratls", "qvl"] } shadow-rs = { version = "1.1.1", features = ["metadata"] } +serde_default_utils = "0.3.1" detee-shared = { git = "ssh://git@gitea.detee.cloud/testnet/proto.git", branch = "credits_app" } # detee-shared = { path = "../detee-shared" } diff --git a/src/bin/detee-cli.rs b/src/bin/detee-cli.rs index 796de02..20cd773 100644 --- a/src/bin/detee-cli.rs +++ b/src/bin/detee-cli.rs @@ -115,6 +115,7 @@ fn clap_cmd() -> Command { .subcommand( Command::new("deploy") .about("create new app from a YAML configuration file") + /* .arg( Arg::new("yaml-path") .long("from-yaml") @@ -123,6 +124,7 @@ fn clap_cmd() -> Command { "\n- deploying to a specific node or to a specific city.") .exclusive(true) ) + */ .arg( Arg::new("vcpus") .long("vcpus") @@ -184,9 +186,10 @@ fn clap_cmd() -> Command { Arg::new("location") .help("deploy to a specific location") .long("location") - .default_value("DE") + .default_value("Any") .value_parser([ PossibleValue::new("DE").help("Frankfurt am Main, Hesse, Germany"), + PossibleValue::new("Any").help("List offers for any location."), ]), ) .arg( diff --git a/src/sgx/cli_handler.rs b/src/sgx/cli_handler.rs index 0ee6ec4..36a6436 100644 --- a/src/sgx/cli_handler.rs +++ b/src/sgx/cli_handler.rs @@ -2,23 +2,19 @@ use crate::config::Config; use crate::name_generator::random_app_name; -use crate::sgx::config::{validate_yaml, DeteeCliExt}; +use crate::sgx::config::validate_yaml; +use crate::sgx::deploy::Reqwest; use crate::sgx::grpc_brain::{delete_app, list_contracts}; use crate::sgx::grpc_dtpm::{get_config, update_config}; use crate::sgx::packaging::package_enclave; -use crate::sgx::utils::{ - deploy_new_app_and_update_config, fetch_config, override_envs_and_args_launch_config, -}; use crate::sgx::{ - append_uuid_list, get_app_node, get_app_node_by_contract, get_one_contract, inspect_node, - package_entry_from_name, print_nodes, write_uuid_list, AppContract, AppDeleteResponse, - AppDeployResponse, + get_app_node_by_contract, get_one_contract, inspect_node, print_nodes, write_uuid_list, + AppContract, AppDeleteResponse, AppDeployResponse, }; use crate::utils::block_on; use crate::{cli_print, SimpleOutput}; use clap::ArgMatches; use detee_shared::app_proto::ListAppContractsReq; -use detee_shared::sgx::types::brain::{AppDeployConfig, Resource}; pub fn handle_app(app_matche: &ArgMatches) { match app_matche.subcommand() { @@ -74,78 +70,37 @@ fn handle_package(package_match: &ArgMatches) -> Result Result> { - let (mut app_deploy_config, app_launch_config) = if let Some(file_path) = - deploy_match.get_one::("yaml-path") - { - // TODO: maybe add launch config on deploy command with --launch-config flag - (AppDeployConfig::from_path(file_path).unwrap(), None) - } else { - let vcpus = *deploy_match.get_one::("vcpus").unwrap(); - let memory_mib = *deploy_match.get_one::("memory").unwrap(); - let disk_size_mib = *deploy_match.get_one::("disk").unwrap() * 1024; - let port = - deploy_match.get_many::("port").unwrap_or_default().cloned().collect::>(); - let package_name = deploy_match.get_one::("package").unwrap().clone(); - let hours = *deploy_match.get_one::("hours").unwrap(); - let node_unit_price = *deploy_match.get_one::("price").unwrap(); - let location = deploy_match.get_one::("location").unwrap().as_str(); - let app_name = - deploy_match.get_one::("name").cloned().unwrap_or_else(random_app_name); - let envs = - deploy_match.get_many::("env").unwrap_or_default().cloned().collect::>(); - let args = - deploy_match.get_many::("arg").unwrap_or_default().cloned().collect::>(); + let vcpus = *deploy_match.get_one::("vcpus").unwrap(); + let memory_mib = *deploy_match.get_one::("memory").unwrap(); + let disk_size_mib = *deploy_match.get_one::("disk").unwrap() * 1024; + let port = + deploy_match.get_many::("port").unwrap_or_default().cloned().collect::>(); + let package_name = deploy_match.get_one::("package").unwrap().clone(); + let hours = *deploy_match.get_one::("hours").unwrap(); + let price = *deploy_match.get_one::("price").unwrap(); + let location = deploy_match.get_one::("location").unwrap().clone(); + let app_name = deploy_match.get_one::("name").cloned().unwrap_or_else(random_app_name); + let envs = + deploy_match.get_many::("env").unwrap_or_default().cloned().collect::>(); + let args = + deploy_match.get_many::("arg").unwrap_or_default().cloned().collect::>(); - let private_package = false; + let app_deploy_config = Reqwest { + app_name, + package_name, + vcpus, + memory_mib, + disk_size_mib, + port, + hours, + location, + price, - let resource = Resource { vcpus, memory_mib, disk_size_mib, port }; - let node_pubkey = match block_on(get_app_node(resource.clone(), location.into())) { - Ok(node) => node.node_pubkey, - Err(e) => { - return Err(Box::new(std::io::Error::other( - format!("Could not get node pubkey due to error: {:?}", e).as_str(), - ))); - } - }; - - let package_entry = package_entry_from_name(&package_name).unwrap(); - let package_url = package_entry.package_url.clone(); - let public_package_mr_enclave = Some(package_entry.mr_enclave.to_vec()); - - let config = block_on(fetch_config(&package_name))?; - - let launch_config = override_envs_and_args_launch_config(config, envs, args); - - ( - AppDeployConfig { - package_url, - resource, - node_unit_price, - hours, - node_pubkey, - private_package, - app_name, - public_package_mr_enclave, - ..Default::default() - }, - Some(launch_config), - ) + envs, + args, }; - if app_deploy_config.app_name.is_empty() { - app_deploy_config.app_name = random_app_name(); - } - - let app_name = app_deploy_config.app_name.clone(); - - match block_on(deploy_new_app_and_update_config(app_deploy_config, app_launch_config)) { - Ok(new_app_res) if new_app_res.error.is_empty() => { - append_uuid_list(&new_app_res.uuid, &app_name)?; - Ok((new_app_res, app_name).into()) - } - Ok(new_app_res) => Err(Box::new(std::io::Error::other(new_app_res.error))), - Err(e) => Err(Box::new(e)), - } + Ok(block_on(app_deploy_config.deploy())?) } fn handle_inspect(inspect_match: &ArgMatches) { diff --git a/src/sgx/deploy.rs b/src/sgx/deploy.rs new file mode 100644 index 0000000..670ed9a --- /dev/null +++ b/src/sgx/deploy.rs @@ -0,0 +1,127 @@ +use crate::{ + config::Config, + name_generator::random_app_name, + sgx::{ + append_uuid_list, + grpc_brain::{get_one_app_node, new_app}, + grpc_dtpm::{dtpm_client, set_config_pb, upload_files_pb}, + package_entry_from_name, + utils::{calculate_nanolp_for_app, fetch_config, hratls_url_and_mr_enclave_from_app_id}, + AppDeployResponse, Error, PackageElement, + }, + snp, +}; +use detee_shared::{ + app_proto::{AppNodeFilters, AppNodeListResp, AppResource, NewAppReq}, + sgx::pb::dtpm_proto::DtpmSetConfigReq, +}; +use serde::{Deserialize, Serialize}; +use serde_default_utils::*; +use tokio_retry::{strategy::FixedInterval, Retry}; + +#[derive(Debug, Clone, Serialize, Deserialize, Default)] +pub struct Reqwest { + #[serde(default = "random_app_name")] + pub app_name: String, + pub package_name: String, + pub vcpus: u32, + pub memory_mib: u32, + pub disk_size_mib: u32, + pub port: Vec, + #[serde(default = "default_u64::<1>")] + pub hours: u64, + #[serde(default)] + pub price: u64, + #[serde(default)] + pub location: String, + + pub envs: Vec, + pub args: Vec, +} + +impl Reqwest { + pub async fn deploy(self) -> Result { + let node = self.get_app_node().await?; + let app_name = self.app_name.clone(); + + let locked_nano = calculate_nanolp_for_app( + self.vcpus, + self.memory_mib, + self.disk_size_mib, + self.hours, + self.price, + ); + + let PackageElement { package_url, mr_enclave, launch_config_url, .. } = + package_entry_from_name(&self.package_name).expect("Unknown package name"); + + let resource = Some(AppResource { + vcpus: self.vcpus, + memory_mib: self.memory_mib, + disk_size_mib: self.disk_size_mib, + ports: self.port.clone(), + }); + + let req = NewAppReq { + package_url: package_url, + node_pubkey: node.node_pubkey, + resource, + uuid: "".to_string(), + admin_pubkey: Config::get_detee_wallet()?, + price_per_unit: self.price, + locked_nano, + hratls_pubkey: Config::get_hratls_pubkey_hex()?, + public_package_mr_enclave: Some(mr_enclave.to_vec()), + app_name: self.app_name, + }; + + let new_app_res = new_app(req).await?; + + let (hratls_uri, mr_enclave) = + hratls_url_and_mr_enclave_from_app_id(&new_app_res.uuid).await?; + + let mr_enclave = mr_enclave.expect("App contract does not have a mr_enclave"); + + log::info!("hratls uri: {hratls_uri} mr_enclave: {mr_enclave:?}"); + + if new_app_res.error.is_empty() { + let launch_config = fetch_config(&launch_config_url).await?; + eprint!("Deploying..."); + tokio::time::sleep(tokio::time::Duration::from_millis(2500)).await; + let dtpm_client = Retry::spawn(FixedInterval::from_millis(1000).take(30), || { + log::debug!("retrying attestation and launch config update"); + eprint!("."); + dtpm_client(&hratls_uri, &mr_enclave) + }) + .await?; + println!(""); + upload_files_pb(launch_config.filesystems.clone(), &dtpm_client).await?; + + let config_data = Some(launch_config.into()); + log::trace!("Decoded the configuration... {:?}", config_data); + let req = DtpmSetConfigReq { config_data, ..Default::default() }; + set_config_pb(req, &dtpm_client).await?; + + append_uuid_list(&new_app_res.uuid, &app_name)?; + Ok((new_app_res, app_name).into()) + } else { + Err(Error::Deployment(new_app_res.error)) + } + } + + pub async fn get_app_node(&self) -> Result { + let location = snp::Location::from(self.location.as_str()); + let app_node_filter = AppNodeFilters { + vcpus: self.vcpus, + memory_mib: self.memory_mib, + storage_mib: self.disk_size_mib, + country: location.country.clone().unwrap_or_default(), + region: location.region.clone().unwrap_or_default(), + city: location.city.clone().unwrap_or_default(), + ip: location.node_ip.clone().unwrap_or_default(), + node_pubkey: String::new(), + free_ports: (self.port.len() + 1) as u32, + }; + Ok(get_one_app_node(app_node_filter).await?) + } +} diff --git a/src/sgx/grpc_brain.rs b/src/sgx/grpc_brain.rs index 1434aae..4cebe3c 100644 --- a/src/sgx/grpc_brain.rs +++ b/src/sgx/grpc_brain.rs @@ -5,13 +5,11 @@ use detee_shared::app_proto::{ AppContract, AppNodeFilters, AppNodeListResp, DelAppReq, ListAppContractsReq, NewAppReq, NewAppRes, }; -use detee_shared::sgx::types::brain::AppDeployConfig; use tokio_stream::StreamExt; use tonic::transport::Channel; use crate::call_with_follow_redirect; use crate::config::Config; -use crate::sgx::utils::calculate_nanolp_for_app; use crate::utils::{self, sign_request}; #[derive(thiserror::Error, Debug)] @@ -83,23 +81,7 @@ async fn client_from_endpoint(reconnect_endpoint: String) -> Result Result { - let resource = app_deploy_config.clone().resource; - let mut req: NewAppReq = app_deploy_config.clone().into(); - - let locked_nano = calculate_nanolp_for_app( - resource.vcpus, - resource.memory_mib, - resource.disk_size_mib, - app_deploy_config.hours, - req.price_per_unit, - ); - - req.uuid = "".to_string(); - req.locked_nano = locked_nano; - req.admin_pubkey = Config::get_detee_wallet()?; - req.hratls_pubkey = Config::get_hratls_pubkey_hex()?; - +pub async fn new_app(req: NewAppReq) -> Result { let client = client().await?; match call_with_follow_redirect!(client, req, new_app).await { Ok(res) => Ok(res.into_inner()), diff --git a/src/sgx/grpc_dtpm.rs b/src/sgx/grpc_dtpm.rs index 5a06186..0e3eea8 100644 --- a/src/sgx/grpc_dtpm.rs +++ b/src/sgx/grpc_dtpm.rs @@ -38,20 +38,18 @@ pub enum Error { type Result = std::result::Result; -pub async fn connect_app_dtpm_client(app_uuid: &str) -> Result> { +pub async fn dtpm_client( + hratls_uri: &str, + mr_enclave: &[u8; 32], +) -> Result> { let private_key_pem = Config::get_hratls_private_key()?; - let (hratls_uri, package_mr_enclave) = hratls_url_and_mr_enclave_from_app_id(app_uuid).await?; - log::info!("hratls uri: {} mr_enclave: {:?}", &hratls_uri, &package_mr_enclave); - let hratls_config = Arc::new(RwLock::new(HRaTlsConfig::new().with_hratls_private_key_pem(private_key_pem))); - if let Some(mr_enclave) = package_mr_enclave { - hratls_config.write().unwrap().allow_more_instance_measurement( - InstanceMeasurement::new().with_mrenclaves(vec![mr_enclave]), - ); - } + hratls_config.write().unwrap().allow_more_instance_measurement( + InstanceMeasurement::new().with_mrenclaves(vec![*mr_enclave]), + ); let client_tls_config = ClientConfig::from_hratls_config(hratls_config.clone())?; let connector = HttpsConnectorBuilder::new() @@ -60,13 +58,17 @@ pub async fn connect_app_dtpm_client(app_uuid: &str) -> Result Result<()> { - let dtpm_client = connect_app_dtpm_client(app_uuid).await?; + let (hratls_uri, mr_enclave) = hratls_url_and_mr_enclave_from_app_id(app_uuid).await?; + let mr_enclave = mr_enclave.expect("App contract does not have a mr_enclave"); + + let dtpm_client = dtpm_client(&hratls_uri, &mr_enclave).await?; upload_files_pb(config.filesystems.clone(), &dtpm_client).await?; let req = DtpmSetConfigReq { config_data: Some(config.into()), ..Default::default() }; @@ -75,7 +77,10 @@ pub async fn update_config(app_uuid: &str, config: DtpmConfig) -> Result<()> { } pub async fn get_config(app_uuid: &str) -> Result { - let dtpm_client = connect_app_dtpm_client(app_uuid).await?; + let (hratls_uri, mr_enclave) = hratls_url_and_mr_enclave_from_app_id(app_uuid).await?; + let mr_enclave = mr_enclave.expect("App contract does not have a mr_enclave"); + + let dtpm_client = dtpm_client(&hratls_uri, &mr_enclave).await?; let config_res = get_config_pb(&dtpm_client).await?; let config: DtpmConfig = config_res.config_data.ok_or(Error::Dtpm("config data not found".to_string()))?.into(); diff --git a/src/sgx/mod.rs b/src/sgx/mod.rs index 9c28f0e..3ffa689 100644 --- a/src/sgx/mod.rs +++ b/src/sgx/mod.rs @@ -2,6 +2,7 @@ pub mod cli_handler; pub mod config; +pub mod deploy; pub mod grpc_brain; pub mod grpc_dtpm; pub mod packaging; @@ -9,13 +10,11 @@ pub mod utils; use crate::config::Config; use crate::constants::HRATLS_APP_PORT; -use crate::snp; use crate::utils::{block_on, shorten_string}; use detee_shared::app_proto::{ AppContract as AppContractPB, AppNodeFilters, AppNodeListResp, AppResource, ListAppContractsReq, NewAppRes, }; -use detee_shared::sgx::types::brain::Resource; use grpc_brain::get_one_app_node; use serde::{Deserialize, Serialize}; use std::sync::LazyLock; @@ -29,8 +28,16 @@ pub enum Error { AppContractNotFound(String), #[error("Brain returned the following error: {0}")] Brain(#[from] grpc_brain::Error), + #[error("{0}")] + Dtpm(#[from] crate::sgx::grpc_dtpm::Error), #[error("Could not read file from disk: {0}")] FileNotFound(#[from] std::io::Error), + #[error("{0}")] + Deployment(String), + #[error(transparent)] + Reqwest(#[from] reqwest::Error), + #[error(transparent)] + Serde(#[from] serde_yaml::Error), } #[derive(Tabled, Debug, Serialize, Deserialize)] @@ -223,24 +230,6 @@ impl crate::HumanOutput for AppDeleteResponse { } } -pub async fn get_app_node( - resource: Resource, - location: snp::Location, -) -> Result { - let app_node_filter = AppNodeFilters { - vcpus: resource.vcpus, - memory_mib: resource.memory_mib, - storage_mib: resource.disk_size_mib, - country: location.country.clone().unwrap_or_default(), - region: location.region.clone().unwrap_or_default(), - city: location.city.clone().unwrap_or_default(), - ip: location.node_ip.clone().unwrap_or_default(), - node_pubkey: String::new(), - free_ports: (resource.port.len() + 1) as u32, - }; - get_one_app_node(app_node_filter).await -} - pub fn inspect_node(ip: String) -> Result { let req = AppNodeFilters { ip, ..Default::default() }; block_on(get_one_app_node(req)) @@ -282,10 +271,10 @@ impl super::HumanOutput for Vec { } } -pub fn print_nodes() -> Result, grpc_brain::Error> { +pub fn print_nodes() -> Result, Error> { log::debug!("This will support flags in the future, but we have only one node atm."); let req = AppNodeFilters { ..Default::default() }; - block_on(grpc_brain::get_app_node_list(req)) + Ok(block_on(grpc_brain::get_app_node_list(req))?) } pub async fn get_app_node_by_contract(uuid: &str) -> Result { diff --git a/src/sgx/utils.rs b/src/sgx/utils.rs index 7a50454..e2e6266 100644 --- a/src/sgx/utils.rs +++ b/src/sgx/utils.rs @@ -1,31 +1,9 @@ // SPDX-License-Identifier: Apache-2.0 use crate::constants::HRATLS_APP_PORT; -use crate::sgx::grpc_brain::new_app; -use crate::sgx::grpc_dtpm::{connect_app_dtpm_client, set_config_pb, upload_files_pb}; -use crate::sgx::{get_one_contract, package_entry_from_name}; -use detee_shared::app_proto::NewAppRes; -use detee_shared::sgx::pb::dtpm_proto::DtpmSetConfigReq; -use detee_shared::sgx::types::brain::AppDeployConfig; +use crate::sgx::get_one_contract; +use crate::sgx::Error; use detee_shared::sgx::types::dtpm::{DtpmConfig, EnvironmentEntry}; -use tokio_retry::strategy::FixedInterval; -use tokio_retry::Retry; - -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error(transparent)] - Reqwest(#[from] reqwest::Error), - #[error(transparent)] - Serde(#[from] serde_yaml::Error), - #[error("{0}")] - Package(std::string::String), - #[error("{0}")] - Brain(#[from] crate::sgx::grpc_brain::Error), - #[error("{0}")] - Dtpm(#[from] crate::sgx::grpc_dtpm::Error), - #[error("{0}")] - Deployment(String), -} pub async fn hratls_url_and_mr_enclave_from_app_id( app_id: &str, @@ -53,16 +31,9 @@ pub async fn hratls_url_and_mr_enclave_from_app_id( Ok((format!("https://{public_ip}:{dtpm_port}"), mr_enclave)) } -pub async fn fetch_config(package_name: &str) -> Result { - let index_package_entry = package_entry_from_name(package_name) - .ok_or(Error::Package("package not found for ".to_string() + package_name))?; - - let launch_config_url = index_package_entry.launch_config_url.clone(); - - let launch_config_str = reqwest::get(launch_config_url).await?.text().await?; - +pub async fn fetch_config(url: &str) -> Result { + let launch_config_str = reqwest::get(url).await?.text().await?; let launch_config = serde_yaml::from_str::(&launch_config_str)?; - Ok(launch_config) } @@ -121,35 +92,3 @@ pub fn override_envs_and_args_launch_config( launch_config } - -pub async fn deploy_new_app_and_update_config( - app_deploy_config: AppDeployConfig, - launch_config: Option, -) -> Result { - let new_app_res = new_app(app_deploy_config).await?; - - if new_app_res.error.is_empty() { - if let Some(launch_config) = launch_config { - eprint!("Deploying..."); - tokio::time::sleep(tokio::time::Duration::from_millis(2500)).await; - let dtpm_client = Retry::spawn(FixedInterval::from_millis(1000).take(30), || { - log::debug!("retrying attestation and launch config update"); - eprint!("."); - connect_app_dtpm_client(&new_app_res.uuid) - }) - .await?; - println!(""); - upload_files_pb(launch_config.filesystems.clone(), &dtpm_client).await?; - - let config_data = Some(launch_config.into()); - log::trace!("Decoded the configuration... {:?}", config_data); - let req = DtpmSetConfigReq { config_data, ..Default::default() }; - set_config_pb(req, &dtpm_client).await?; - Ok(new_app_res) - } else { - Ok(new_app_res) - } - } else { - Err(Error::Deployment(new_app_res.error)) - } -}