// SPDX-License-Identifier: Apache-2.0 use crate::{config::Config, global::*, grpc::snp_proto}; use anyhow::{anyhow, Result}; use log::{debug, info}; use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, fs, fs::{remove_file, File}, io::Write, process::Command, }; #[derive(Debug)] pub enum VMCreationErrors { PriceIsTooLow, VMAlreadyExists(VM), NATandIPv4Conflict, ServerFull, NotEnoughPorts, NotEnoughCPU, NotEnoughMemory, NotEnoughStorage, IPv4NotAvailable, IPv6NotAvailable, MemoryTooLow, DiskTooSmall, DowngradeNotSupported, UnbalancedVM, ServerDiskError(String), BootFileError(String), HypervizorError(String), } #[derive(Serialize, Deserialize, Debug, Default)] pub struct HwReservation { vcpus: usize, memory: usize, disk: usize, } pub struct State { pub config: Config, pub res: Resources, } impl State { fn available_ports(&mut self, extra_ports: usize) -> Vec { use rand::Rng; let total_ports = extra_ports + 1; if self.config.public_port_range.len() < self.res.reserved_ports.len() + total_ports as usize { return Vec::new(); } if total_ports > self.config.max_ports_per_vm as usize { return Vec::new(); } let mut published_ports = Vec::new(); for _ in 0..total_ports { for _ in 0..5 { let port = rand::thread_rng().gen_range(self.config.public_port_range.clone()); if self.res.reserved_ports.get(&port).is_none() { published_ports.push(port); } break; } } published_ports.sort(); published_ports } fn nr_of_available_ips(&self) -> (u32, u32) { let mut avail_ipv4 = 0; let mut avail_ipv6 = 0; for nic in self.config.network_interfaces.iter() { for range in nic.ipv4_ranges.iter() { avail_ipv4 += (range.last_ip.to_bits() + 1) - range.first_ip.to_bits(); } for range in nic.ipv6_ranges.iter() { avail_ipv6 += (range.last_ip.to_bits() + 1) - range.first_ip.to_bits(); } } ( avail_ipv4.saturating_sub(self.res.reserved_ipv4.len() as u32), avail_ipv6.saturating_sub(self.res.reserved_ipv6.len() as u128) as u32, ) } pub fn available_resources(&self) -> snp_proto::VmNodeResources { let (avail_ipv4, avail_ipv6) = self.nr_of_available_ips(); let mut offer_resources: Vec = Vec::new(); for config_offer in self.config.offers.iter() { let offer = match self.res.reserved_hw.get(&config_offer.storage_path) { Some(used_resources) => snp_proto::VmNodeOffer { price: config_offer.price, vcpus: config_offer.max_vcpu_reservation.saturating_sub(used_resources.vcpus) as u64, memory_mib: config_offer .max_mem_reservation_mib .saturating_sub(used_resources.memory) as u64, disk_mib: config_offer.max_storage_mib.saturating_sub(used_resources.disk) as u64, }, None => snp_proto::VmNodeOffer { price: config_offer.price, vcpus: config_offer.max_vcpu_reservation as u64, memory_mib: config_offer.max_mem_reservation_mib as u64, disk_mib: config_offer.max_storage_mib as u64, }, }; offer_resources.push(offer); } let res = snp_proto::VmNodeResources { node_pubkey: PUBLIC_KEY.clone(), avail_ports: (self.config.public_port_range.len() - self.res.reserved_ports.len()) as u32, avail_ipv4, avail_ipv6, offers: offer_resources, max_ports_per_vm: self.config.max_ports_per_vm as u32, }; debug!("Node resources are currently: {res:?}"); res } pub fn get_offer_path(&self, req: &NewVMRequest) -> Result { let vm_mem_per_cpu = req.memory_mib / req.vcpus; let vm_disk_per_cpu = req.disk_size_mib / req.vcpus; let mut offer_path = String::new(); let mut error = VMCreationErrors::UnbalancedVM; for offer in self.config.offers.iter() { let available_resources = match self.res.reserved_hw.get(&offer.storage_path) { Some(used_resources) => HwReservation { vcpus: offer.max_vcpu_reservation.saturating_sub(used_resources.vcpus), memory: offer.max_mem_reservation_mib.saturating_sub(used_resources.memory), disk: offer.max_storage_mib.saturating_sub(used_resources.disk), }, None => HwReservation { vcpus: offer.max_vcpu_reservation, memory: offer.max_mem_reservation_mib, disk: offer.max_storage_mib, }, }; let mem_per_cpu = available_resources.memory / available_resources.vcpus; let disk_per_cpu = available_resources.disk / available_resources.vcpus; if !within_10_percent(vm_mem_per_cpu, mem_per_cpu) || !within_10_percent(vm_disk_per_cpu, disk_per_cpu) { continue; } if req.vcpus > available_resources.vcpus { error = VMCreationErrors::NotEnoughCPU; continue; } if req.memory_mib > available_resources.memory + 50 { error = VMCreationErrors::NotEnoughMemory; continue; } if req.disk_size_mib > available_resources.disk + 100 { error = VMCreationErrors::NotEnoughStorage; continue; } if req.price < offer.price { error = VMCreationErrors::PriceIsTooLow; continue; } offer_path = offer.storage_path.clone(); } if offer_path.is_empty() { return Err(error); } Ok(offer_path) } pub fn new_vm(&mut self, req: NewVMRequest) -> Result { let offer_path = self.get_offer_path(&req)?; if self.res.existing_vms.contains(&req.vm_id) { let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &req.vm_id + ".yaml") .map_err(|e| VMCreationErrors::ServerDiskError(e.to_string()))?; let vm: crate::state::VM = serde_yaml::from_str(&content) .map_err(|e| VMCreationErrors::ServerDiskError(e.to_string()))?; return Err(VMCreationErrors::VMAlreadyExists(vm)); } if req.extra_ports.len() > 0 && req.public_ipv4 { return Err(VMCreationErrors::NATandIPv4Conflict); } if req.memory_mib < 800 { return Err(VMCreationErrors::MemoryTooLow); } if req.disk_size_mib < 4000 { return Err(VMCreationErrors::DiskTooSmall); } if let Err(ovmf_err) = self.res.find_or_download_file(OVMF_URL.to_string(), OVMF_HASH.to_string()) { return Err(VMCreationErrors::BootFileError(format!( "Could not get OVMF: {ovmf_err:?}" ))); }; if let Err(kernel_err) = self.res.find_or_download_file(req.kernel_url, req.kernel_sha.clone()) { return Err(VMCreationErrors::BootFileError(format!( "Could not get kernel: {kernel_err:?}" ))); }; if let Err(dtrfs_err) = self.res.find_or_download_file(req.dtrfs_url, req.dtrfs_sha.clone()) { return Err(VMCreationErrors::BootFileError(format!( "Could not get dtrfs: {dtrfs_err:?}" ))); }; let mut vm_nics = Vec::new(); if req.public_ipv4 { match self.res.available_ipv4(&self.config.network_interfaces) { Some(vmnic) => vm_nics.push(vmnic), None => return Err(VMCreationErrors::IPv4NotAvailable), } } if req.public_ipv6 { match self.res.available_ipv6(&self.config.network_interfaces) { Some(mut vmnic) => { if let Some(mut existing_vmnic) = vm_nics.pop() { if vmnic.if_config.device_name() == existing_vmnic.if_config.device_name() { vmnic.ips.append(&mut existing_vmnic.ips); vm_nics.push(vmnic); } else { vm_nics.push(existing_vmnic); vm_nics.push(vmnic); } } else { vm_nics.push(vmnic); } } None => { return Err(VMCreationErrors::IPv6NotAvailable); } } } let mut host_ports: Vec = Vec::new(); let mut port_pairs: Vec<(u16, u16)> = Vec::new(); if !req.public_ipv4 { host_ports.append(self.available_ports(req.extra_ports.len()).as_mut()); if host_ports.len() == 0 { return Err(VMCreationErrors::NotEnoughPorts); } port_pairs.push((host_ports[0], 22)); for i in 0..req.extra_ports.len() { port_pairs.push((host_ports[i + 1], req.extra_ports[i])); } } let vm = VM { vm_id: req.vm_id, admin_key: req.admin_key, nics: vm_nics, vcpus: req.vcpus, memory_mib: req.memory_mib, disk_size_mib: req.disk_size_mib, kernel_sha: req.kernel_sha, dtrfs_sha: req.dtrfs_sha, fw_ports: port_pairs, storage_dir: offer_path, }; if let Err(e) = vm.write_config() { return Err(VMCreationErrors::ServerDiskError(e.to_string())); } self.res.reserve_vm_resources(&vm); Ok(vm) } } #[derive(Serialize, Deserialize, Debug, Default)] pub struct Resources { pub existing_vms: HashSet, // QEMU does not support MHz limiation pub reserved_ports: HashSet, // maps storage path to hardware reservation pub reserved_hw: HashMap, pub reserved_ipv4: HashSet, pub reserved_ipv6: HashSet, pub reserved_if_names: HashSet, // sha256sum -> absolute path boot_files: HashSet, } impl Resources { fn save_to_disk(&self) -> Result<()> { let mut file = File::create(USED_RESOURCES)?; file.write_all(serde_yaml::to_string(self)?.as_bytes())?; Ok(()) } pub fn load_from_disk() -> Result { let mut res = Self { ..Default::default() }; log::debug!("Reading VMs saved to disk to calculate used resources..."); for entry in fs::read_dir(VM_CONFIG_DIR)? { let entry = entry?; let path = entry.path(); if path.is_file() && path.to_string_lossy().ends_with(".yaml") { log::info!("Found VM config: {:?}", path.to_str()); let content = std::fs::read_to_string(path)?; let vm: VM = serde_yaml::from_str(&content)?; res.existing_vms.insert(vm.vm_id); res.reserved_hw .entry(vm.storage_dir) .and_modify(|utilization| { utilization.vcpus = utilization.vcpus.saturating_add(vm.vcpus); utilization.memory = utilization.memory.saturating_add(vm.memory_mib); utilization.disk = utilization.disk.saturating_add(vm.disk_size_mib); }) .or_insert(HwReservation { vcpus: vm.vcpus, memory: vm.memory_mib, disk: vm.disk_size_mib, }); for (port, _) in vm.fw_ports.iter() { res.reserved_ports.insert(*port); } for nic in vm.nics { for ip in nic.ips { if let Ok(ip_address) = ip.address.parse::() { if ip_address.is_ipv4() { res.reserved_ipv4.insert(ip.address.clone()); } if ip_address.is_ipv6() { res.reserved_ipv6.insert(ip.address); } } } if let Some(vtap_name) = nic.if_config.vtap_name() { res.reserved_if_names.insert(vtap_name); } } } } res.scan_boot_files().unwrap(); res.save_to_disk()?; Ok(res) } fn available_if_name(&mut self) -> String { use rand::{distributions::Alphanumeric, Rng}; loop { let mut interface_name: String = rand::thread_rng().sample_iter(&Alphanumeric).take(9).map(char::from).collect(); interface_name = "detee".to_string() + &interface_name; if !self.reserved_if_names.contains(&interface_name) { return interface_name; } } } fn available_ipv4(&mut self, nics: &Vec) -> Option { for nic in nics.iter() { for range in nic.ipv4_ranges.iter() { let first_ip = range.first_ip.to_bits(); let last_ip = range.last_ip.to_bits(); for ip in first_ip..last_ip + 1 { let ip_addr = std::net::Ipv4Addr::from_bits(ip); if !self.reserved_ipv4.contains(&ip_addr.to_string()) && ip_addr != range.gateway { let if_config = match nic.driver { crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP { name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP { name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::Bridge => { InterfaceConfig::Bridge { device: nic.device.clone() } } }; let mut ips = Vec::new(); ips.push(IPConfig { address: ip_addr.to_string(), mask: range.netmask.clone(), gateway: range.gateway.to_string(), }); return Some(VMNIC { if_config, ips }); } } } } None } fn available_ipv6(&mut self, nics: &Vec) -> Option { for nic in nics.iter() { for range in nic.ipv6_ranges.iter() { let first_ip = range.first_ip.to_bits(); let last_ip = range.last_ip.to_bits(); for ip in first_ip..last_ip + 1 { let ip_addr = std::net::Ipv6Addr::from_bits(ip); if !self.reserved_ipv6.contains(&ip_addr.to_string()) && ip_addr != range.gateway { let if_config = match nic.driver { crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP { name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP { name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::Bridge => { InterfaceConfig::Bridge { device: nic.device.clone() } } }; let mut ips = Vec::new(); ips.push(IPConfig { address: ip_addr.to_string(), mask: range.netmask.clone(), gateway: range.gateway.to_string(), }); return Some(VMNIC { if_config, ips }); } } } } None } fn scan_boot_files(&mut self) -> Result<()> { for entry in fs::read_dir(VM_BOOT_DIR)? { let entry = entry?; let path = entry.path(); if path.is_file() { match compute_sha256(&path) { Ok(hash) => { if *hash == *entry.file_name() { self.boot_files.insert(hash); } else { // TODO: rename file and insert } } Err(e) => return Err(anyhow!("Error computing hash for {:?}: {}", path, e)), } } } Ok(()) } fn find_or_download_file(&mut self, url: String, sha: String) -> Result<()> { if !self.boot_files.contains(&sha) { download_and_check_sha(&url, &sha)?; } self.boot_files.insert(sha); Ok(()) } fn reserve_vm_resources(&mut self, vm: &VM) { self.existing_vms.insert(vm.vm_id.clone()); for nic in vm.nics.iter() { if let Some(vtap) = nic.if_config.vtap_name() { self.reserved_if_names.insert(vtap); } for ip in nic.ips.iter() { if let Ok(ip_address) = ip.address.parse::() { if ip_address.is_ipv4() { self.reserved_ipv4.insert(ip.address.clone()); } if ip_address.is_ipv6() { self.reserved_ipv6.insert(ip.address.clone()); } } } } for (host_port, _) in vm.fw_ports.iter() { self.reserved_ports.insert(*host_port); } self.reserved_hw .entry(vm.storage_dir.clone()) .and_modify(|hw_res| { hw_res.vcpus = hw_res.vcpus.saturating_add(vm.vcpus); hw_res.memory = hw_res.memory.saturating_add(vm.memory_mib); hw_res.disk = hw_res.disk.saturating_add(vm.disk_size_mib); }) .or_insert(HwReservation { vcpus: vm.vcpus, memory: vm.memory_mib, disk: vm.disk_size_mib, }); let _ = self.save_to_disk(); } fn free_vm_resources(&mut self, vm: &VM) { if !self.existing_vms.remove(&vm.vm_id) { return; } for nic in vm.nics.iter() { if let Some(vtap) = nic.if_config.vtap_name() { self.reserved_if_names.remove(&vtap); } for ip in nic.ips.iter() { if let Ok(ip_address) = ip.address.parse::() { if ip_address.is_ipv4() { self.reserved_ipv4.remove(&ip.address); } if ip_address.is_ipv6() { self.reserved_ipv6.remove(&ip.address); } } } } for (host_port, _) in vm.fw_ports.iter() { self.reserved_ports.remove(host_port); } self.reserved_hw.entry(vm.storage_dir.clone()).and_modify(|hw_res| { hw_res.vcpus = hw_res.vcpus.saturating_sub(vm.vcpus); hw_res.memory = hw_res.memory.saturating_sub(vm.memory_mib); hw_res.disk = hw_res.disk.saturating_sub(vm.disk_size_mib); }); if let Err(e) = self.save_to_disk() { log::error!("Could not save resources to disk: {e}"); } } } #[derive(Serialize, Deserialize, Debug)] pub enum InterfaceConfig { // TODO: instead of QEMU userspace NAT, use iptables kernelspace NAT // in case of QEMU-base NAT, device name is not needed NAT { device: String }, MACVTAP { name: String, device: String }, IPVTAP { name: String, device: String }, Bridge { device: String }, } impl InterfaceConfig { fn if_type(&self) -> String { match self { InterfaceConfig::IPVTAP { .. } => format!("ipvtap"), InterfaceConfig::MACVTAP { .. } => format!("macvtap"), InterfaceConfig::NAT { .. } => format!("nat"), InterfaceConfig::Bridge { .. } => format!("bridge"), } } fn device_name(&self) -> String { match self { InterfaceConfig::IPVTAP { device, .. } => device.clone(), InterfaceConfig::MACVTAP { device, .. } => device.clone(), InterfaceConfig::NAT { device, .. } => device.clone(), InterfaceConfig::Bridge { device, .. } => device.clone(), } } fn vtap_name(&self) -> Option { match self { InterfaceConfig::IPVTAP { name, .. } => Some(name.clone()), InterfaceConfig::MACVTAP { name, .. } => Some(name.clone()), InterfaceConfig::NAT { .. } | InterfaceConfig::Bridge { .. } => None, } } // We will need this when we enable more complex NIC setup. // fn is_vtap(&self) -> bool { // match self { // InterfaceConfig::IPVTAP { .. } | InterfaceConfig::MACVTAP { .. } => true, // InterfaceConfig::NAT { .. } | InterfaceConfig::Bridge { .. } => false, // } // } } #[derive(Serialize, Deserialize, Debug)] struct IPConfig { address: String, // requires short format (example: 24) mask: String, gateway: String, } #[derive(Serialize, Deserialize, Debug)] pub struct VMNIC { if_config: InterfaceConfig, ips: Vec, } #[derive(Serialize, Deserialize, Debug)] pub struct VM { pub vm_id: String, admin_key: String, fw_ports: Vec<(u16, u16)>, nics: Vec, // currently hardcoded to EPYC-v4 // cpu_type: String, vcpus: usize, memory_mib: usize, disk_size_mib: usize, kernel_sha: String, dtrfs_sha: String, storage_dir: String, } impl From for snp_proto::MeasurementArgs { fn from(vm: VM) -> Self { let mut nic_index: u32 = if vm.fw_ports.is_empty() { 0 } else { 1 }; let mut ips = Vec::new(); let mut dtrfs_api_endpoint = String::new(); // TODO: when brain supports multiple IPs per VM, fix this for nic in vm.nics { for ip in nic.ips { if ip.address.parse::().is_ok() { dtrfs_api_endpoint = ip.address.clone(); } ips.push(snp_proto::MeasurementIp { nic_index, address: ip.address, mask: ip.mask, gateway: ip.gateway, }); } nic_index += 1; } if !dtrfs_api_endpoint.is_empty() { dtrfs_api_endpoint += ":22"; } else { dtrfs_api_endpoint += &format!("{}:{}", IP_INFO.ip, vm.fw_ports[0].0); } snp_proto::MeasurementArgs { dtrfs_api_endpoint, exposed_ports: vm.fw_ports.iter().map(|(host_port, _)| *host_port as u32).collect(), ips, ovmf_hash: OVMF_HASH.to_string(), } } } impl From for snp_proto::NewVmResp { fn from(vm: VM) -> Self { let vm_id = vm.vm_id.clone(); snp_proto::NewVmResp { vm_id, args: Some(vm.into()), error: "".to_string() } } } impl From for snp_proto::UpdateVmResp { fn from(vm: VM) -> Self { let vm_id = vm.vm_id.clone(); snp_proto::UpdateVmResp { vm_id, args: Some(vm.into()), error: "".to_string() } } } #[derive(Deserialize, Debug)] pub struct NewVMRequest { vm_id: String, admin_key: String, extra_ports: Vec, public_ipv4: bool, public_ipv6: bool, disk_size_mib: usize, vcpus: usize, memory_mib: usize, kernel_url: String, kernel_sha: String, dtrfs_url: String, dtrfs_sha: String, price: u64, } impl From for NewVMRequest { fn from(req: snp_proto::NewVmReq) -> Self { Self { vm_id: req.vm_id, admin_key: req.admin_pubkey, extra_ports: req.extra_ports.iter().map(|&port| port as u16).collect(), public_ipv4: req.public_ipv4, public_ipv6: req.public_ipv6, disk_size_mib: req.disk_size_mib as usize, vcpus: req.vcpus as usize, memory_mib: req.memory_mib as usize, kernel_url: req.kernel_url, kernel_sha: req.kernel_sha, dtrfs_url: req.dtrfs_url, dtrfs_sha: req.dtrfs_sha, price: req.price_per_unit, } } } #[derive(Deserialize, Debug)] pub struct UpdateVMReq { pub vm_id: String, vcpus: usize, memory_mib: usize, disk_size_mib: usize, // we are not using Option, as these will be passed from gRPC kernel_url: String, kernel_sha: String, dtrfs_url: String, dtrfs_sha: String, } impl From for UpdateVMReq { fn from(req: snp_proto::UpdateVmReq) -> Self { Self { vm_id: req.vm_id, vcpus: req.vcpus as usize, memory_mib: req.memory_mib as usize, disk_size_mib: req.disk_size_mib as usize, kernel_url: req.kernel_url, kernel_sha: req.kernel_sha, dtrfs_url: req.dtrfs_url, dtrfs_sha: req.dtrfs_sha, } } } impl VM { pub fn update(&mut self, req: UpdateVMReq, state: &mut State) -> Result<(), VMCreationErrors> { if req.vcpus < self.vcpus && req.vcpus != 0 && req.memory_mib != 0 && req.disk_size_mib != 0 { // Downgrade will be upported only after we implement deviation for VMs. // (Deviation from the slot size allows management of VMs with unbalanced resources, // without fully saturating a node. We are disabling downgrades to avoid complexity at // this stage of the product) return Err(VMCreationErrors::DowngradeNotSupported); } let offer_path = self.storage_dir.clone(); let mut available_vcpus = 0; let mut available_mem = 0; let mut available_disk = 0; for offer in state.config.offers.iter() { if offer.storage_path != offer_path { continue; } let used_hw_resources = match state.res.reserved_hw.get(&offer_path) { Some(r) => r, None => continue, }; available_vcpus = offer.max_vcpu_reservation.saturating_sub(used_hw_resources.vcpus); available_mem = offer.max_mem_reservation_mib.saturating_sub(used_hw_resources.memory); available_disk = offer.max_storage_mib.saturating_sub(used_hw_resources.disk); } if req.vcpus > 0 && available_vcpus.saturating_add(self.vcpus) < req.vcpus { return Err(VMCreationErrors::NotEnoughCPU); } if req.memory_mib > 0 && available_mem.saturating_add(self.memory_mib) < req.memory_mib { return Err(VMCreationErrors::NotEnoughMemory); } if req.disk_size_mib > 0 { if available_disk.saturating_add(self.disk_size_mib) < req.disk_size_mib { return Err(VMCreationErrors::NotEnoughStorage); } if req.disk_size_mib < self.disk_size_mib { return Err(VMCreationErrors::DiskTooSmall); } } if !req.kernel_sha.is_empty() || !req.dtrfs_sha.is_empty() { if req.kernel_sha.is_empty() || req.dtrfs_sha.is_empty() { return Err(VMCreationErrors::BootFileError( "Kernel and DTRFS can be upgraded only as a bundle".to_string(), )); } if let Err(kern_err) = state.res.find_or_download_file(req.kernel_url, req.kernel_sha.clone()) { return Err(VMCreationErrors::BootFileError(format!( "Could not get kernel: {kern_err:?}" ))); }; if let Err(dtrfs_err) = state.res.find_or_download_file(req.dtrfs_url, req.dtrfs_sha.clone()) { return Err(VMCreationErrors::BootFileError(format!( "Could not get dtrfs: {dtrfs_err:?}" ))); }; info!( "Kernel and DTRFS updated for VM: {}, kernel {}, dtrfs: {}", self.vm_id, req.kernel_sha, req.dtrfs_sha ); self.kernel_sha = req.kernel_sha; self.dtrfs_sha = req.dtrfs_sha; } // Update the resources state.res.reserved_hw.entry(offer_path).and_modify(|hw_res| { hw_res.vcpus = hw_res.vcpus.saturating_add(req.vcpus); hw_res.vcpus = hw_res.vcpus.saturating_sub(self.vcpus); hw_res.memory = hw_res.memory.saturating_add(req.memory_mib); hw_res.memory = hw_res.memory.saturating_sub(self.memory_mib); hw_res.disk = hw_res.disk.saturating_add(req.disk_size_mib); hw_res.disk = hw_res.disk.saturating_sub(self.disk_size_mib); }); let _ = state.res.save_to_disk(); if req.memory_mib != 0 { self.memory_mib = req.memory_mib; } if req.vcpus != 0 { self.vcpus = req.vcpus; } if req.disk_size_mib != 0 { self.disk_size_mib = req.disk_size_mib; } if let Err(e) = systemctl_stop_and_disable(&self.vm_id) { return Err(VMCreationErrors::HypervizorError(e.to_string())); } if let Err(e) = self.write_config() { return Err(VMCreationErrors::ServerDiskError(e.to_string())); } if let Err(e) = self.write_sh_exports() { return Err(VMCreationErrors::ServerDiskError(e.to_string())); } if let Err(e) = self.resize_disk() { return Err(VMCreationErrors::HypervizorError(e.to_string())); } if let Err(e) = systemctl_start_and_enable(&self.vm_id) { return Err(VMCreationErrors::HypervizorError(e.to_string())); } Ok(()) } pub fn start(&self) -> Result<()> { self.create_disk()?; self.write_sh_exports()?; self.write_systemd_unit_file()?; systemctl_reload()?; systemctl_start_and_enable(&self.vm_id)?; Ok(()) } pub fn delete(&self, res: &mut Resources) -> Result<()> { let _ = systemctl_stop_and_disable(&self.vm_id); let _ = self.delete_systemd_unit_file(); let _ = systemctl_reload(); let _ = self.delete_disk(); let _ = self.delete_sh_exports(); let _ = self.delete_vtap_interfaces(); let _ = self.delete_config(); res.free_vm_resources(&self); Ok(()) } // For the MVP, the T-Contract offers VM+IP+Disk as a bundle. // This means we can enforce the path to the disk. // This may change in the future as the VM is allowed to have multiple disks. pub fn disk_path(&self) -> String { let dir = match self.storage_dir.ends_with("/") { true => self.storage_dir.clone(), false => self.storage_dir.clone() + "/", }; dir + &self.vm_id + ".qcow2" } // If you change this here, you also have to change it in the CLI. // The kernel params must match on both daemon and CLI to build the measurement. pub fn kernel_params(&self) -> String { let mut ip_string = String::new(); let mut i = 0; if self.fw_ports.len() > 0 { ip_string += &format!("detee_net_eth{}={}_{}_{} ", i, "10.0.2.15", "24", "10.0.2.2"); i += 1; } for nic in self.nics.iter() { for ip in nic.ips.iter() { ip_string += &format!("detee_net_eth{}={}_{}_{} ", i, ip.address, ip.mask, ip.gateway); } i += 1; } let admin_key = format!("detee_admin={} ", self.admin_key); let vm_id = format!("detee_vm_id={}", self.vm_id); format!("{}{}{}", ip_string, admin_key, vm_id) } fn write_config(&self) -> Result<()> { let mut file = File::create(VM_CONFIG_DIR.to_string() + &self.vm_id + ".yaml")?; file.write_all(serde_yaml::to_string(self)?.as_bytes())?; Ok(()) } fn delete_config(&self) -> Result<()> { remove_file(VM_CONFIG_DIR.to_string() + &self.vm_id + ".yaml")?; Ok(()) } fn write_sh_exports(&self) -> Result<()> { let mut vars = String::new(); let mut i = 0; for nic in self.nics.iter() { let mut interface = String::new(); interface += &format!(r#"export NETWORK_INTERFACE_{}="{}"#, i, nic.if_config.if_type()); // device is currently ignored in case of NAT cause we assume QEMU userspace NAT if let Some(vtap_name) = nic.if_config.vtap_name() { interface += &format!("_{}_{}", nic.if_config.device_name(), vtap_name); } interface += r#"""#; vars += &format!(r#"{}"#, interface); vars += "\n"; i += 1; } let mut ports = String::new(); for port in self.fw_ports.iter() { ports += &format!("{}:{} ", port.0, port.1); } if ports != "" { vars += &format!(r#"export NAT_PORT_FW="{}""#, ports.trim_end()); vars += "\n"; vars += "export NETWORK_INTERFACE_0000=NAT\n"; } vars += &format!(r#"export KERNEL="{}""#, VM_BOOT_DIR.to_string() + &self.kernel_sha); vars += "\n"; vars += &format!(r#"export INITRD="{}""#, VM_BOOT_DIR.to_string() + &self.dtrfs_sha); vars += "\n"; vars += &format!(r#"export PARAMS="{}""#, self.kernel_params()); vars += "\n"; vars += &format!(r#"export CPU_TYPE="{}""#, QEMU_VM_CPU_TYPE); vars += "\n"; vars += &format!(r#"export VCPUS="{}""#, self.vcpus); vars += "\n"; vars += &format!(r#"export MEMORY="{}M""#, (self.memory_mib / 2 * 2)); vars += "\n"; vars += &format!(r#"export MAX_MEMORY="{}M""#, (self.memory_mib / 2 * 2) + 256); vars += "\n"; vars += &format!(r#"export DISK="{}""#, self.disk_path()); vars += "\n"; let mut file = File::create(VM_CONFIG_DIR.to_string() + &self.vm_id + ".sh")?; file.write_all(vars.as_bytes())?; Ok(()) } fn delete_sh_exports(&self) -> Result<()> { remove_file(VM_CONFIG_DIR.to_string() + &self.vm_id + ".sh")?; Ok(()) } fn delete_vtap_interfaces(&self) -> Result<()> { for nic in self.nics.iter() { if let Some(name) = nic.if_config.vtap_name() { let result = Command::new("ip").arg("link").arg("del").arg(&name).output()?; if !result.status.success() { return Err(anyhow!( "Could not delete vtap interface {:?}:\n{:?}\n{:?}", name, result.stdout, result.stderr )); } } } Ok(()) } fn write_systemd_unit_file(&self) -> Result<()> { let mut contents = String::new(); contents += &format!("[Unit]\n"); contents += &format!("Description=DeTEE {}\n", self.vm_id); contents += &format!("After=network.target\n"); contents += &format!("\n"); contents += &format!("[Service]\n"); contents += &format!("Type=simple\n"); contents += &format!("Environment=VM_UUID={}\n", self.vm_id); contents += &format!("ExecStart={}\n", START_VM_SCRIPT); contents += &format!("ExecStop=/bin/kill -s SIGINT $MAINPID\n"); contents += &format!("Restart=always\n"); contents += &format!("\n"); contents += &format!("[Install]\n"); contents += &format!("WantedBy=multi-user.target\n"); let mut file = File::create_new("/etc/systemd/system/".to_string() + &self.vm_id + ".service")?; file.write_all(contents.as_bytes())?; Ok(()) } fn delete_systemd_unit_file(&self) -> Result<()> { remove_file("/etc/systemd/system/".to_string() + &self.vm_id + ".service")?; Ok(()) } fn create_disk(&self) -> Result<()> { if std::path::Path::new(&self.disk_path()).exists() { return Err(anyhow!("Could not create {}. The file already exists.", self.disk_path())); } let result = Command::new("qemu-img") .arg("create") .arg("-f") .arg("qcow2") .arg(self.disk_path()) .arg(self.disk_size_mib.to_string() + "M") .output()?; if !result.status.success() { return Err(anyhow!( "Could not create VM Disk:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } Ok(()) } fn resize_disk(&self) -> Result<()> { let result = Command::new("qemu-img") .arg("resize") .arg(self.disk_path()) .arg(self.disk_size_mib.to_string() + "M") .output()?; if !result.status.success() { return Err(anyhow!( "Could not create VM Disk:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } Ok(()) } fn delete_disk(&self) -> Result<()> { remove_file(self.disk_path())?; Ok(()) } } fn systemctl_start_and_enable(vm_id: &str) -> Result<()> { let result = Command::new("systemctl").arg("start").arg(vm_id.to_string() + ".service").output()?; if !result.status.success() { return Err(anyhow!( "Could not reload systemctl daemon:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } let result = Command::new("systemctl").arg("enable").arg(vm_id.to_string() + ".service").output()?; if !result.status.success() { return Err(anyhow!( "Could not reload systemctl daemon:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } Ok(()) } fn systemctl_stop_and_disable(vm_id: &str) -> Result<()> { let result = Command::new("systemctl").arg("stop").arg(vm_id.to_string() + ".service").output()?; if !result.status.success() { return Err(anyhow!( "Could not reload systemctl daemon:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } let result = Command::new("systemctl").arg("disable").arg(vm_id.to_string() + ".service").output()?; if !result.status.success() { return Err(anyhow!( "Could not reload systemctl daemon:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } Ok(()) } fn systemctl_reload() -> Result<()> { let result = Command::new("systemctl").arg("daemon-reload").output()?; if !result.status.success() { return Err(anyhow!( "Could not reload systemctl daemon:\n{:?}\n{:?}", String::from_utf8(result.stdout) .unwrap_or("Could not grab stdout from creation script.".to_string()), String::from_utf8(result.stderr) .unwrap_or("Could not grab stderr from creation script.".to_string()), )); } Ok(()) } fn download_and_check_sha(url: &str, sha: &str) -> Result<()> { use reqwest::blocking::get; use std::{fs::File, io::copy, path::Path}; let save_path = VM_BOOT_DIR.to_string() + sha; let response = get(url)?; if !response.status().is_success() { return Err(anyhow!("Failed to download file: {}", response.status())); } let mut file = File::create(Path::new(&save_path))?; copy(&mut response.bytes()?.as_ref(), &mut file)?; match crate::global::compute_sha256(&save_path) { Ok(hash) => { if hash != sha { return Err(anyhow!( "Sha of the downloaded file did not match supplised sha: {} vs {}", hash, sha )); } } Err(e) => return Err(anyhow!("Error computing hash for {:?}: {}", save_path, e)), } Ok(()) } fn within_10_percent(a: usize, b: usize) -> bool { let diff = a.abs_diff(b); // u32 let reference = a.max(b); // the larger of the two diff * 10 <= reference }