From 98459d27ca31ede552d05abbaf8843f47c86d019 Mon Sep 17 00:00:00 2001 From: ghe0 Date: Thu, 12 Dec 2024 22:38:50 +0200 Subject: [PATCH] prepared prod server config --- prod_setting/config1.yaml | 29 +++++++ src/config.rs | 6 +- src/constants.rs | 1 - src/main.rs | 6 +- src/state.rs | 161 +++++++++++++++++++++++++------------ test_data/config1.yaml | 4 +- test_data/config2.yaml | 10 +-- test_data/config3.yaml | 4 +- test_data/config4.yaml | 4 +- test_data/config5.yaml | 4 +- test_data/new_vm_req1.yaml | 4 +- test_data/new_vm_req2.yaml | 4 +- test_data/new_vm_req3.yaml | 4 +- test_data/new_vm_req4.yaml | 4 +- 14 files changed, 163 insertions(+), 82 deletions(-) create mode 100644 prod_setting/config1.yaml diff --git a/prod_setting/config1.yaml b/prod_setting/config1.yaml new file mode 100644 index 0000000..435562c --- /dev/null +++ b/prod_setting/config1.yaml @@ -0,0 +1,29 @@ +max_cores_per_vm: 4 +max_vcpu_reservation: 8 +max_mem_reservation_mb: 25000 +network_interfaces: + - driver: "MACVTAP" + device: "eno8303" + ipv4: + - subnet: "173.234.136.152/29" + gateway: "173.234.136.158" + reserved_addrs: + - "173.234.136.153" + - "173.234.136.156" + - "173.234.136.157" + - "173.234.136.158" + - subnet: "173.234.137.16/31" + gateway: "173.234.137.30" + reserved_addrs: + - "173.234.137.16" + ipv6: + - subnet: "2a0d:3003:b666:a00c:2::/112" + gateway: "2a0d:3003:b666:a00c::1" + reserved_addrs: [] +volumes: + - path: "/opt/detee_vms/" + max_reservation_gb: 200 +public_port_range: + start: 30000 + end: 50000 +max_ports_per_vm: 5 diff --git a/src/config.rs b/src/config.rs index 89c2c43..d0feab5 100644 --- a/src/config.rs +++ b/src/config.rs @@ -7,8 +7,8 @@ use std::ops::Range; #[derive(Deserialize, Debug)] pub struct Volume { - path: String, - max_reservation: u64, + pub path: String, + pub max_reservation_gb: usize, } #[derive(Deserialize, Debug)] @@ -44,7 +44,7 @@ pub enum InterfaceType { pub struct Config { pub max_cores_per_vm: usize, pub max_vcpu_reservation: usize, - pub max_mem_reservation: usize, + pub max_mem_reservation_mb: usize, pub network_interfaces: Vec, pub volumes: Vec, #[serde(with = "range_format")] diff --git a/src/constants.rs b/src/constants.rs index 6728c7f..1810165 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -2,7 +2,6 @@ pub(crate) const DEFAULT_OVMF: &str = "/usr/share/edk2/ovmf/OVMF.amdsev.fd"; pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/"; -pub(crate) const VM_DISK_DIR: &str = "/var/lib/detee/vms/"; pub(crate) const VM_CONFIG_DIR: &str = "/etc/detee/daemon/vms/"; pub(crate) const DAEMON_CONFIG_PATH: &str = "/etc/detee/daemon/config.json"; pub(crate) const START_VM_SCRIPT: &str = "/usr/local/bin/detee/start_qemu_vm.sh"; diff --git a/src/main.rs b/src/main.rs index 2cf5f14..7cf7cda 100644 --- a/src/main.rs +++ b/src/main.rs @@ -7,8 +7,8 @@ use crate::config::Config; use crate::state::NewVMRequest; fn main() -> Result<(), Box> { - let mut res = state::Resources::new(); - let config = Config::from_file("test_data/config2.yaml")?; + let config = Config::from_file("prod_setting/config1.yaml")?; + let mut res = state::Resources::new(&config.volumes); // println!("{:#?}", config); // let config = Config::from_file("test_data/config2.yaml")?; @@ -19,7 +19,7 @@ fn main() -> Result<(), Box> { // println!("{:#?}", config); // let config = Config::from_file("test_data/config5.yaml")?; // println!("{:#?}", config); - let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req2.yaml")?; + let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req3.yaml")?; // println!("{:#?}", new_vm_req); // let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req2.yaml")?; diff --git a/src/state.rs b/src/state.rs index 22f1b7b..46ddc2e 100644 --- a/src/state.rs +++ b/src/state.rs @@ -11,6 +11,7 @@ use std::fs::remove_file; use std::fs::File; use std::io::Read; use std::io::Write; +use std::net::{Ipv4Addr, Ipv6Addr}; use std::path::Path; use std::process::Command; @@ -28,18 +29,36 @@ pub struct Resources { } impl Resources { - pub fn new() -> Self { + pub fn new(config_volumes: &Vec) -> Self { + let mut storage_pools = Vec::new(); + for config_vol in config_volumes.iter() { + storage_pools.push(StoragePool { + path: config_vol.path.clone(), + // TODO: check if the storage is actualy available at that path + available_gb: config_vol.max_reservation_gb, + }); + } Resources { reserved_vcpus: 0, reserved_memory: 0, reserved_ports: HashSet::new(), - storage_pools: Vec::new(), + storage_pools, reserved_ips: HashSet::new(), reserved_if_names: HashSet::new(), boot_files: HashSet::new(), } } - fn get_free_ports(&mut self, extra_ports: usize, config: &Config) -> Vec { + + fn available_storage_pool(&mut self, required_gb: usize) -> Option { + self.storage_pools.sort_by_key(|p| p.available_gb); + let pool = self.storage_pools.last()?; + if pool.available_gb > required_gb { + return Some(pool.path.clone()); + } + None + } + + fn available_ports(&mut self, extra_ports: usize, config: &Config) -> Vec { use rand::Rng; let total_ports = extra_ports + 1; if config.public_port_range.len() < self.reserved_ports.len() + total_ports as usize { @@ -62,7 +81,7 @@ impl Resources { published_ports } - fn get_free_vm_name(&mut self) -> String { + fn available_if_name(&mut self) -> String { use rand::{distributions::Alphanumeric, Rng}; loop { let mut interface_name: String = rand::thread_rng() @@ -77,7 +96,7 @@ impl Resources { } } - fn get_free_ipv4(&mut self, config: &Config) -> Option { + fn available_ipv4(&mut self, config: &Config) -> Option { for nic in config.network_interfaces.iter() { for range in nic.ipv4.iter() { for ip in range.subnet.iter().skip(1) { @@ -87,11 +106,11 @@ impl Resources { { let if_config = match nic.driver { crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP { - name: self.get_free_vm_name(), + name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP { - name: self.get_free_vm_name(), + name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge { @@ -99,16 +118,9 @@ impl Resources { }, }; let mut ips = Vec::new(); - let mask = ip - .network() - .to_string() - .split('/') - .nth(1) - .unwrap() - .to_string(); ips.push(IPConfig { address: ip.address().to_string(), - mask, + mask: calc_ipv4_netmask(ip.address(), range.gateway), gateway: range.gateway.to_string(), }); return Some(VMNIC { if_config, ips }); @@ -120,7 +132,7 @@ impl Resources { } // TODO: refactor this garbage cause it's only one char different from the previous one - fn get_free_ipv6(&mut self, config: &Config) -> Option { + fn available_ipv6(&mut self, config: &Config) -> Option { for nic in config.network_interfaces.iter() { for range in nic.ipv6.iter() { for ip in range.subnet.iter().skip(1) { @@ -130,11 +142,11 @@ impl Resources { { let if_config = match nic.driver { crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP { - name: self.get_free_vm_name(), + name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP { - name: self.get_free_vm_name(), + name: self.available_if_name(), device: nic.device.clone(), }, crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge { @@ -142,16 +154,9 @@ impl Resources { }, }; let mut ips = Vec::new(); - let mask = ip - .network() - .to_string() - .split('/') - .nth(1) - .unwrap() - .to_string(); ips.push(IPConfig { address: ip.address().to_string(), - mask, + mask: calc_ipv6_netmask(ip.address(), range.gateway), gateway: range.gateway.to_string(), }); return Some(VMNIC { if_config, ips }); @@ -192,7 +197,7 @@ impl Resources { fn reserve_vm_resources(&mut self, vm: &VM) { self.reserved_vcpus += vm.vcpus; - self.reserved_memory += vm.memory; + self.reserved_memory += vm.memory_mb; for nic in vm.nics.iter() { if let Some(vtap) = nic.if_config.vtap_name() { self.reserved_if_names.insert(vtap); @@ -204,11 +209,18 @@ impl Resources { for (host_port, _) in vm.fw_ports.iter() { self.reserved_ports.insert(*host_port); } + + for storage_pool in self.storage_pools.iter_mut() { + if storage_pool.path == vm.storage_pool_path { + storage_pool.available_gb -= vm.disk_size_gb; + break; + } + } } fn free_vm_resources(&mut self, vm: &VM) { self.reserved_vcpus -= vm.vcpus; - self.reserved_memory -= vm.memory; + self.reserved_memory -= vm.memory_mb; for nic in vm.nics.iter() { if let Some(vtap) = nic.if_config.vtap_name() { self.reserved_if_names.remove(&vtap); @@ -226,7 +238,7 @@ impl Resources { #[derive(Debug)] pub struct StoragePool { path: String, - current_reservation: u64, + available_gb: usize, // add mechanic to detect storage tier // tier: StorageTier, } @@ -308,11 +320,12 @@ pub struct VM { // cpu_type: String, vcpus: usize, // memory in MB - memory: usize, + memory_mb: usize, // disk size in GB - disk_size: usize, + disk_size_gb: usize, kernel_sha: String, dtrfs_sha: String, + storage_pool_path: String, } #[derive(Deserialize, Debug)] @@ -323,9 +336,9 @@ pub struct NewVMRequest { extra_ports: Vec, public_ipv4: bool, public_ipv6: bool, - disk_size: usize, + disk_size_gb: usize, vcpus: usize, - memory: usize, + memory_mb: usize, kernel_url: String, kernel_sha: String, dtrfs_url: String, @@ -368,7 +381,7 @@ impl VM { if config.max_vcpu_reservation < res.reserved_vcpus.saturating_add(req.vcpus) { return Err(VMCreationErrors::NotEnoughCPU); } - if config.max_mem_reservation < res.reserved_memory.saturating_add(req.memory) { + if config.max_mem_reservation_mb < res.reserved_memory.saturating_add(req.memory_mb) { return Err(VMCreationErrors::NotEnoughMemory); } @@ -388,13 +401,13 @@ impl VM { let mut vm_nics = Vec::new(); if req.public_ipv4 { - match res.get_free_ipv4(config) { + match res.available_ipv4(config) { Some(vmnic) => vm_nics.push(vmnic), None => return Err(VMCreationErrors::IPv4NotAvailable), } } if req.public_ipv6 { - match res.get_free_ipv6(config) { + match res.available_ipv6(config) { Some(mut vmnic) => { if let Some(mut existing_vmnic) = vm_nics.pop() { if vmnic.if_config.device_name() == existing_vmnic.if_config.device_name() { @@ -417,7 +430,7 @@ impl VM { let mut host_ports: Vec = Vec::new(); let mut port_pairs: Vec<(u16, u16)> = Vec::new(); if !req.public_ipv4 { - host_ports.append(res.get_free_ports(req.extra_ports.len(), &config).as_mut()); + host_ports.append(res.available_ports(req.extra_ports.len(), &config).as_mut()); if host_ports.len() == 0 { return Err(VMCreationErrors::NotEnoughPorts); } @@ -427,18 +440,25 @@ impl VM { } } + let storage_pool_path = match res.available_storage_pool(req.disk_size_gb) { + Some(path) => path, + None => return Err(VMCreationErrors::NotEnoughStorage), + }; + let vm = VM { uuid: req.uuid, hostname: req.hostname, admin_key: req.admin_key, nics: vm_nics, vcpus: req.vcpus, - memory: req.memory, - disk_size: req.disk_size, + memory_mb: req.memory_mb, + disk_size_gb: req.disk_size_gb, kernel_sha: req.kernel_sha, dtrfs_sha: req.dtrfs_sha, fw_ports: port_pairs, + storage_pool_path, }; + res.reserve_vm_resources(&vm); Ok(vm) } @@ -477,7 +497,11 @@ impl VM { // This means we can enforce the path to the disk. // This may change in the future as the VM is allowed to have multiple disks. pub fn disk_path(&self) -> String { - VM_DISK_DIR.to_string() + &self.uuid + ".qcow2" + let dir = match self.storage_pool_path.ends_with("/") { + true => self.storage_pool_path.clone(), + false => self.storage_pool_path.clone() + "/", + }; + dir + &self.uuid + ".qcow2" } pub fn kernel_params(&self) -> String { @@ -520,19 +544,13 @@ impl VM { vars += &format!("NAT_PORT_FW={}", ports.trim_end()); } - vars += &format!( - "KERNEL={}\n", - VM_BOOT_DIR.to_string() + &self.kernel_sha - ); - vars += &format!( - "INITRD={}\n", - VM_BOOT_DIR.to_string() + &self.dtrfs_sha - ); + vars += &format!("KERNEL={}\n", VM_BOOT_DIR.to_string() + &self.kernel_sha); + vars += &format!("INITRD={}\n", VM_BOOT_DIR.to_string() + &self.dtrfs_sha); vars += &format!("PARAMS={}\n", self.kernel_params()); vars += &format!("CPU_TYPE={}\n", QEMU_VM_CPU_TYPE); vars += &format!("VCPUS={}\n", self.vcpus); - vars += &format!("MEMORY={}MB\n", self.memory); - vars += &format!("MAX_MEMORY={}MB\n", self.memory + 256); + vars += &format!("MEMORY={}MB\n", self.memory_mb); + vars += &format!("MAX_MEMORY={}MB\n", self.memory_mb + 256); vars += &format!("DISK={}\n", self.disk_path()); let mut file = File::create(VM_CONFIG_DIR.to_string() + &self.uuid)?; @@ -582,7 +600,8 @@ impl VM { contents += &format!("[Install]\n"); contents += &format!("WantedBy=multi-user.target\n"); - let mut file = File::create("/tmp/etc/systemd/system/".to_string() + &self.uuid + ".service")?; + let mut file = + File::create("/tmp/etc/systemd/system/".to_string() + &self.uuid + ".service")?; file.write_all(contents.as_bytes())?; Ok(()) } @@ -598,7 +617,7 @@ impl VM { .arg("-f") .arg("qcow2") .arg(self.disk_path()) - .arg(self.disk_size.to_string() + "G") + .arg(self.disk_size_gb.to_string() + "G") .output()?; if !result.status.success() { return Err(anyhow!( @@ -659,3 +678,41 @@ fn compute_sha256>(path: P) -> Result { let result = hasher.finalize(); Ok(format!("{:x}", result)) } + +fn calc_ipv4_netmask(ip: Ipv4Addr, gateway: Ipv4Addr) -> String { + // Convert the IPs to u32 for easier bit manipulation + let ip_u32 = u32::from(ip); + let gateway_u32 = u32::from(gateway); + + // Find the smallest common prefix + let mut prefix_len = 0; + for i in 1..=32 { + if (ip_u32 >> (32 - i)) == (gateway_u32 >> (32 - i)) { + prefix_len = i; + } else { + break; + } + } + + // Return the mask as a string + prefix_len.to_string() +} + +fn calc_ipv6_netmask(ip: Ipv6Addr, gateway: Ipv6Addr) -> String { + // Convert the IPs to u128 for easier bit manipulation + let ip_u128 = u128::from(ip); + let gateway_u128 = u128::from(gateway); + + // Find the smallest common prefix + let mut prefix_len = 0; + for i in 1..=128 { + if (ip_u128 >> (128 - i)) == (gateway_u128 >> (128 - i)) { + prefix_len = i; + } else { + break; + } + } + + // Return the mask as a string + prefix_len.to_string() +} diff --git a/test_data/config1.yaml b/test_data/config1.yaml index 1c78c19..c06ab95 100644 --- a/test_data/config1.yaml +++ b/test_data/config1.yaml @@ -1,6 +1,6 @@ max_cores_per_vm: 4 max_vcpu_reservation: 8 -max_mem_reservation: 16384 +max_mem_reservation_mb: 16384 network_interfaces: - driver: "MACVTAP" device: "eth0" @@ -18,7 +18,7 @@ network_interfaces: - "2001:db8::5678" volumes: - path: "/mnt/storage" - max_reservation: 200 + max_reservation_gb: 200 public_port_range: start: 8000 end: 9000 diff --git a/test_data/config2.yaml b/test_data/config2.yaml index 9e79f2e..4e8fe62 100644 --- a/test_data/config2.yaml +++ b/test_data/config2.yaml @@ -1,6 +1,6 @@ max_cores_per_vm: 16 max_vcpu_reservation: 32 -max_mem_reservation: 1265536 +max_mem_reservation_mb: 1265536 network_interfaces: - driver: "Bridge" device: "br0" @@ -27,12 +27,8 @@ network_interfaces: - "2001:db8:abcd:1234::dead" - "2001:db8:abcd:1234::beef" volumes: - - path: "/data/volume1" - max_reservation: 500 - - path: "/data/volume2" - max_reservation: 1000 - - path: "/backup" - max_reservation: 2000 + - path: "/etc/detee/daemon/vms/" + max_reservation_gb: 500 public_port_range: start: 10000 end: 11000 diff --git a/test_data/config3.yaml b/test_data/config3.yaml index dd61c56..7994387 100644 --- a/test_data/config3.yaml +++ b/test_data/config3.yaml @@ -1,6 +1,6 @@ max_cores_per_vm: 12 max_vcpu_reservation: 24 -max_mem_reservation: 49152 +max_mem_reservation_mb: 49152 network_interfaces: - driver: "IPVTAP" device: "tap0" @@ -13,7 +13,7 @@ network_interfaces: - "2001:db8:abcd:1234::beef" volumes: - path: "/ipv6/volume" - max_reservation: 600 + max_reservation_gb: 600 public_port_range: start: 15000 end: 16000 diff --git a/test_data/config4.yaml b/test_data/config4.yaml index 86beea4..8f699f2 100644 --- a/test_data/config4.yaml +++ b/test_data/config4.yaml @@ -1,6 +1,6 @@ max_cores_per_vm: 2 max_vcpu_reservation: 4 -max_mem_reservation: 8192 +max_mem_reservation_mb: 8192 network_interfaces: - driver: "MACVTAP" device: "eth0" @@ -11,7 +11,7 @@ network_interfaces: ipv6: [] volumes: - path: "/minimal/volume" - max_reservation: 100 + max_reservation_gb: 100 public_port_range: start: 5000 end: 5100 diff --git a/test_data/config5.yaml b/test_data/config5.yaml index 23ad6dc..db2b7c4 100644 --- a/test_data/config5.yaml +++ b/test_data/config5.yaml @@ -1,6 +1,6 @@ max_cores_per_vm: 8 max_vcpu_reservation: 16 -max_mem_reservation: 32768 +max_mem_reservation_mb: 32768 network_interfaces: - driver: "Bridge" device: "br1" @@ -14,7 +14,7 @@ network_interfaces: reserved_addrs: [] volumes: - path: "/network/volume" - max_reservation: 750 + max_reservation_gb: 750 public_port_range: start: 6000 end: 7000 diff --git a/test_data/new_vm_req1.yaml b/test_data/new_vm_req1.yaml index f6225da..ea18dc3 100644 --- a/test_data/new_vm_req1.yaml +++ b/test_data/new_vm_req1.yaml @@ -4,9 +4,9 @@ admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample" extra_ports: [ ] public_ipv4: true public_ipv6: false -disk_size: 50 +disk_size_gb: 50 vcpus: 4 -memory: 8192 +memory_mb: 8192 kernel_url: "http://pb1n.de/?d25eec" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e" dtrfs_url: "http://pb1n.de/?e46db9" diff --git a/test_data/new_vm_req2.yaml b/test_data/new_vm_req2.yaml index f087123..afc0aaa 100644 --- a/test_data/new_vm_req2.yaml +++ b/test_data/new_vm_req2.yaml @@ -4,9 +4,9 @@ admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQAnotherExampleKey" extra_ports: [] public_ipv4: false public_ipv6: false -disk_size: 10 +disk_size_gb: 10 vcpus: 1 -memory: 2048 +memory_mb: 2048 kernel_url: "http://pb1n.de/?d25eec" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e" dtrfs_url: "http://pb1n.de/?e46db9" diff --git a/test_data/new_vm_req3.yaml b/test_data/new_vm_req3.yaml index e016f92..ab5e9b4 100644 --- a/test_data/new_vm_req3.yaml +++ b/test_data/new_vm_req3.yaml @@ -4,9 +4,9 @@ admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample" extra_ports: [] public_ipv4: true public_ipv6: true -disk_size: 200 +disk_size_gb: 35 vcpus: 2 -memory: 65536 +memory_mb: 5000 kernel_url: "http://pb1n.de/?d25eec" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e" dtrfs_url: "http://pb1n.de/?e46db9" diff --git a/test_data/new_vm_req4.yaml b/test_data/new_vm_req4.yaml index 9bca679..9ba4bb4 100644 --- a/test_data/new_vm_req4.yaml +++ b/test_data/new_vm_req4.yaml @@ -4,9 +4,9 @@ admin_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAKeyExampleForTesting" extra_ports: [1234, 5678] public_ipv4: false public_ipv6: true -disk_size: 25 +disk_size_gb: 25 vcpus: 2 -memory: 4096 +memory_mb: 4096 kernel_url: "http://pb1n.de/?d25eec" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e" dtrfs_url: "http://pb1n.de/?e46db9"