testnet/snp-daemon
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

correctly assigning and storing IPs and interfaces

Browse Source
This commit is contained in:
ghe0 2024-12-12 02:08:53 +02:00
parent 64a84c48be
commit d4c5cc2634
Signed by: ghe0
GPG Key ID: 451028EE56A0FBB4
8 changed files with 110 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/constants.rs
View File

@ -2,7 +2,7 @@
pub(crate) const DEFAULT_OVMF: &str = "/usr/share/edk2/ovmf/OVMF.amdsev.fd"; pub(crate) const DEFAULT_OVMF: &str = "/usr/share/edk2/ovmf/OVMF.amdsev.fd";
pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/"; pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/";
pub(crate) const VM_DISK_DIR: &str = "/var/lib/detee/boot/"; pub(crate) const VM_DISK_DIR: &str = "/var/lib/detee/vms/";
pub(crate) const VM_CONFIG_DIR: &str = "/etc/detee/daemon/vms/"; pub(crate) const VM_CONFIG_DIR: &str = "/etc/detee/daemon/vms/";
pub(crate) const DAEMON_CONFIG_PATH: &str = "/etc/detee/daemon/config.json"; pub(crate) const DAEMON_CONFIG_PATH: &str = "/etc/detee/daemon/config.json";
pub(crate) const START_VM_SCRIPT: &str = "/usr/local/bin/detee/start_qemu_vm.sh"; pub(crate) const START_VM_SCRIPT: &str = "/usr/local/bin/detee/start_qemu_vm.sh";

26
src/main.rs
View File

@ -7,8 +7,10 @@ use crate::config::Config;
use crate::state::NewVMRequest; use crate::state::NewVMRequest;
fn main() -> Result<(), Box<dyn std::error::Error>> { fn main() -> Result<(), Box<dyn std::error::Error>> {
// let config = Config::from_file("test_data/config1.yaml")?; let mut res = state::Resources::new();
let config = Config::from_file("test_data/config2.yaml")?;
// println!("{:#?}", config); // println!("{:#?}", config);
// let config = Config::from_file("test_data/config2.yaml")?; // let config = Config::from_file("test_data/config2.yaml")?;
// println!("{:#?}", config); // println!("{:#?}", config);
// let config = Config::from_file("test_data/config3.yaml")?; // let config = Config::from_file("test_data/config3.yaml")?;
@ -17,13 +19,21 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// println!("{:#?}", config); // println!("{:#?}", config);
// let config = Config::from_file("test_data/config5.yaml")?; // let config = Config::from_file("test_data/config5.yaml")?;
// println!("{:#?}", config); // println!("{:#?}", config);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req1.yaml")?;
println!("{:#?}", new_vm_req);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req2.yaml")?;
println!("{:#?}", new_vm_req);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req3.yaml")?;
println!("{:#?}", new_vm_req);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req4.yaml")?; let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req4.yaml")?;
println!("{:#?}", new_vm_req); // println!("{:#?}", new_vm_req);
// let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req2.yaml")?;
// println!("{:#?}", new_vm_req);
// let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req3.yaml")?;
// println!("{:#?}", new_vm_req);
// let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req4.yaml")?;
// println!("{:#?}", new_vm_req);
let vm = state::VM::new(new_vm_req, &config, &mut res);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req4.yaml")?;
let vm = state::VM::new(new_vm_req, &config, &mut res);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req4.yaml")?;
let vm = state::VM::new(new_vm_req, &config, &mut res);
println!("vm: {:#?}", vm);
println!("res: {:#?}", res);
Ok(()) Ok(())
} }

100
src/state.rs
View File

@ -3,6 +3,7 @@ use crate::config::Config;
use crate::constants::*; use crate::constants::*;
use anyhow::anyhow; use anyhow::anyhow;
use anyhow::Result; use anyhow::Result;
use serde::Deserialize;
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
use std::collections::HashSet; use std::collections::HashSet;
use std::fs; use std::fs;
@ -12,8 +13,8 @@ use std::io::Read;
use std::io::Write; use std::io::Write;
use std::path::Path; use std::path::Path;
use std::process::Command; use std::process::Command;
use serde::Deserialize;
#[derive(Debug)]
pub struct Resources { pub struct Resources {
// QEMU does not support MHz limiation // QEMU does not support MHz limiation
reserved_vcpus: usize, reserved_vcpus: usize,
@ -27,7 +28,18 @@ pub struct Resources {
} }
impl Resources { impl Resources {
fn reserve_ports(&mut self, extra_ports: usize, config: &Config) -> Vec<u16> { pub fn new() -> Self {
Resources {
reserved_vcpus: 0,
reserved_memory: 0,
reserved_ports: HashSet::new(),
storage_pools: Vec::new(),
reserved_ips: HashSet::new(),
reserved_if_names: HashSet::new(),
boot_files: HashSet::new(),
}
}
fn get_free_ports(&mut self, extra_ports: usize, config: &Config) -> Vec<u16> {
use rand::Rng; use rand::Rng;
let total_ports = extra_ports + 1; let total_ports = extra_ports + 1;
if config.public_port_range.len() < self.reserved_ports.len() + total_ports as usize { if config.public_port_range.len() < self.reserved_ports.len() + total_ports as usize {
@ -40,7 +52,7 @@ impl Resources {
for _ in 0..total_ports { for _ in 0..total_ports {
for _ in 0..5 { for _ in 0..5 {
let port = rand::thread_rng().gen_range(config.public_port_range.clone()); let port = rand::thread_rng().gen_range(config.public_port_range.clone());
if self.reserved_ports.insert(port) { if self.reserved_ports.get(&port).is_none() {
published_ports.push(port); published_ports.push(port);
} }
break; break;
@ -50,7 +62,7 @@ impl Resources {
published_ports published_ports
} }
fn reserve_vm_if(&mut self) -> String { fn get_free_vm_name(&mut self) -> String {
use rand::{distributions::Alphanumeric, Rng}; use rand::{distributions::Alphanumeric, Rng};
loop { loop {
let mut interface_name: String = rand::thread_rng() let mut interface_name: String = rand::thread_rng()
@ -65,20 +77,21 @@ impl Resources {
} }
} }
fn reserve_public_ipv4(&mut self, config: &Config) -> Option<VMNIC> { fn get_free_ipv4(&mut self, config: &Config) -> Option<VMNIC> {
for nic in config.network_interfaces.iter() { for nic in config.network_interfaces.iter() {
for range in nic.ipv4.iter() { for range in nic.ipv4.iter() {
for ip in range.subnet.iter() { for ip in range.subnet.iter().skip(1) {
if !range.reserved_addrs.contains(&ip.address()) if !range.reserved_addrs.contains(&ip.address())
&& !self.reserved_ips.contains(&ip.to_string()) && !self.reserved_ips.contains(&ip.address().to_string())
&& ip.address() != range.gateway
{ {
let if_config = match nic.driver { let if_config = match nic.driver {
crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP { crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP {
name: self.reserve_vm_if(), name: self.get_free_vm_name(),
device: nic.device.clone(), device: nic.device.clone(),
}, },
crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP { crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP {
name: self.reserve_vm_if(), name: self.get_free_vm_name(),
device: nic.device.clone(), device: nic.device.clone(),
}, },
crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge { crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge {
@ -90,12 +103,12 @@ impl Resources {
.network() .network()
.to_string() .to_string()
.split('/') .split('/')
.next() .nth(1)
.unwrap() .unwrap()
.to_string(); .to_string();
ips.push(IPConfig { ips.push(IPConfig {
address: ip.address().to_string(), address: ip.address().to_string(),
subnet: mask, mask,
gateway: range.gateway.to_string(), gateway: range.gateway.to_string(),
}); });
return Some(VMNIC { if_config, ips }); return Some(VMNIC { if_config, ips });
@ -107,20 +120,21 @@ impl Resources {
} }
// TODO: refactor this garbage cause it's only one char different from the previous one // TODO: refactor this garbage cause it's only one char different from the previous one
fn reserve_public_ipv6(&mut self, config: &Config) -> Option<VMNIC> { fn get_free_ipv6(&mut self, config: &Config) -> Option<VMNIC> {
for nic in config.network_interfaces.iter() { for nic in config.network_interfaces.iter() {
for range in nic.ipv6.iter() { for range in nic.ipv6.iter() {
for ip in range.subnet.iter() { for ip in range.subnet.iter().skip(1) {
if !range.reserved_addrs.contains(&ip.address()) if !range.reserved_addrs.contains(&ip.address())
&& !self.reserved_ips.contains(&ip.to_string()) && !self.reserved_ips.contains(&ip.address().to_string())
&& ip.address() != range.gateway
{ {
let if_config = match nic.driver { let if_config = match nic.driver {
crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP { crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP {
name: self.reserve_vm_if(), name: self.get_free_vm_name(),
device: nic.device.clone(), device: nic.device.clone(),
}, },
crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP { crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP {
name: self.reserve_vm_if(), name: self.get_free_vm_name(),
device: nic.device.clone(), device: nic.device.clone(),
}, },
crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge { crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge {
@ -132,12 +146,12 @@ impl Resources {
.network() .network()
.to_string() .to_string()
.split('/') .split('/')
.next() .nth(1)
.unwrap() .unwrap()
.to_string(); .to_string();
ips.push(IPConfig { ips.push(IPConfig {
address: ip.address().to_string(), address: ip.address().to_string(),
subnet: mask, mask,
gateway: range.gateway.to_string(), gateway: range.gateway.to_string(),
}); });
return Some(VMNIC { if_config, ips }); return Some(VMNIC { if_config, ips });
@ -175,8 +189,25 @@ impl Resources {
self.boot_files.insert(sha); self.boot_files.insert(sha);
Ok(()) Ok(())
} }
fn reserve_vm_resources(&mut self, vm: &VM) {
self.reserved_vcpus += vm.vcpus;
self.reserved_memory += vm.memory;
for nic in vm.nics.iter() {
if let Some(vtap) = nic.if_config.vtap_name() {
self.reserved_if_names.insert(vtap);
}
for ip in nic.ips.iter() {
self.reserved_ips.insert(ip.address.clone());
}
}
for (host_port, _) in vm.fw_ports.iter() {
self.reserved_ports.insert(*host_port);
}
}
} }
#[derive(Debug)]
pub struct StoragePool { pub struct StoragePool {
path: String, path: String,
current_reservation: u64, current_reservation: u64,
@ -184,6 +215,7 @@ pub struct StoragePool {
// tier: StorageTier, // tier: StorageTier,
} }
#[derive(Debug)]
pub enum InterfaceConfig { pub enum InterfaceConfig {
// TODO: instead of QEMU userspace NAT, use iptables kernelspace NAT // TODO: instead of QEMU userspace NAT, use iptables kernelspace NAT
// in case of QEMU-base NAT, device name is not needed // in case of QEMU-base NAT, device name is not needed
@ -229,13 +261,15 @@ impl InterfaceConfig {
} }
} }
#[derive(Debug)]
struct IPConfig { struct IPConfig {
address: String, address: String,
// requires short format (example: 24) // requires short format (example: 24)
subnet: String, mask: String,
gateway: String, gateway: String,
} }
#[derive(Debug)]
pub struct VMNIC { pub struct VMNIC {
if_config: InterfaceConfig, if_config: InterfaceConfig,
ips: Vec<IPConfig>, ips: Vec<IPConfig>,
@ -247,6 +281,7 @@ impl VMNIC {
} }
} }
#[derive(Debug)]
pub struct VM { pub struct VM {
uuid: String, uuid: String,
hostname: String, hostname: String,
@ -289,6 +324,7 @@ impl NewVMRequest {
} }
} }
#[derive(Debug)]
pub enum VMCreationErrors { pub enum VMCreationErrors {
NATandIPv4Conflict, NATandIPv4Conflict,
TooManyCores, TooManyCores,
@ -336,13 +372,13 @@ impl VM {
let mut vm_nics = Vec::new(); let mut vm_nics = Vec::new();
if req.public_ipv4 { if req.public_ipv4 {
match res.reserve_public_ipv4(config) { match res.get_free_ipv4(config) {
Some(vmnic) => vm_nics.push(vmnic), Some(vmnic) => vm_nics.push(vmnic),
None => return Err(VMCreationErrors::IPv4NotAvailable), None => return Err(VMCreationErrors::IPv4NotAvailable),
} }
} }
if req.public_ipv6 { if req.public_ipv6 {
match res.reserve_public_ipv4(config) { match res.get_free_ipv6(config) {
Some(mut vmnic) => { Some(mut vmnic) => {
if let Some(mut existing_vmnic) = vm_nics.pop() { if let Some(mut existing_vmnic) = vm_nics.pop() {
if vmnic.if_config.device_name() == existing_vmnic.if_config.device_name() { if vmnic.if_config.device_name() == existing_vmnic.if_config.device_name() {
@ -357,11 +393,6 @@ impl VM {
} }
} }
None => { None => {
if let Some(existing_vmnic) = vm_nics.pop() {
for ip in existing_vmnic.ips {
res.reserved_ips.remove(&ip.address);
}
}
return Err(VMCreationErrors::IPv4NotAvailable); return Err(VMCreationErrors::IPv4NotAvailable);
} }
} }
@ -370,22 +401,17 @@ impl VM {
let mut host_ports: Vec<u16> = Vec::new(); let mut host_ports: Vec<u16> = Vec::new();
let mut port_pairs: Vec<(u16, u16)> = Vec::new(); let mut port_pairs: Vec<(u16, u16)> = Vec::new();
if !req.public_ipv4 { if !req.public_ipv4 {
host_ports.append(res.reserve_ports(req.extra_ports.len(), &config).as_mut()); host_ports.append(res.get_free_ports(req.extra_ports.len(), &config).as_mut());
if host_ports.len() == 0 { if host_ports.len() == 0 {
for nic in vm_nics {
for ip in nic.ips {
res.reserved_ips.remove(&ip.address);
}
}
return Err(VMCreationErrors::NotEnoughPorts); return Err(VMCreationErrors::NotEnoughPorts);
} }
port_pairs.push((host_ports[0], 22)); port_pairs.push((host_ports[0], 22));
for i in 0..req.extra_ports.len() { for i in 0..req.extra_ports.len() {
port_pairs.push((host_ports[i+1], req.extra_ports[i])); port_pairs.push((host_ports[i + 1], req.extra_ports[i]));
} }
} }
Ok(VM { let vm = VM {
uuid: req.uuid, uuid: req.uuid,
hostname: req.hostname, hostname: req.hostname,
admin_key: req.admin_key, admin_key: req.admin_key,
@ -396,7 +422,9 @@ impl VM {
kernel_sha: req.kernel_sha, kernel_sha: req.kernel_sha,
dtrfs_sha: req.dtrfs_sha, dtrfs_sha: req.dtrfs_sha,
fw_ports: port_pairs, fw_ports: port_pairs,
}) };
res.reserve_vm_resources(&vm);
Ok(vm)
} }
pub fn deploy(&self) -> Result<()> { pub fn deploy(&self) -> Result<()> {
@ -430,7 +458,7 @@ impl VM {
for ip in nic.ips.iter() { for ip in nic.ips.iter() {
ip_string += &format!( ip_string += &format!(
"detee_net_eth{}={}_{}_{}", "detee_net_eth{}={}_{}_{}",
i, ip.address, ip.subnet, ip.gateway i, ip.address, ip.mask, ip.gateway
); );
} }
i += 1; i += 1;

15
test_data/config2.yaml
View File

@ -1,6 +1,6 @@
max_cores_per_vm: 16 max_cores_per_vm: 16
max_vcpu_reservation: 32 max_vcpu_reservation: 32
max_mem_reservation: 65536 max_mem_reservation: 1265536
network_interfaces: network_interfaces:
- driver: "Bridge" - driver: "Bridge"
device: "br0" device: "br0"
@ -11,11 +11,7 @@ network_interfaces:
- "10.0.0.100" - "10.0.0.100"
- "10.0.0.101" - "10.0.0.101"
- "10.0.0.102" - "10.0.0.102"
ipv6: ipv6: []
- subnet: "fd00::/48"
gateway: "fd00::1"
reserved_addrs:
- "fd00::1000"
- driver: "IPVTAP" - driver: "IPVTAP"
device: "tap1" device: "tap1"
ipv4: ipv4:
@ -24,7 +20,12 @@ network_interfaces:
reserved_addrs: reserved_addrs:
- "172.16.0.10" - "172.16.0.10"
- "172.16.0.11" - "172.16.0.11"
ipv6: [] ipv6:
- subnet: "2001:db8:abcd:1234::/64"
gateway: "2001:db8:abcd:1234::1"
reserved_addrs:
- "2001:db8:abcd:1234::dead"
- "2001:db8:abcd:1234::beef"
volumes: volumes:
- path: "/data/volume1" - path: "/data/volume1"
max_reservation: 500 max_reservation: 500

12
test_data/new_vm_req1.yaml
View File

@ -1,15 +1,13 @@
uuid: "123e4567-e89b-12d3-a456-426614174000" uuid: "123e4567-e89b-12d3-a456-426614174000"
hostname: "test-vm-01" hostname: "test-vm-01"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample" admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample"
extra_ports: extra_ports: [ ]
- 8080
- 8443
public_ipv4: true public_ipv4: true
public_ipv6: false public_ipv6: false
disk_size: 50 disk_size: 50
vcpus: 4 vcpus: 4
memory: 8192 memory: 8192
kernel_url: "http://example.com/kernel" kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "abc123def4567890ghij" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://example.com/dtrfs" dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "xyz9876543210mnop" dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

9
test_data/new_vm_req2.yaml
View File

@ -7,8 +7,7 @@ public_ipv6: false
disk_size: 10 disk_size: 10
vcpus: 1 vcpus: 1
memory: 2048 memory: 2048
kernel_url: "http://minimal.com/kernel" kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "minimalsha123" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://minimal.com/dtrfs" dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "dtrfssha456" dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

17
test_data/new_vm_req3.yaml
View File

@ -1,19 +1,14 @@
uuid: "246e1357-e98b-76d3-f345-129874650000" uuid: "246e1357-e98b-76d3-f345-129874650000"
hostname: "extensive-vm" hostname: "extensive-vm"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample" admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample"
extra_ports: extra_ports: []
- 80
- 443
- 3000
- 5000
- 6000
public_ipv4: true public_ipv4: true
public_ipv6: true public_ipv6: true
disk_size: 200 disk_size: 200
vcpus: 16 vcpus: 2
memory: 65536 memory: 65536
kernel_url: "http://large.com/kernel" kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "largekernelsha" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://large.com/dtrfs" dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "largedtrfssha" dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

9
test_data/new_vm_req4.yaml
View File

@ -7,8 +7,7 @@ public_ipv6: true
disk_size: 25 disk_size: 25
vcpus: 2 vcpus: 2
memory: 4096 memory: 4096
kernel_url: "http://testing.com/kernel" kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "testkernelsha" kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://testing.com/dtrfs" dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "testdtrfssha" dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"