testnet/snp-daemon
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

prepared prod server config

Browse Source
This commit is contained in:
ghe0 2024-12-12 22:38:50 +02:00
parent df34fc719c
commit 98459d27ca
Signed by: ghe0
GPG Key ID: 451028EE56A0FBB4
14 changed files with 163 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
prod_setting/config1.yaml Normal file
View File

@ -0,0 +1,29 @@
max_cores_per_vm: 4
max_vcpu_reservation: 8
max_mem_reservation_mb: 25000
network_interfaces:
- driver: "MACVTAP"
device: "eno8303"
ipv4:
- subnet: "173.234.136.152/29"
gateway: "173.234.136.158"
reserved_addrs:
- "173.234.136.153"
- "173.234.136.156"
- "173.234.136.157"
- "173.234.136.158"
- subnet: "173.234.137.16/31"
gateway: "173.234.137.30"
reserved_addrs:
- "173.234.137.16"
ipv6:
- subnet: "2a0d:3003:b666:a00c:2::/112"
gateway: "2a0d:3003:b666:a00c::1"
reserved_addrs: []
volumes:
- path: "/opt/detee_vms/"
max_reservation_gb: 200
public_port_range:
start: 30000
end: 50000
max_ports_per_vm: 5

6
src/config.rs
View File

@ -7,8 +7,8 @@ use std::ops::Range;
#[derive(Deserialize, Debug)]
pub struct Volume {
path: String,
max_reservation: u64,
pub path: String,
pub max_reservation_gb: usize,
}
#[derive(Deserialize, Debug)]
@ -44,7 +44,7 @@ pub enum InterfaceType {
pub struct Config {
pub max_cores_per_vm: usize,
pub max_vcpu_reservation: usize,
pub max_mem_reservation: usize,
pub max_mem_reservation_mb: usize,
pub network_interfaces: Vec<Interface>,
pub volumes: Vec<Volume>,
#[serde(with = "range_format")]

1
src/constants.rs
View File

@ -2,7 +2,6 @@
pub(crate) const DEFAULT_OVMF: &str = "/usr/share/edk2/ovmf/OVMF.amdsev.fd";
pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/";
pub(crate) const VM_DISK_DIR: &str = "/var/lib/detee/vms/";
pub(crate) const VM_CONFIG_DIR: &str = "/etc/detee/daemon/vms/";
pub(crate) const DAEMON_CONFIG_PATH: &str = "/etc/detee/daemon/config.json";
pub(crate) const START_VM_SCRIPT: &str = "/usr/local/bin/detee/start_qemu_vm.sh";

6
src/main.rs
View File

@ -7,8 +7,8 @@ use crate::config::Config;
use crate::state::NewVMRequest;
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut res = state::Resources::new();
let config = Config::from_file("test_data/config2.yaml")?;
let config = Config::from_file("prod_setting/config1.yaml")?;
let mut res = state::Resources::new(&config.volumes);
// println!("{:#?}", config);
// let config = Config::from_file("test_data/config2.yaml")?;
@ -19,7 +19,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
// println!("{:#?}", config);
// let config = Config::from_file("test_data/config5.yaml")?;
// println!("{:#?}", config);
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req2.yaml")?;
let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req3.yaml")?;
// println!("{:#?}", new_vm_req);
// let new_vm_req = NewVMRequest::from_file("test_data/new_vm_req2.yaml")?;

161
src/state.rs
View File

@ -11,6 +11,7 @@ use std::fs::remove_file;
use std::fs::File;
use std::io::Read;
use std::io::Write;
use std::net::{Ipv4Addr, Ipv6Addr};
use std::path::Path;
use std::process::Command;
@ -28,18 +29,36 @@ pub struct Resources {
}
impl Resources {
pub fn new() -> Self {
pub fn new(config_volumes: &Vec<crate::config::Volume>) -> Self {
let mut storage_pools = Vec::new();
for config_vol in config_volumes.iter() {
storage_pools.push(StoragePool {
path: config_vol.path.clone(),
// TODO: check if the storage is actualy available at that path
available_gb: config_vol.max_reservation_gb,
});
}
Resources {
reserved_vcpus: 0,
reserved_memory: 0,
reserved_ports: HashSet::new(),
storage_pools: Vec::new(),
storage_pools,
reserved_ips: HashSet::new(),
reserved_if_names: HashSet::new(),
boot_files: HashSet::new(),
}
}
fn get_free_ports(&mut self, extra_ports: usize, config: &Config) -> Vec<u16> {
fn available_storage_pool(&mut self, required_gb: usize) -> Option<String> {
self.storage_pools.sort_by_key(|p| p.available_gb);
let pool = self.storage_pools.last()?;
if pool.available_gb > required_gb {
return Some(pool.path.clone());
}
None
}
fn available_ports(&mut self, extra_ports: usize, config: &Config) -> Vec<u16> {
use rand::Rng;
let total_ports = extra_ports + 1;
if config.public_port_range.len() < self.reserved_ports.len() + total_ports as usize {
@ -62,7 +81,7 @@ impl Resources {
published_ports
}
fn get_free_vm_name(&mut self) -> String {
fn available_if_name(&mut self) -> String {
use rand::{distributions::Alphanumeric, Rng};
loop {
let mut interface_name: String = rand::thread_rng()
@ -77,7 +96,7 @@ impl Resources {
}
}
fn get_free_ipv4(&mut self, config: &Config) -> Option<VMNIC> {
fn available_ipv4(&mut self, config: &Config) -> Option<VMNIC> {
for nic in config.network_interfaces.iter() {
for range in nic.ipv4.iter() {
for ip in range.subnet.iter().skip(1) {
@ -87,11 +106,11 @@ impl Resources {
{
let if_config = match nic.driver {
crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP {
name: self.get_free_vm_name(),
name: self.available_if_name(),
device: nic.device.clone(),
},
crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP {
name: self.get_free_vm_name(),
name: self.available_if_name(),
device: nic.device.clone(),
},
crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge {
@ -99,16 +118,9 @@ impl Resources {
},
};
let mut ips = Vec::new();
let mask = ip
.network()
.to_string()
.split('/')
.nth(1)
.unwrap()
.to_string();
ips.push(IPConfig {
address: ip.address().to_string(),
mask,
mask: calc_ipv4_netmask(ip.address(), range.gateway),
gateway: range.gateway.to_string(),
});
return Some(VMNIC { if_config, ips });
@ -120,7 +132,7 @@ impl Resources {
}
// TODO: refactor this garbage cause it's only one char different from the previous one
fn get_free_ipv6(&mut self, config: &Config) -> Option<VMNIC> {
fn available_ipv6(&mut self, config: &Config) -> Option<VMNIC> {
for nic in config.network_interfaces.iter() {
for range in nic.ipv6.iter() {
for ip in range.subnet.iter().skip(1) {
@ -130,11 +142,11 @@ impl Resources {
{
let if_config = match nic.driver {
crate::config::InterfaceType::MACVTAP => InterfaceConfig::MACVTAP {
name: self.get_free_vm_name(),
name: self.available_if_name(),
device: nic.device.clone(),
},
crate::config::InterfaceType::IPVTAP => InterfaceConfig::IPVTAP {
name: self.get_free_vm_name(),
name: self.available_if_name(),
device: nic.device.clone(),
},
crate::config::InterfaceType::Bridge => InterfaceConfig::Bridge {
@ -142,16 +154,9 @@ impl Resources {
},
};
let mut ips = Vec::new();
let mask = ip
.network()
.to_string()
.split('/')
.nth(1)
.unwrap()
.to_string();
ips.push(IPConfig {
address: ip.address().to_string(),
mask,
mask: calc_ipv6_netmask(ip.address(), range.gateway),
gateway: range.gateway.to_string(),
});
return Some(VMNIC { if_config, ips });
@ -192,7 +197,7 @@ impl Resources {
fn reserve_vm_resources(&mut self, vm: &VM) {
self.reserved_vcpus += vm.vcpus;
self.reserved_memory += vm.memory;
self.reserved_memory += vm.memory_mb;
for nic in vm.nics.iter() {
if let Some(vtap) = nic.if_config.vtap_name() {
self.reserved_if_names.insert(vtap);
@ -204,11 +209,18 @@ impl Resources {
for (host_port, _) in vm.fw_ports.iter() {
self.reserved_ports.insert(*host_port);
}
for storage_pool in self.storage_pools.iter_mut() {
if storage_pool.path == vm.storage_pool_path {
storage_pool.available_gb -= vm.disk_size_gb;
break;
}
}
}
fn free_vm_resources(&mut self, vm: &VM) {
self.reserved_vcpus -= vm.vcpus;
self.reserved_memory -= vm.memory;
self.reserved_memory -= vm.memory_mb;
for nic in vm.nics.iter() {
if let Some(vtap) = nic.if_config.vtap_name() {
self.reserved_if_names.remove(&vtap);
@ -226,7 +238,7 @@ impl Resources {
#[derive(Debug)]
pub struct StoragePool {
path: String,
current_reservation: u64,
available_gb: usize,
// add mechanic to detect storage tier
// tier: StorageTier,
}
@ -308,11 +320,12 @@ pub struct VM {
// cpu_type: String,
vcpus: usize,
// memory in MB
memory: usize,
memory_mb: usize,
// disk size in GB
disk_size: usize,
disk_size_gb: usize,
kernel_sha: String,
dtrfs_sha: String,
storage_pool_path: String,
}
#[derive(Deserialize, Debug)]
@ -323,9 +336,9 @@ pub struct NewVMRequest {
extra_ports: Vec<u16>,
public_ipv4: bool,
public_ipv6: bool,
disk_size: usize,
disk_size_gb: usize,
vcpus: usize,
memory: usize,
memory_mb: usize,
kernel_url: String,
kernel_sha: String,
dtrfs_url: String,
@ -368,7 +381,7 @@ impl VM {
if config.max_vcpu_reservation < res.reserved_vcpus.saturating_add(req.vcpus) {
return Err(VMCreationErrors::NotEnoughCPU);
}
if config.max_mem_reservation < res.reserved_memory.saturating_add(req.memory) {
if config.max_mem_reservation_mb < res.reserved_memory.saturating_add(req.memory_mb) {
return Err(VMCreationErrors::NotEnoughMemory);
}
@ -388,13 +401,13 @@ impl VM {
let mut vm_nics = Vec::new();
if req.public_ipv4 {
match res.get_free_ipv4(config) {
match res.available_ipv4(config) {
Some(vmnic) => vm_nics.push(vmnic),
None => return Err(VMCreationErrors::IPv4NotAvailable),
}
}
if req.public_ipv6 {
match res.get_free_ipv6(config) {
match res.available_ipv6(config) {
Some(mut vmnic) => {
if let Some(mut existing_vmnic) = vm_nics.pop() {
if vmnic.if_config.device_name() == existing_vmnic.if_config.device_name() {
@ -417,7 +430,7 @@ impl VM {
let mut host_ports: Vec<u16> = Vec::new();
let mut port_pairs: Vec<(u16, u16)> = Vec::new();
if !req.public_ipv4 {
host_ports.append(res.get_free_ports(req.extra_ports.len(), &config).as_mut());
host_ports.append(res.available_ports(req.extra_ports.len(), &config).as_mut());
if host_ports.len() == 0 {
return Err(VMCreationErrors::NotEnoughPorts);
}
@ -427,18 +440,25 @@ impl VM {
}
}
let storage_pool_path = match res.available_storage_pool(req.disk_size_gb) {
Some(path) => path,
None => return Err(VMCreationErrors::NotEnoughStorage),
};
let vm = VM {
uuid: req.uuid,
hostname: req.hostname,
admin_key: req.admin_key,
nics: vm_nics,
vcpus: req.vcpus,
memory: req.memory,
disk_size: req.disk_size,
memory_mb: req.memory_mb,
disk_size_gb: req.disk_size_gb,
kernel_sha: req.kernel_sha,
dtrfs_sha: req.dtrfs_sha,
fw_ports: port_pairs,
storage_pool_path,
};
res.reserve_vm_resources(&vm);
Ok(vm)
}
@ -477,7 +497,11 @@ impl VM {
// This means we can enforce the path to the disk.
// This may change in the future as the VM is allowed to have multiple disks.
pub fn disk_path(&self) -> String {
VM_DISK_DIR.to_string() + &self.uuid + ".qcow2"
let dir = match self.storage_pool_path.ends_with("/") {
true => self.storage_pool_path.clone(),
false => self.storage_pool_path.clone() + "/",
};
dir + &self.uuid + ".qcow2"
}
pub fn kernel_params(&self) -> String {
@ -520,19 +544,13 @@ impl VM {
vars += &format!("NAT_PORT_FW={}", ports.trim_end());
}
vars += &format!(
"KERNEL={}\n",
VM_BOOT_DIR.to_string() + &self.kernel_sha
);
vars += &format!(
"INITRD={}\n",
VM_BOOT_DIR.to_string() + &self.dtrfs_sha
);
vars += &format!("KERNEL={}\n", VM_BOOT_DIR.to_string() + &self.kernel_sha);
vars += &format!("INITRD={}\n", VM_BOOT_DIR.to_string() + &self.dtrfs_sha);
vars += &format!("PARAMS={}\n", self.kernel_params());
vars += &format!("CPU_TYPE={}\n", QEMU_VM_CPU_TYPE);
vars += &format!("VCPUS={}\n", self.vcpus);
vars += &format!("MEMORY={}MB\n", self.memory);
vars += &format!("MAX_MEMORY={}MB\n", self.memory + 256);
vars += &format!("MEMORY={}MB\n", self.memory_mb);
vars += &format!("MAX_MEMORY={}MB\n", self.memory_mb + 256);
vars += &format!("DISK={}\n", self.disk_path());
let mut file = File::create(VM_CONFIG_DIR.to_string() + &self.uuid)?;
@ -582,7 +600,8 @@ impl VM {
contents += &format!("[Install]\n");
contents += &format!("WantedBy=multi-user.target\n");
let mut file = File::create("/tmp/etc/systemd/system/".to_string() + &self.uuid + ".service")?;
let mut file =
File::create("/tmp/etc/systemd/system/".to_string() + &self.uuid + ".service")?;
file.write_all(contents.as_bytes())?;
Ok(())
}
@ -598,7 +617,7 @@ impl VM {
.arg("-f")
.arg("qcow2")
.arg(self.disk_path())
.arg(self.disk_size.to_string() + "G")
.arg(self.disk_size_gb.to_string() + "G")
.output()?;
if !result.status.success() {
return Err(anyhow!(
@ -659,3 +678,41 @@ fn compute_sha256<P: AsRef<Path>>(path: P) -> Result<String> {
let result = hasher.finalize();
Ok(format!("{:x}", result))
}
fn calc_ipv4_netmask(ip: Ipv4Addr, gateway: Ipv4Addr) -> String {
// Convert the IPs to u32 for easier bit manipulation
let ip_u32 = u32::from(ip);
let gateway_u32 = u32::from(gateway);
// Find the smallest common prefix
let mut prefix_len = 0;
for i in 1..=32 {
if (ip_u32 >> (32 - i)) == (gateway_u32 >> (32 - i)) {
prefix_len = i;
} else {
break;
}
}
// Return the mask as a string
prefix_len.to_string()
}
fn calc_ipv6_netmask(ip: Ipv6Addr, gateway: Ipv6Addr) -> String {
// Convert the IPs to u128 for easier bit manipulation
let ip_u128 = u128::from(ip);
let gateway_u128 = u128::from(gateway);
// Find the smallest common prefix
let mut prefix_len = 0;
for i in 1..=128 {
if (ip_u128 >> (128 - i)) == (gateway_u128 >> (128 - i)) {
prefix_len = i;
} else {
break;
}
}
// Return the mask as a string
prefix_len.to_string()
}

4
test_data/config1.yaml
View File

@ -1,6 +1,6 @@
max_cores_per_vm: 4
max_vcpu_reservation: 8
max_mem_reservation: 16384
max_mem_reservation_mb: 16384
network_interfaces:
- driver: "MACVTAP"
device: "eth0"
@ -18,7 +18,7 @@ network_interfaces:
- "2001:db8::5678"
volumes:
- path: "/mnt/storage"
max_reservation: 200
max_reservation_gb: 200
public_port_range:
start: 8000
end: 9000

10
test_data/config2.yaml
View File

@ -1,6 +1,6 @@
max_cores_per_vm: 16
max_vcpu_reservation: 32
max_mem_reservation: 1265536
max_mem_reservation_mb: 1265536
network_interfaces:
- driver: "Bridge"
device: "br0"
@ -27,12 +27,8 @@ network_interfaces:
- "2001:db8:abcd:1234::dead"
- "2001:db8:abcd:1234::beef"
volumes:
- path: "/data/volume1"
max_reservation: 500
- path: "/data/volume2"
max_reservation: 1000
- path: "/backup"
max_reservation: 2000
- path: "/etc/detee/daemon/vms/"
max_reservation_gb: 500
public_port_range:
start: 10000
end: 11000

4
test_data/config3.yaml
View File

@ -1,6 +1,6 @@
max_cores_per_vm: 12
max_vcpu_reservation: 24
max_mem_reservation: 49152
max_mem_reservation_mb: 49152
network_interfaces:
- driver: "IPVTAP"
device: "tap0"
@ -13,7 +13,7 @@ network_interfaces:
- "2001:db8:abcd:1234::beef"
volumes:
- path: "/ipv6/volume"
max_reservation: 600
max_reservation_gb: 600
public_port_range:
start: 15000
end: 16000

4
test_data/config4.yaml
View File

@ -1,6 +1,6 @@
max_cores_per_vm: 2
max_vcpu_reservation: 4
max_mem_reservation: 8192
max_mem_reservation_mb: 8192
network_interfaces:
- driver: "MACVTAP"
device: "eth0"
@ -11,7 +11,7 @@ network_interfaces:
ipv6: []
volumes:
- path: "/minimal/volume"
max_reservation: 100
max_reservation_gb: 100
public_port_range:
start: 5000
end: 5100

4
test_data/config5.yaml
View File

@ -1,6 +1,6 @@
max_cores_per_vm: 8
max_vcpu_reservation: 16
max_mem_reservation: 32768
max_mem_reservation_mb: 32768
network_interfaces:
- driver: "Bridge"
device: "br1"
@ -14,7 +14,7 @@ network_interfaces:
reserved_addrs: []
volumes:
- path: "/network/volume"
max_reservation: 750
max_reservation_gb: 750
public_port_range:
start: 6000
end: 7000

4
test_data/new_vm_req1.yaml
View File

@ -4,9 +4,9 @@ admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample"
extra_ports: [ ]
public_ipv4: true
public_ipv6: false
disk_size: 50
disk_size_gb: 50
vcpus: 4
memory: 8192
memory_mb: 8192
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"

4
test_data/new_vm_req2.yaml
View File

@ -4,9 +4,9 @@ admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQAnotherExampleKey"
extra_ports: []
public_ipv4: false
public_ipv6: false
disk_size: 10
disk_size_gb: 10
vcpus: 1
memory: 2048
memory_mb: 2048
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"

4
test_data/new_vm_req3.yaml
View File

@ -4,9 +4,9 @@ admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample"
extra_ports: []
public_ipv4: true
public_ipv6: true
disk_size: 200
disk_size_gb: 35
vcpus: 2
memory: 65536
memory_mb: 5000
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"

4
test_data/new_vm_req4.yaml
View File

@ -4,9 +4,9 @@ admin_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAKeyExampleForTesting"
extra_ports: [1234, 5678]
public_ipv4: false
public_ipv6: true
disk_size: 25
disk_size_gb: 25
vcpus: 2
memory: 4096
memory_mb: 4096
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"