testnet/snp-daemon
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: testnet:75b25ab7d311c6c7ae7013219cc9f8ea22c3fdba
Branches Tags
testnet:main
testnet:remove_uuid
..
compare: testnet:1a4cec421b78593379dd33f9541d0e69b288c79b
Branches Tags
testnet:main
testnet:remove_uuid

No commits in common. "75b25ab7d311c6c7ae7013219cc9f8ea22c3fdba" and "1a4cec421b78593379dd33f9541d0e69b288c79b" have entirely different histories.
75b25ab7d3 ... 1a4cec421b

26 changed files with 550 additions and 612 deletions
Show Stats Expand all files Collapse all files

71
Cargo.lock generated
View File

@ -26,21 +26,6 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.18" version = "0.6.18"
@ -272,20 +257,6 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"wasm-bindgen",
"windows-targets",
]
[[package]] [[package]]
name = "colorchoice" name = "colorchoice"
version = "1.0.3" version = "1.0.3"
@ -376,7 +347,6 @@ version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"bs58", "bs58",
"chrono",
"ed25519-dalek", "ed25519-dalek",
"env_logger", "env_logger",
"lazy_static", "lazy_static",
@ -796,29 +766,6 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "iana-time-zone"
version = "0.1.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]] [[package]]
name = "icu_collections" name = "icu_collections"
version = "1.5.0" version = "1.5.0"
@ -1106,15 +1053,6 @@ dependencies = [
"tempfile", "tempfile",
] ]
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]] [[package]]
name = "object" name = "object"
version = "0.36.5" version = "0.36.5"
@ -2145,15 +2083,6 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "windows-core"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
"windows-targets",
]
[[package]] [[package]]
name = "windows-registry" name = "windows-registry"
version = "0.2.0" version = "0.2.0"

7
Cargo.toml
View File

@ -22,13 +22,6 @@ tokio-stream = "0.1.17"
tonic = "0.12" tonic = "0.12"
serde_json = "1.0.135" serde_json = "1.0.135"
bs58 = "0.5.1" bs58 = "0.5.1"
chrono = "0.4.39"
[build-dependencies] [build-dependencies]
tonic-build = "0.12" tonic-build = "0.12"
[profile.release]
opt-level = "z"
lto = true
codegen-units = 1
strip = true

2
build.rs
View File

@ -1,6 +1,6 @@
fn main() { fn main() {
tonic_build::configure() tonic_build::configure()
.build_server(true) .build_server(true)
.compile_protos(&["vm.proto"], &["proto"]) .compile_protos(&["snp.proto"], &["proto"])
.unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e)); .unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e));
} }

13
prod_setting/8443_and_ipv6.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "uuid-0001"
hostname: "ghe-vm-1"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: [80]
public_ipv4: false
public_ipv6: true
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

28
prod_setting/config1.yaml Normal file
View File

@ -0,0 +1,28 @@
brain_url: "http://164.92.249.180:31337"
max_cores_per_vm: 4
max_vcpu_reservation: 8
max_mem_reservation_mb: 25000
network_interfaces:
- driver: "MACVTAP"
device: "eno8303"
ipv4_ranges:
- first_ip: "173.234.136.154"
last_ip: "173.234.136.155"
netmask: "27"
gateway: "173.234.136.158"
- first_ip: "173.234.137.17"
last_ip: "173.234.137.17"
netmask: "27"
gateway: "173.234.137.30"
ipv6_ranges:
- first_ip: "2a0d:3003:b666:a00c:0002:0000:0000:0011"
last_ip: "2a0d:3003:b666:a00c:0002:0000:0000:fffc"
netmask: "64"
gateway: "2a0d:3003:b666:a00c::1"
volumes:
- path: "/opt/detee_vms/"
max_reservation_gb: 200
public_port_range:
start: 30000
end: 50000
max_ports_per_vm: 5

13
prod_setting/ipv4_and_ipv6.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "uuid-0002"
hostname: "ghe-vm-2"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: []
public_ipv4: true
public_ipv6: true
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

13
prod_setting/minimal_nat.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "uuid-0004"
hostname: "ghe-vm-4"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: []
public_ipv4: false
public_ipv6: false
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

13
prod_setting/only_ipv4.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "uuid-0003"
hostname: "ghe-vm-3"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: []
public_ipv4: true
public_ipv6: false
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

26
scripts/install_daemon.sh
View File

@ -1,26 +0,0 @@
#!/bin/bash
set -e
echo "Creating folders..."
mkdir -p /var/lib/detee/boot/
mkdir -p /etc/detee/daemon/vms/
mkdir -p /usr/local/bin/detee/
mkdir -p /opt/detee_vms/
echo "Installing qemu-system-x86..."
pacman -S qemu-system-x86 qemu-img --noconfirm
echo "Downloading detee-snp-daemon, systemd unit file and config..."
wget -O /etc/detee/daemon/sample_config.yaml https://registry.detee.ltd/daemon/config.yaml
wget -O /usr/local/bin/detee-snp-daemon https://registry.detee.ltd/daemon/detee-snp-daemon
chmod +x /usr/local/bin/detee-snp-daemon
wget -O /usr/local/bin/detee/start_qemu_vm.sh https://registry.detee.ltd/daemon/start_qemu_vm.sh
chmod +x /usr/local/bin/detee/start_qemu_vm.sh
wget -O /etc/systemd/system/detee-snp-daemon.service https://registry.detee.ltd/daemon/detee-snp-daemon.service
echo "Take a look at /etc/detee/daemon/sample_config.yaml"
echo "Modify config based on your setup and save it to /etc/detee/daemon/config.yaml"
echo "Press enter when done (this will attempt to start the daemon)"
read my_var
echo "Starting detee-snp-daemon..."
systemctl daemon-reload
systemctl start detee-snp-daemon.service

3
scripts/start_qemu_vm.sh
View File

@ -24,7 +24,6 @@ add_nft_rules() {
nft add chain netdev deteemacvtap ${ifname}_ou "{ type filter hook egress device ${ifname} priority 0; policy accept; }" nft add chain netdev deteemacvtap ${ifname}_ou "{ type filter hook egress device ${ifname} priority 0; policy accept; }"
# return if the rules already exist # return if the rules already exist
nft list chain netdev deteemacvtap ${ifname}_in | grep ether && return 0 nft list chain netdev deteemacvtap ${ifname}_in | grep ether && return 0
nft add rule netdev deteemacvtap ${ifname}_in ether type arp accept
nft add rule netdev deteemacvtap ${ifname}_in ether daddr != ${vtap_addr} drop nft add rule netdev deteemacvtap ${ifname}_in ether daddr != ${vtap_addr} drop
nft list chain netdev deteemacvtap ${ifname}_ou | grep ether && return 0 nft list chain netdev deteemacvtap ${ifname}_ou | grep ether && return 0
nft add rule netdev deteemacvtap ${ifname}_ou ether saddr != ${vtap_addr} drop nft add rule netdev deteemacvtap ${ifname}_ou ether saddr != ${vtap_addr} drop
@ -88,7 +87,7 @@ while read -r interface; do
for port_pair in $NAT_PORT_FW; do for port_pair in $NAT_PORT_FW; do
host_port="$( echo $port_pair | cut -d ':' -f1 )" host_port="$( echo $port_pair | cut -d ':' -f1 )"
guest_port="$( echo $port_pair | cut -d ':' -f2 )" guest_port="$( echo $port_pair | cut -d ':' -f2 )"
ports+=",hostfwd=tcp::${host_port}-:${guest_port},hostfwd=udp::${host_port}-:${guest_port}" ports+=",hostfwd=tcp::${host_port}-:${guest_port}"
done done
qemu_device_params+=" -netdev user,id=natnic${ports}" qemu_device_params+=" -netdev user,id=natnic${ports}"
qemu_device_params+=" -device virtio-net-pci,netdev=natnic,romfile=" qemu_device_params+=" -device virtio-net-pci,netdev=natnic,romfile="

179
snp.proto Normal file
View File

@ -0,0 +1,179 @@
syntax = "proto3";
package snp_proto;
message Empty {
}
message Pubkey {
string pubkey = 1;
}
message Contract {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 exposed_ports = 5;
string public_ipv4 = 6;
string public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_sha = 11;
string dtrfs_sha = 12;
string created_at = 13;
string updated_at = 14;
// total nanotoken cost per minute (for all units)
uint64 nano_per_minute = 15;
uint64 locked_nano = 16;
string collected_at = 17;
}
message MeasurementArgs {
// this will be IP:Port of the dtrfs API
// actually not a measurement arg, but needed for the injector
string dtrfs_api_endpoint = 1;
repeated uint32 exposed_ports = 2;
string ovmf_hash = 5;
// This is needed to allow the CLI to build the kernel params from known data.
// The CLI will use the kernel params to get the measurement.
repeated MeasurementIP ips = 6;
}
message MeasurementIP {
uint32 nic_index = 1;
string address = 2;
string mask = 3;
string gateway = 4;
}
message RegisterNodeReq {
string node_pubkey = 1;
string owner_pubkey = 2;
string main_ip = 3;
string country = 4;
string region = 5;
string city = 6;
// nanotokens per unit per minute
uint64 price = 7;
}
message NodeResources {
string node_pubkey = 1;
uint32 avail_ports = 2;
uint32 avail_ipv4 = 3;
uint32 avail_ipv6 = 4;
uint32 avail_vcpus = 5;
uint32 avail_memory_mb = 6;
uint32 avail_storage_gb = 7;
uint32 max_ports_per_vm = 8;
}
message NewVmReq {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 extra_ports = 5;
bool public_ipv4 = 6;
bool public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_url = 11;
string kernel_sha = 12;
string dtrfs_url = 13;
string dtrfs_sha = 14;
uint64 price_per_unit = 15;
uint64 locked_nano = 16;
}
message NewVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message UpdateVmReq {
string uuid = 1;
uint32 disk_size_gb = 2;
uint32 vcpus = 3;
uint32 memory_mb = 4;
string kernel_url = 5;
string kernel_sha = 6;
string dtrfs_url = 7;
string dtrfs_sha = 8;
}
message UpdateVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message DeleteVmReq {
string uuid = 1;
}
message BrainMessage {
oneof Msg {
NewVmReq new_vm_req = 1;
UpdateVmReq update_vm_req = 2;
DeleteVmReq delete_vm = 3;
}
}
message DaemonMessage {
oneof Msg {
Pubkey pubkey = 1;
NewVmResp new_vm_resp = 2;
UpdateVmResp update_vm_resp = 3;
NodeResources node_resources = 4;
}
}
service BrainDaemon {
rpc RegisterNode (RegisterNodeReq) returns (stream Contract);
rpc BrainMessages (Pubkey) returns (stream BrainMessage);
rpc DaemonMessages (stream DaemonMessage) returns (Empty);
}
message ListContractsReq {
string admin_pubkey = 1;
string node_pubkey = 2;
string uuid = 3;
}
message NodeFilters {
uint32 free_ports = 1;
bool offers_ipv4 = 2;
bool offers_ipv6 = 3;
uint32 vcpus = 4;
uint32 memory_mb = 5;
uint32 storage_gb = 6;
string country = 7;
string region = 8;
string city = 9;
string ip = 10;
}
message NodeListResp {
string node_pubkey = 1;
string country = 2;
string region = 3;
string city = 4;
string ip = 5; // required for latency test
uint32 server_rating = 6;
uint32 provider_rating = 7;
// nanotokens per unit per minute
uint64 price = 8;
}
service BrainCli {
rpc NewVm (NewVmReq) returns (NewVmResp);
rpc ListContracts (ListContractsReq) returns (stream Contract);
rpc ListNodes (NodeFilters) returns (stream NodeListResp);
rpc GetOneNode (NodeFilters) returns (NodeListResp);
rpc DeleteVm (DeleteVmReq) returns (Empty);
rpc UpdateVm (UpdateVmReq) returns (UpdateVmResp);
}

1
src/config.rs
View File

@ -44,7 +44,6 @@ pub enum InterfaceType {
#[derive(Deserialize, Debug)] #[derive(Deserialize, Debug)]
pub struct Config { pub struct Config {
pub owner_wallet: String,
pub brain_url: String, pub brain_url: String,
pub max_cores_per_vm: usize, pub max_cores_per_vm: usize,
pub max_vcpu_reservation: usize, pub max_vcpu_reservation: usize,

24
src/global.rs
View File

@ -2,8 +2,7 @@ use anyhow::Result;
use ed25519_dalek::SigningKey; use ed25519_dalek::SigningKey;
use lazy_static::lazy_static; use lazy_static::lazy_static;
use log::{info, warn}; use log::{info, warn};
use sha2::{Digest, Sha256}; use std::{fs::File, io::Write};
use std::{fs::File, io::Read, io::Write};
pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/"; pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/";
pub(crate) const USED_RESOURCES: &str = "/etc/detee/daemon/used_resources.yaml"; pub(crate) const USED_RESOURCES: &str = "/etc/detee/daemon/used_resources.yaml";
@ -50,12 +49,6 @@ fn load_secret_key() -> Result<ed25519_dalek::SigningKey> {
)) ))
} }
pub fn sign_message(msg: &str) -> Result<String> {
use ed25519_dalek::Signer;
let key = load_secret_key()?;
Ok(bs58::encode(key.sign(msg.as_bytes()).to_bytes()).into_string())
}
pub fn get_public_key() -> String { pub fn get_public_key() -> String {
let pubkey = bs58::encode(load_secret_key().unwrap().verifying_key().to_bytes()).into_string(); let pubkey = bs58::encode(load_secret_key().unwrap().verifying_key().to_bytes()).into_string();
log::info!("Loaded the following public key: {pubkey}"); log::info!("Loaded the following public key: {pubkey}");
@ -75,18 +68,3 @@ fn get_ip_info() -> anyhow::Result<IPInfo> {
info!("Got the following data from ipinfo.io: {body}"); info!("Got the following data from ipinfo.io: {body}");
Ok(serde_json::de::from_str(&body)?) Ok(serde_json::de::from_str(&body)?)
} }
pub fn compute_sha256<P: AsRef<std::path::Path>>(path: P) -> Result<String> {
let mut file = File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 8192];
loop {
let bytes_read = file.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let result = hasher.finalize();
Ok(format!("{:x}", result))
}

96
src/grpc.rs
View File

@ -1,66 +1,54 @@
use crate::global::*; use crate::snp_proto::DaemonMessage;
use crate::snp_proto::VmDaemonMessage;
use anyhow::Result; use anyhow::Result;
use log::{debug, info, warn}; use log::{debug, info, warn};
use snp_proto::{brain_vm_daemon_client::BrainVmDaemonClient, BrainVmMessage, VmContract, RegisterVmNodeReq}; use snp_proto::{
brain_daemon_client::BrainDaemonClient, BrainMessage, Contract, Pubkey, RegisterNodeReq,
};
use tokio::{ use tokio::{
sync::mpsc::{Receiver, Sender}, sync::mpsc::{Receiver, Sender},
task::JoinSet, task::JoinSet,
}; };
use tokio_stream::{wrappers::ReceiverStream, StreamExt}; use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::transport::Channel; use tonic::transport::Channel;
use crate::global::*;
pub mod snp_proto { pub mod snp_proto {
tonic::include_proto!("vm_proto"); tonic::include_proto!("snp_proto");
} }
impl From<snp_proto::NewVmResp> for snp_proto::VmDaemonMessage { impl From<snp_proto::NewVmResp> for snp_proto::DaemonMessage {
fn from(value: snp_proto::NewVmResp) -> Self { fn from(value: snp_proto::NewVmResp) -> Self {
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::NewVmResp(value)) } snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::NewVmResp(value)) }
} }
} }
impl From<snp_proto::UpdateVmResp> for snp_proto::VmDaemonMessage { impl From<snp_proto::UpdateVmResp> for snp_proto::DaemonMessage {
fn from(value: snp_proto::UpdateVmResp) -> Self { fn from(value: snp_proto::UpdateVmResp) -> Self {
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::UpdateVmResp(value)) } snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::UpdateVmResp(value)) }
} }
} }
impl From<snp_proto::VmNodeResources> for snp_proto::VmDaemonMessage { impl From<snp_proto::NodeResources> for snp_proto::DaemonMessage {
fn from(value: snp_proto::VmNodeResources) -> Self { fn from(value: snp_proto::NodeResources) -> Self {
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::VmNodeResources(value)) } snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::NodeResources(value)) }
} }
} }
pub async fn register_node(config: &crate::config::Config) -> Result<Vec<VmContract>> { pub async fn register_node(config: &crate::config::Config) -> Result<Vec<Contract>> {
use tonic::metadata::AsciiMetadataValue; let mut client = BrainDaemonClient::connect(config.brain_url.clone()).await?;
use tonic::Request;
let mut client = BrainVmDaemonClient::connect(config.brain_url.clone()).await?;
debug!("Starting node registration..."); debug!("Starting node registration...");
let ip_info = IP_INFO.clone(); let ip_info = IP_INFO.clone();
let req = RegisterVmNodeReq { let req = RegisterNodeReq {
node_pubkey: PUBLIC_KEY.clone(), node_pubkey: PUBLIC_KEY.clone(),
operator_wallet: config.owner_wallet.clone(), owner_pubkey: "IamTheOwnerOf".to_string() + &PUBLIC_KEY,
main_ip: ip_info.ip, main_ip: ip_info.ip,
country: ip_info.country, country: ip_info.country,
region: ip_info.region, region: ip_info.region,
city: ip_info.city, city: ip_info.city,
price: config.price, price: config.price,
}; };
let pubkey = PUBLIC_KEY.clone();
let timestamp = chrono::Utc::now().to_rfc3339();
let signature = crate::global::sign_message(&format!("{timestamp}{req:?}"))?;
let timestamp: AsciiMetadataValue = timestamp.parse()?;
let pubkey: AsciiMetadataValue = pubkey.parse()?;
let signature: AsciiMetadataValue = signature.parse()?;
let mut req = Request::new(req);
req.metadata_mut().insert("timestamp", timestamp);
req.metadata_mut().insert("pubkey", pubkey);
req.metadata_mut().insert("request-signature", signature);
let mut contracts = Vec::new(); let mut contracts = Vec::new();
let mut grpc_stream = client.register_vm_node(req).await?.into_inner(); let mut grpc_stream = client.register_node(req).await?.into_inner();
while let Some(stream_update) = grpc_stream.next().await { while let Some(stream_update) = grpc_stream.next().await {
match stream_update { match stream_update {
Ok(node) => { Ok(node) => {
@ -76,21 +64,13 @@ pub async fn register_node(config: &crate::config::Config) -> Result<Vec<VmContr
Ok(contracts) Ok(contracts)
} }
fn sign_stream_auth(contracts: Vec<String>) -> Result<snp_proto::DaemonStreamAuth> {
let pubkey = PUBLIC_KEY.clone();
let timestamp = chrono::Utc::now().to_rfc3339();
let signature =
crate::global::sign_message(&(timestamp.to_string() + &format!("{contracts:?}")))?;
Ok(snp_proto::DaemonStreamAuth { timestamp, pubkey, contracts, signature })
}
async fn receive_messages( async fn receive_messages(
mut client: BrainVmDaemonClient<Channel>, mut client: BrainDaemonClient<Channel>,
contracts: Vec<String>, tx: Sender<BrainMessage>,
tx: Sender<BrainVmMessage>,
) -> Result<()> { ) -> Result<()> {
debug!("starting to listen for messages from brain"); debug!("starting to listen for messages from brain");
let mut grpc_stream = client.brain_messages(sign_stream_auth(contracts)?).await?.into_inner(); let pubkey = PUBLIC_KEY.clone();
let mut grpc_stream = client.brain_messages(Pubkey { pubkey }).await?.into_inner();
while let Some(stream_update) = grpc_stream.next().await { while let Some(stream_update) = grpc_stream.next().await {
match stream_update { match stream_update {
Ok(msg) => { Ok(msg) => {
@ -107,41 +87,33 @@ async fn receive_messages(
} }
async fn send_messages( async fn send_messages(
mut client: BrainVmDaemonClient<Channel>, mut client: BrainDaemonClient<Channel>,
contracts: Vec<String>, rx: Receiver<DaemonMessage>,
rx: Receiver<VmDaemonMessage>, tx: Sender<DaemonMessage>,
tx: Sender<VmDaemonMessage>,
) -> Result<()> { ) -> Result<()> {
debug!("starting daemon message stream to brain"); debug!("starting daemon message stream to brain");
let pubkey = PUBLIC_KEY.clone();
let rx_stream = ReceiverStream::new(rx); let rx_stream = ReceiverStream::new(rx);
tx.send(VmDaemonMessage { tx.send(DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::Pubkey(Pubkey { pubkey })) })
msg: Some(snp_proto::vm_daemon_message::Msg::Auth(sign_stream_auth(contracts)?)), .await?;
})
.await?;
client.daemon_messages(rx_stream).await?; client.daemon_messages(rx_stream).await?;
debug!("send_newvm_resp is about to exit"); debug!("send_newvm_resp is about to exit");
Ok(()) Ok(())
} }
pub struct ConnectionData { pub struct ConnectionData {
pub contracts: Vec<String>,
pub brain_url: String, pub brain_url: String,
pub brain_msg_tx: Sender<BrainVmMessage>, pub brain_msg_tx: Sender<BrainMessage>,
pub daemon_msg_rx: Receiver<VmDaemonMessage>, pub daemon_msg_rx: Receiver<DaemonMessage>,
pub daemon_msg_tx: Sender<VmDaemonMessage>, pub daemon_msg_tx: Sender<DaemonMessage>,
} }
pub async fn connect_and_run(cd: ConnectionData) -> Result<()> { pub async fn connect_and_run(cd: ConnectionData) -> Result<()> {
let client = BrainVmDaemonClient::connect(cd.brain_url).await?; let client = BrainDaemonClient::connect(cd.brain_url).await?;
let mut streaming_tasks = JoinSet::new(); let mut streaming_tasks = JoinSet::new();
streaming_tasks.spawn(receive_messages(client.clone(), cd.contracts.clone(), cd.brain_msg_tx)); streaming_tasks.spawn(receive_messages(client.clone(), cd.brain_msg_tx));
streaming_tasks.spawn(send_messages( streaming_tasks.spawn(send_messages(client.clone(), cd.daemon_msg_rx, cd.daemon_msg_tx));
client.clone(),
cd.contracts,
cd.daemon_msg_rx,
cd.daemon_msg_tx,
));
let task_output = streaming_tasks.join_next().await; let task_output = streaming_tasks.join_next().await;
warn!("One stream exited: {task_output:?}"); warn!("One stream exited: {task_output:?}");

103
src/main.rs
View File

@ -5,9 +5,8 @@ mod state;
use crate::global::*; use crate::global::*;
use crate::{config::Config, grpc::snp_proto}; use crate::{config::Config, grpc::snp_proto};
use anyhow::{anyhow, Result}; use anyhow::Result;
use log::{debug, info, warn}; use log::{debug, info, warn};
use std::{fs::File, path::Path};
use tokio::{ use tokio::{
sync::mpsc::{Receiver, Sender}, sync::mpsc::{Receiver, Sender},
time::{sleep, Duration}, time::{sleep, Duration},
@ -15,8 +14,8 @@ use tokio::{
#[allow(dead_code)] #[allow(dead_code)]
struct VMHandler { struct VMHandler {
receiver: Receiver<snp_proto::BrainVmMessage>, receiver: Receiver<snp_proto::BrainMessage>,
sender: Sender<snp_proto::VmDaemonMessage>, sender: Sender<snp_proto::DaemonMessage>,
config: Config, config: Config,
res: state::Resources, res: state::Resources,
} }
@ -24,8 +23,8 @@ struct VMHandler {
#[allow(dead_code)] #[allow(dead_code)]
impl VMHandler { impl VMHandler {
fn new( fn new(
receiver: Receiver<snp_proto::BrainVmMessage>, receiver: Receiver<snp_proto::BrainMessage>,
sender: Sender<snp_proto::VmDaemonMessage>, sender: Sender<snp_proto::DaemonMessage>,
) -> Self { ) -> Self {
let config = match Config::load_from_disk(DAEMON_CONFIG_PATH) { let config = match Config::load_from_disk(DAEMON_CONFIG_PATH) {
Ok(config) => config, Ok(config) => config,
@ -34,8 +33,9 @@ impl VMHandler {
let res = match state::Resources::load_from_disk() { let res = match state::Resources::load_from_disk() {
Ok(res) => res, Ok(res) => res,
Err(e) => { Err(e) => {
log::error!("Could calculate resources: {e:?}"); warn!("Could not load resources from disk: {e:?}");
std::process::exit(1); info!("Creating new resource calculator.");
state::Resources::new(&config.volumes)
} }
}; };
Self { receiver, sender, config, res } Self { receiver, sender, config, res }
@ -60,19 +60,15 @@ impl VMHandler {
async fn send_node_resources(&mut self) { async fn send_node_resources(&mut self) {
let (avail_ipv4, avail_ipv6) = self.get_available_ips(); let (avail_ipv4, avail_ipv6) = self.get_available_ips();
let mut total_gb_available = 0; let mut avail_storage_gb = 0;
for volume in self.config.volumes.iter() { for volume in self.config.volumes.iter() {
let reservation: usize = match self.res.reserved_storage.get(&volume.path) { avail_storage_gb += volume.max_reservation_gb;
Some(reserved) => *reserved, if let Some(reservation) = self.res.reserved_storage.get(&volume.path) {
None => 0 as usize, avail_storage_gb -= reservation;
};
let volume_gb_available = volume.max_reservation_gb - reservation;
if total_gb_available < volume_gb_available {
total_gb_available = volume_gb_available;
} }
} }
let avail_storage_gb = total_gb_available as u32; let avail_storage_gb = avail_storage_gb as u32;
let res = snp_proto::VmNodeResources { let res = snp_proto::NodeResources {
node_pubkey: PUBLIC_KEY.clone(), node_pubkey: PUBLIC_KEY.clone(),
avail_ports: (self.config.public_port_range.len() - self.res.reserved_ports.len()) avail_ports: (self.config.public_port_range.len() - self.res.reserved_ports.len())
as u32, as u32,
@ -144,7 +140,8 @@ impl VMHandler {
async fn handle_update_vm_req(&mut self, update_vm_req: snp_proto::UpdateVmReq) -> Result<()> { async fn handle_update_vm_req(&mut self, update_vm_req: snp_proto::UpdateVmReq) -> Result<()> {
debug!("Processing update vm request: {update_vm_req:?}"); debug!("Processing update vm request: {update_vm_req:?}");
let vm_id = update_vm_req.uuid.clone(); let vm_id = update_vm_req.uuid.clone();
let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?; let content =
std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
let mut vm: state::VM = serde_yaml::from_str(&content)?; let mut vm: state::VM = serde_yaml::from_str(&content)?;
match vm.update(update_vm_req.into(), &self.config, &mut self.res) { match vm.update(update_vm_req.into(), &self.config, &mut self.res) {
Ok(_) => { Ok(_) => {
@ -173,26 +170,26 @@ impl VMHandler {
fn handle_delete_vm(&mut self, delete_vm_req: snp_proto::DeleteVmReq) -> Result<()> { fn handle_delete_vm(&mut self, delete_vm_req: snp_proto::DeleteVmReq) -> Result<()> {
let vm_id = delete_vm_req.uuid; let vm_id = delete_vm_req.uuid;
let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?; let content =
std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
let vm: state::VM = serde_yaml::from_str(&content)?; let vm: state::VM = serde_yaml::from_str(&content)?;
vm.delete(&mut self.res)?; vm.delete(&mut self.res)?;
Ok(()) Ok(())
} }
async fn run(mut self) { async fn run(mut self) {
sleep(Duration::from_millis(500)).await;
self.send_node_resources().await; self.send_node_resources().await;
while let Some(brain_msg) = self.receiver.recv().await { while let Some(brain_msg) = self.receiver.recv().await {
match brain_msg.msg { match brain_msg.msg {
Some(snp_proto::brain_vm_message::Msg::NewVmReq(new_vm_req)) => { Some(snp_proto::brain_message::Msg::NewVmReq(new_vm_req)) => {
self.handle_new_vm_req(new_vm_req).await; self.handle_new_vm_req(new_vm_req).await;
} }
Some(snp_proto::brain_vm_message::Msg::UpdateVmReq(update_vm_req)) => { Some(snp_proto::brain_message::Msg::UpdateVmReq(update_vm_req)) => {
if let Err(e) = self.handle_update_vm_req(update_vm_req).await { if let Err(e) = self.handle_update_vm_req(update_vm_req).await {
log::error!("Could not update vm: {e:?}"); log::error!("Could not update vm: {e:?}");
} }
} }
Some(snp_proto::brain_vm_message::Msg::DeleteVm(delete_vm_req)) => { Some(snp_proto::brain_message::Msg::DeleteVm(delete_vm_req)) => {
let uuid = delete_vm_req.uuid.clone(); let uuid = delete_vm_req.uuid.clone();
if let Err(e) = self.handle_delete_vm(delete_vm_req) { if let Err(e) = self.handle_delete_vm(delete_vm_req) {
log::error!("Could not delete vm {uuid}: {e:?}"); log::error!("Could not delete vm {uuid}: {e:?}");
@ -205,20 +202,19 @@ impl VMHandler {
} }
} }
fn clear_deleted_contracts(&mut self, contracts: Vec<snp_proto::VmContract>) { fn clear_deleted_contracts(&mut self, contracts: Vec<snp_proto::Contract>) {
for uuid in self.res.existing_vms.clone() { for uuid in self.res.existing_vms.clone() {
if contracts.iter().find(|c| c.uuid == uuid).is_none() { if contracts.iter().find(|c| c.uuid == uuid).is_none() {
info!("VM {uuid} exists locally but not found in brain. Deleting..."); info!("VM {uuid} exists locally but not found in brain. Deleting...");
let content = let content = match std::fs::read_to_string(
match std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &uuid + ".yaml") { VM_CONFIG_DIR.to_string() + &uuid + ".yaml",
Ok(content) => content, ) {
Err(e) => { Ok(content) => content,
log::error!( Err(e) => {
"Could not find VM config for {uuid}. Cannot delete VM: {e:?}" log::error!("Could not find VM config for {uuid}. Cannot delete VM: {e:?}");
); continue;
continue; }
} };
};
let vm: crate::state::VM = match serde_yaml::from_str(&content) { let vm: crate::state::VM = match serde_yaml::from_str(&content) {
Ok(vm) => vm, Ok(vm) => vm,
Err(e) => { Err(e) => {
@ -240,13 +236,6 @@ async fn main() {
env_logger::builder().filter_level(log::LevelFilter::Debug).init(); env_logger::builder().filter_level(log::LevelFilter::Debug).init();
loop { loop {
if std::env::var("DAEMON_AUTO_UPGRADE") != Ok("OFF".to_string()) {
// This upgrade procedure will get replaced in prod. We need this for the testnet.
if let Err(e) = download_and_replace_binary() {
log::error!("Failed to upgrade detee-snp-daemon to newer version: {e}");
}
}
let (brain_msg_tx, brain_msg_rx) = tokio::sync::mpsc::channel(6); let (brain_msg_tx, brain_msg_rx) = tokio::sync::mpsc::channel(6);
let (daemon_msg_tx, daemon_msg_rx) = tokio::sync::mpsc::channel(6); let (daemon_msg_tx, daemon_msg_rx) = tokio::sync::mpsc::channel(6);
@ -254,12 +243,8 @@ async fn main() {
let brain_url = vm_handler.config.brain_url.clone(); let brain_url = vm_handler.config.brain_url.clone();
info!("Registering with the brain and getting back VM Contracts (if they exist)."); info!("Registering with the brain and getting back VM Contracts (if they exist).");
let mut contracts: Vec<String> = Vec::new();
match grpc::register_node(&vm_handler.config).await { match grpc::register_node(&vm_handler.config).await {
Ok(c) => { Ok(contracts) => vm_handler.clear_deleted_contracts(contracts),
contracts.append(&mut c.iter().map(|c| c.uuid.clone()).collect());
vm_handler.clear_deleted_contracts(c)
}
Err(e) => log::error!("Could not get contracts from brain: {e:?}"), Err(e) => log::error!("Could not get contracts from brain: {e:?}"),
}; };
@ -269,7 +254,6 @@ async fn main() {
info!("Connecting to brain..."); info!("Connecting to brain...");
if let Err(e) = grpc::connect_and_run(grpc::ConnectionData { if let Err(e) = grpc::connect_and_run(grpc::ConnectionData {
contracts,
brain_url, brain_url,
brain_msg_tx, brain_msg_tx,
daemon_msg_rx, daemon_msg_rx,
@ -282,26 +266,3 @@ async fn main() {
sleep(Duration::from_secs(3)).await; sleep(Duration::from_secs(3)).await;
} }
} }
fn download_and_replace_binary() -> Result<()> {
use reqwest::blocking::get;
use std::os::unix::fs::PermissionsExt;
const TMP_DAEMON: &str = "/usr/local/bin/detee/new-daemon";
const BINARY: &str = "/usr/local/bin/detee-snp-daemon";
let response = get("https://registry.detee.ltd/daemon/detee-snp-daemon")?;
if !response.status().is_success() {
return Err(anyhow!("Failed to download file: {}", response.status()));
}
let mut tmp_file = File::create(Path::new(&TMP_DAEMON))?;
std::io::copy(&mut response.bytes()?.as_ref(), &mut tmp_file)?;
let new_hash = crate::global::compute_sha256(&TMP_DAEMON)?;
let old_hash = crate::global::compute_sha256(&BINARY)?;
log::debug!("Old binary hash: {old_hash}. New binary hash: {new_hash}");
if new_hash != old_hash {
std::fs::rename(BINARY, BINARY.to_string() + "_BACKUP")?;
std::fs::rename(TMP_DAEMON, BINARY)?;
std::fs::set_permissions(BINARY, std::fs::Permissions::from_mode(0o775))?;
std::process::exit(0);
}
Ok(())
}

123
src/state.rs
View File

@ -3,15 +3,17 @@ use crate::{config::Config, global::*, grpc::snp_proto};
use anyhow::{anyhow, Result}; use anyhow::{anyhow, Result};
use log::info; use log::info;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{ use std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
fs, fs,
fs::{remove_file, File}, fs::{remove_file, File},
io::Write, io::{Read, Write},
path::Path,
process::Command, process::Command,
}; };
#[derive(Serialize, Deserialize, Debug, Default)] #[derive(Serialize, Deserialize, Debug)]
pub struct Resources { pub struct Resources {
pub existing_vms: HashSet<String>, pub existing_vms: HashSet<String>,
// QEMU does not support MHz limiation // QEMU does not support MHz limiation
@ -34,49 +36,9 @@ impl Resources {
} }
pub fn load_from_disk() -> Result<Self> { pub fn load_from_disk() -> Result<Self> {
let mut res = Self { ..Default::default() }; let content = std::fs::read_to_string(USED_RESOURCES)?;
let mut res: Self = serde_yaml::from_str(&content)?;
log::debug!("Reading VMs saved to disk to calculate used resources...");
for entry in fs::read_dir(VM_CONFIG_DIR)? {
let entry = entry?;
let path = entry.path();
if path.is_file() && path.to_string_lossy().ends_with(".yaml") {
log::info!("Found VM config: {:?}", path.to_str());
let content = std::fs::read_to_string(path)?;
let vm: VM = serde_yaml::from_str(&content)?;
res.existing_vms.insert(vm.uuid);
res.reserved_vcpus = res.reserved_vcpus.saturating_add(vm.vcpus);
res.reserved_memory = res.reserved_memory.saturating_add(vm.memory_mb);
for (port, _) in vm.fw_ports.iter() {
res.reserved_ports.insert(*port);
}
res.reserved_storage
.entry(vm.storage_dir.clone())
.and_modify(|gb| {
*gb = gb.saturating_add(vm.disk_size_gb);
})
.or_insert(vm.disk_size_gb);
for nic in vm.nics {
for ip in nic.ips {
if let Ok(ip_address) = ip.address.parse::<std::net::IpAddr>() {
if ip_address.is_ipv4() {
res.reserved_ipv4.insert(ip.address.clone());
}
if ip_address.is_ipv6() {
res.reserved_ipv6.insert(ip.address);
}
}
}
if let Some(vtap_name) = nic.if_config.vtap_name() {
res.reserved_if_names.insert(vtap_name);
}
}
}
}
res.scan_boot_files().unwrap(); res.scan_boot_files().unwrap();
res.save_to_disk()?;
Ok(res) Ok(res)
} }
@ -109,7 +71,7 @@ impl Resources {
let mut volumes = config.volumes.clone(); let mut volumes = config.volumes.clone();
for volume in volumes.iter_mut() { for volume in volumes.iter_mut() {
if let Some(reservation) = self.reserved_storage.get(&volume.path) { if let Some(reservation) = self.reserved_storage.get(&volume.path) {
volume.max_reservation_gb = volume.max_reservation_gb.saturating_sub(*reservation); volume.max_reservation_gb -= reservation;
} }
} }
volumes.sort_by_key(|v| v.max_reservation_gb); volumes.sort_by_key(|v| v.max_reservation_gb);
@ -289,9 +251,7 @@ impl Resources {
} }
fn free_vm_resources(&mut self, vm: &VM) { fn free_vm_resources(&mut self, vm: &VM) {
if !self.existing_vms.remove(&vm.uuid) { self.existing_vms.remove(&vm.uuid);
return;
}
self.reserved_vcpus = self.reserved_vcpus.saturating_sub(vm.vcpus); self.reserved_vcpus = self.reserved_vcpus.saturating_sub(vm.vcpus);
self.reserved_memory = self.reserved_memory.saturating_sub(vm.memory_mb); self.reserved_memory = self.reserved_memory.saturating_sub(vm.memory_mb);
for nic in vm.nics.iter() { for nic in vm.nics.iter() {
@ -312,12 +272,8 @@ impl Resources {
for (host_port, _) in vm.fw_ports.iter() { for (host_port, _) in vm.fw_ports.iter() {
self.reserved_ports.remove(host_port); self.reserved_ports.remove(host_port);
} }
self.reserved_storage self.reserved_storage.entry(vm.storage_dir.clone()).and_modify(|gb| *gb -= vm.disk_size_gb);
.entry(vm.storage_dir.clone()) let _ = self.save_to_disk();
.and_modify(|gb| *gb = gb.saturating_sub(vm.disk_size_gb));
if let Err(e) = self.save_to_disk() {
log::error!("Could not save resources to disk: {e}");
}
} }
} }
@ -445,14 +401,22 @@ impl From<VM> for snp_proto::MeasurementArgs {
impl From<VM> for snp_proto::NewVmResp { impl From<VM> for snp_proto::NewVmResp {
fn from(vm: VM) -> Self { fn from(vm: VM) -> Self {
let uuid = vm.uuid.clone(); let uuid = vm.uuid.clone();
snp_proto::NewVmResp { uuid, args: Some(vm.into()), error: "".to_string() } snp_proto::NewVmResp {
uuid,
args: Some(vm.into()),
error: "".to_string(),
}
} }
} }
impl From<VM> for snp_proto::UpdateVmResp { impl From<VM> for snp_proto::UpdateVmResp {
fn from(vm: VM) -> Self { fn from(vm: VM) -> Self {
let uuid = vm.uuid.clone(); let uuid = vm.uuid.clone();
snp_proto::UpdateVmResp { uuid, args: Some(vm.into()), error: "".to_string() } snp_proto::UpdateVmResp {
uuid,
args: Some(vm.into()),
error: "".to_string(),
}
} }
} }
@ -664,22 +628,20 @@ impl VM {
config: &Config, config: &Config,
res: &mut Resources, res: &mut Resources,
) -> Result<(), VMCreationErrors> { ) -> Result<(), VMCreationErrors> {
if req.vcpus > 0 && config.max_cores_per_vm < req.vcpus { if config.max_cores_per_vm < req.vcpus {
return Err(VMCreationErrors::TooManyCores); return Err(VMCreationErrors::TooManyCores);
} }
if req.vcpus > 0 if config.max_vcpu_reservation
&& config.max_vcpu_reservation < res.reserved_vcpus.saturating_sub(self.vcpus).saturating_add(req.vcpus)
< res.reserved_vcpus.saturating_sub(self.vcpus).saturating_add(req.vcpus)
{ {
return Err(VMCreationErrors::NotEnoughCPU); return Err(VMCreationErrors::NotEnoughCPU);
} }
if req.memory_mb > 0 if config.max_mem_reservation_mb
&& config.max_mem_reservation_mb < res.reserved_memory.saturating_sub(self.memory_mb).saturating_add(req.memory_mb)
< res.reserved_memory.saturating_sub(self.memory_mb).saturating_add(req.memory_mb)
{ {
return Err(VMCreationErrors::NotEnoughMemory); return Err(VMCreationErrors::NotEnoughMemory);
} }
if req.disk_size_gb > 0 && req.disk_size_gb < self.disk_size_gb { if req.disk_size_gb < self.disk_size_gb {
return Err(VMCreationErrors::DiskTooSmall); return Err(VMCreationErrors::DiskTooSmall);
} }
@ -721,15 +683,9 @@ impl VM {
}); });
let _ = res.save_to_disk(); let _ = res.save_to_disk();
if req.memory_mb != 0 { self.memory_mb = req.memory_mb;
self.memory_mb = req.memory_mb; self.vcpus = req.vcpus;
} self.disk_size_gb = req.disk_size_gb;
if req.vcpus != 0 {
self.vcpus = req.vcpus;
}
if req.disk_size_gb != 0 {
self.disk_size_gb = req.disk_size_gb;
}
if let Err(e) = systemctl_stop_and_disable(&self.uuid) { if let Err(e) = systemctl_stop_and_disable(&self.uuid) {
return Err(VMCreationErrors::HypervizorError(e.to_string())); return Err(VMCreationErrors::HypervizorError(e.to_string()));
@ -855,9 +811,9 @@ impl VM {
vars += "\n"; vars += "\n";
vars += &format!(r#"export VCPUS="{}""#, self.vcpus); vars += &format!(r#"export VCPUS="{}""#, self.vcpus);
vars += "\n"; vars += "\n";
vars += &format!(r#"export MEMORY="{}M""#, (self.memory_mb / 2 * 2)); vars += &format!(r#"export MEMORY="{}M""#, self.memory_mb);
vars += "\n"; vars += "\n";
vars += &format!(r#"export MAX_MEMORY="{}M""#, (self.memory_mb / 2 * 2) + 256); vars += &format!(r#"export MAX_MEMORY="{}M""#, self.memory_mb + 256);
vars += "\n"; vars += "\n";
vars += &format!(r#"export DISK="{}""#, self.disk_path()); vars += &format!(r#"export DISK="{}""#, self.disk_path());
vars += "\n"; vars += "\n";
@ -1040,7 +996,7 @@ fn download_and_check_sha(url: &str, sha: &str) -> Result<()> {
} }
let mut file = File::create(Path::new(&save_path))?; let mut file = File::create(Path::new(&save_path))?;
copy(&mut response.bytes()?.as_ref(), &mut file)?; copy(&mut response.bytes()?.as_ref(), &mut file)?;
match crate::global::compute_sha256(&save_path) { match compute_sha256(&save_path) {
Ok(hash) => { Ok(hash) => {
if hash != sha { if hash != sha {
return Err(anyhow!( return Err(anyhow!(
@ -1054,3 +1010,18 @@ fn download_and_check_sha(url: &str, sha: &str) -> Result<()> {
} }
Ok(()) Ok(())
} }
fn compute_sha256<P: AsRef<Path>>(path: P) -> Result<String> {
let mut file = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 8192];
loop {
let bytes_read = file.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let result = hasher.finalize();
Ok(format!("{:x}", result))
}

25
test_data/config1.yaml Normal file
View File

@ -0,0 +1,25 @@
max_cores_per_vm: 4
max_vcpu_reservation: 8
max_mem_reservation_mb: 16384
network_interfaces:
- driver: "MACVTAP"
device: "eth0"
ipv4:
- subnet: "192.168.1.0/24"
gateway: "192.168.1.1"
reserved_addrs:
- "192.168.1.100"
- "192.168.1.101"
ipv6:
- subnet: "2001:db8::/32"
gateway: "2001:db8::1"
reserved_addrs:
- "2001:db8::1234"
- "2001:db8::5678"
volumes:
- path: "/mnt/storage"
max_reservation_gb: 200
public_port_range:
start: 8000
end: 9000
max_ports_per_vm: 5

36
test_data/config2.yaml Normal file
View File

@ -0,0 +1,36 @@
max_cores_per_vm: 16
max_vcpu_reservation: 32
max_mem_reservation_mb: 1265536
network_interfaces:
- driver: "Bridge"
device: "br0"
ipv4:
- subnet: "10.0.0.0/16"
gateway: "10.0.0.1"
reserved_addrs:
- "10.0.0.100"
- "10.0.0.101"
- "10.0.0.102"
ipv6: []
- driver: "IPVTAP"
device: "tap1"
ipv4:
- subnet: "172.16.0.0/20"
gateway: "172.16.0.1"
reserved_addrs:
- "172.16.0.10"
- "172.16.0.11"
ipv6:
- subnet: "2001:db8:abcd:1234::/64"
gateway: "2001:db8:abcd:1234::1"
reserved_addrs:
- "2001:db8:abcd:1234::dead"
- "2001:db8:abcd:1234::beef"
volumes:
- path: "/etc/detee/daemon/vms/"
max_reservation_gb: 500
public_port_range:
start: 10000
end: 11000
max_ports_per_vm: 20

20
test_data/config3.yaml Normal file
View File

@ -0,0 +1,20 @@
max_cores_per_vm: 12
max_vcpu_reservation: 24
max_mem_reservation_mb: 49152
network_interfaces:
- driver: "IPVTAP"
device: "tap0"
ipv4: []
ipv6:
- subnet: "2001:db8:abcd:1234::/64"
gateway: "2001:db8:abcd:1234::1"
reserved_addrs:
- "2001:db8:abcd:1234::dead"
- "2001:db8:abcd:1234::beef"
volumes:
- path: "/ipv6/volume"
max_reservation_gb: 600
public_port_range:
start: 15000
end: 16000
max_ports_per_vm: 10

19
test_data/config4.yaml Normal file
View File

@ -0,0 +1,19 @@
max_cores_per_vm: 2
max_vcpu_reservation: 4
max_mem_reservation_mb: 8192
network_interfaces:
- driver: "MACVTAP"
device: "eth0"
ipv4:
- subnet: "192.168.0.0/24"
gateway: "192.168.0.1"
reserved_addrs: []
ipv6: []
volumes:
- path: "/minimal/volume"
max_reservation_gb: 100
public_port_range:
start: 5000
end: 5100
max_ports_per_vm: 3

22
test_data/config5.yaml Normal file
View File

@ -0,0 +1,22 @@
max_cores_per_vm: 8
max_vcpu_reservation: 16
max_mem_reservation_mb: 32768
network_interfaces:
- driver: "Bridge"
device: "br1"
ipv4:
- subnet: "192.168.100.0/24"
gateway: "192.168.100.1"
reserved_addrs: []
ipv6:
- subnet: "2001:abcd::/48"
gateway: "2001:abcd::1"
reserved_addrs: []
volumes:
- path: "/network/volume"
max_reservation_gb: 750
public_port_range:
start: 6000
end: 7000
max_ports_per_vm: 8

13
test_data/new_vm_req1.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "123e4567-e89b-12d3-a456-426614174000"
hostname: "test-vm-01"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample"
extra_ports: [ ]
public_ipv4: true
public_ipv6: false
disk_size_gb: 50
vcpus: 4
memory_mb: 8192
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

13
test_data/new_vm_req2.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "987e6543-e21b-43d3-c321-654987210000"
hostname: "minimal-vm"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQAnotherExampleKey"
extra_ports: []
public_ipv4: false
public_ipv6: false
disk_size_gb: 10
vcpus: 1
memory_mb: 2048
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

14
test_data/new_vm_req3.yaml Normal file
View File

@ -0,0 +1,14 @@
uuid: "246e1357-e98b-76d3-f345-129874650000"
hostname: "extensive-vm"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample"
extra_ports: []
public_ipv4: true
public_ipv6: true
disk_size_gb: 35
vcpus: 2
memory_mb: 5000
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

13
test_data/new_vm_req4.yaml Normal file
View File

@ -0,0 +1,13 @@
uuid: "DuTenPulaMeaCuUUIDulTauCuTot"
hostname: "testing-vm"
admin_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAKeyExampleForTesting"
extra_ports: [1234, 5678]
public_ipv4: false
public_ipv6: true
disk_size_gb: 25
vcpus: 2
memory_mb: 4096
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

272
vm.proto
View File

@ -1,272 +0,0 @@
syntax = "proto3";
package vm_proto;
message Empty {
}
message Pubkey {
string pubkey = 1;
}
message AccountBalance {
uint64 balance = 1;
uint64 tmp_locked = 2;
}
message VmContract {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 exposed_ports = 5;
string public_ipv4 = 6;
string public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_sha = 11;
string dtrfs_sha = 12;
string created_at = 13;
string updated_at = 14;
// total nanoLP cost per minute (for all units)
uint64 nano_per_minute = 15;
uint64 locked_nano = 16;
string collected_at = 17;
}
message MeasurementArgs {
// this will be IP:Port of the dtrfs API
// actually not a measurement arg, but needed for the injector
string dtrfs_api_endpoint = 1;
repeated uint32 exposed_ports = 2;
string ovmf_hash = 5;
// This is needed to allow the CLI to build the kernel params from known data.
// The CLI will use the kernel params to get the measurement.
repeated MeasurementIP ips = 6;
}
message MeasurementIP {
uint32 nic_index = 1;
string address = 2;
string mask = 3;
string gateway = 4;
}
// This should also include a block hash or similar, for auth
message RegisterVmNodeReq {
string node_pubkey = 1;
string operator_wallet = 2;
string main_ip = 3;
string country = 4;
string region = 5;
string city = 6;
// nanoLP per unit per minute
uint64 price = 7;
}
message VmNodeResources {
string node_pubkey = 1;
uint32 avail_ports = 2;
uint32 avail_ipv4 = 3;
uint32 avail_ipv6 = 4;
uint32 avail_vcpus = 5;
uint32 avail_memory_mb = 6;
uint32 avail_storage_gb = 7;
uint32 max_ports_per_vm = 8;
}
message NewVmReq {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 extra_ports = 5;
bool public_ipv4 = 6;
bool public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_url = 11;
string kernel_sha = 12;
string dtrfs_url = 13;
string dtrfs_sha = 14;
uint64 price_per_unit = 15;
uint64 locked_nano = 16;
}
message NewVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message UpdateVmReq {
string uuid = 1;
string admin_pubkey = 2;
uint32 disk_size_gb = 3;
uint32 vcpus = 4;
uint32 memory_mb = 5;
string kernel_url = 6;
string kernel_sha = 7;
string dtrfs_url = 8;
string dtrfs_sha = 9;
}
message UpdateVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message DeleteVmReq {
string uuid = 1;
string admin_pubkey = 2;
}
message BrainVmMessage {
oneof Msg {
NewVmReq new_vm_req = 1;
UpdateVmReq update_vm_req = 2;
DeleteVmReq delete_vm = 3;
}
}
message DaemonStreamAuth {
string timestamp = 1;
string pubkey = 2;
repeated string contracts = 3;
string signature = 4;
}
message VmDaemonMessage {
oneof Msg {
DaemonStreamAuth auth = 1;
NewVmResp new_vm_resp = 2;
UpdateVmResp update_vm_resp = 3;
VmNodeResources vm_node_resources = 4;
}
}
service BrainVmDaemon {
rpc RegisterVmNode (RegisterVmNodeReq) returns (stream VmContract);
rpc BrainMessages (DaemonStreamAuth) returns (stream BrainVmMessage);
rpc DaemonMessages (stream VmDaemonMessage) returns (Empty);
}
message ListVmContractsReq {
string admin_pubkey = 1;
string node_pubkey = 2;
string uuid = 3;
}
message VmNodeFilters {
uint32 free_ports = 1;
bool offers_ipv4 = 2;
bool offers_ipv6 = 3;
uint32 vcpus = 4;
uint32 memory_mb = 5;
uint32 storage_gb = 6;
string country = 7;
string region = 8;
string city = 9;
string ip = 10;
string node_pubkey = 11;
}
message VmNodeListResp {
string operator = 1;
string node_pubkey = 2;
string country = 3;
string region = 4;
string city = 5;
string ip = 6; // required for latency test
repeated string reports = 7; // TODO: this will become an enum
uint64 price = 8; // nanoLP per unit per minute
}
message ExtendVmReq {
string uuid = 1;
string admin_pubkey = 2;
uint64 locked_nano = 3;
}
message AirdropReq {
string pubkey = 1;
uint64 tokens = 2;
}
message SlashReq {
string pubkey = 1;
uint64 tokens = 2;
}
message Account {
string pubkey = 1;
uint64 balance = 2;
uint64 tmp_locked = 3;
}
message RegOperatorReq {
string pubkey = 1;
uint64 escrow = 2;
string email = 3;
}
message ListOperatorsResp {
string pubkey = 1;
uint64 escrow = 2;
string email = 3;
uint64 app_nodes = 4;
uint64 vm_nodes = 5;
uint64 reports = 6;
}
message InspectOperatorResp {
ListOperatorsResp operator = 1;
repeated VmNodeListResp nodes = 2;
}
message ReportNodeReq {
string admin_pubkey = 1;
string node_pubkey = 2;
string contract = 3;
string reason = 4;
}
message KickReq {
string operator_wallet = 1;
string contract_uuid = 2;
string reason = 3;
}
message BanUserReq {
string operator_wallet = 1;
string user_wallet = 2;
}
message KickResp {
uint64 nano_lp = 1;
}
service BrainCli {
rpc GetBalance (Pubkey) returns (AccountBalance);
rpc NewVm (NewVmReq) returns (NewVmResp);
rpc ListVmContracts (ListVmContractsReq) returns (stream VmContract);
rpc ListVmNodes (VmNodeFilters) returns (stream VmNodeListResp);
rpc GetOneVmNode (VmNodeFilters) returns (VmNodeListResp);
rpc DeleteVm (DeleteVmReq) returns (Empty);
rpc UpdateVm (UpdateVmReq) returns (UpdateVmResp);
rpc ExtendVm (ExtendVmReq) returns (Empty);
rpc ReportNode (ReportNodeReq) returns (Empty);
rpc ListOperators (Empty) returns (stream ListOperatorsResp);
rpc InspectOperator (Pubkey) returns (InspectOperatorResp);
rpc RegisterOperator (RegOperatorReq) returns (Empty);
rpc KickContract (KickReq) returns (KickResp);
rpc BanUser (BanUserReq) returns (Empty);
// admin commands
rpc Airdrop (AirdropReq) returns (Empty);
rpc Slash (SlashReq) returns (Empty);
rpc ListAllVmContracts (Empty) returns (stream VmContract);
rpc ListAccounts (Empty) returns (stream Account);
}