Compare commits

..

10 Commits

26 changed files with 612 additions and 550 deletions

71
Cargo.lock generated

@ -26,6 +26,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "android-tzdata"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
[[package]]
name = "android_system_properties"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
dependencies = [
"libc",
]
[[package]]
name = "anstream"
version = "0.6.18"
@ -257,6 +272,20 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "chrono"
version = "0.4.39"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
dependencies = [
"android-tzdata",
"iana-time-zone",
"js-sys",
"num-traits",
"wasm-bindgen",
"windows-targets",
]
[[package]]
name = "colorchoice"
version = "1.0.3"
@ -347,6 +376,7 @@ version = "0.1.0"
dependencies = [
"anyhow",
"bs58",
"chrono",
"ed25519-dalek",
"env_logger",
"lazy_static",
@ -766,6 +796,29 @@ dependencies = [
"tracing",
]
[[package]]
name = "iana-time-zone"
version = "0.1.61"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
dependencies = [
"android_system_properties",
"core-foundation-sys",
"iana-time-zone-haiku",
"js-sys",
"wasm-bindgen",
"windows-core",
]
[[package]]
name = "iana-time-zone-haiku"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
dependencies = [
"cc",
]
[[package]]
name = "icu_collections"
version = "1.5.0"
@ -1053,6 +1106,15 @@ dependencies = [
"tempfile",
]
[[package]]
name = "num-traits"
version = "0.2.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
dependencies = [
"autocfg",
]
[[package]]
name = "object"
version = "0.36.5"
@ -2083,6 +2145,15 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "windows-core"
version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
"windows-targets",
]
[[package]]
name = "windows-registry"
version = "0.2.0"

@ -22,6 +22,13 @@ tokio-stream = "0.1.17"
tonic = "0.12"
serde_json = "1.0.135"
bs58 = "0.5.1"
chrono = "0.4.39"
[build-dependencies]
tonic-build = "0.12"
[profile.release]
opt-level = "z"
lto = true
codegen-units = 1
strip = true

@ -1,6 +1,6 @@
fn main() {
tonic_build::configure()
.build_server(true)
.compile_protos(&["snp.proto"], &["proto"])
.compile_protos(&["vm.proto"], &["proto"])
.unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e));
}

@ -1,13 +0,0 @@
uuid: "uuid-0001"
hostname: "ghe-vm-1"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: [80]
public_ipv4: false
public_ipv6: true
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

@ -1,28 +0,0 @@
brain_url: "http://164.92.249.180:31337"
max_cores_per_vm: 4
max_vcpu_reservation: 8
max_mem_reservation_mb: 25000
network_interfaces:
- driver: "MACVTAP"
device: "eno8303"
ipv4_ranges:
- first_ip: "173.234.136.154"
last_ip: "173.234.136.155"
netmask: "27"
gateway: "173.234.136.158"
- first_ip: "173.234.137.17"
last_ip: "173.234.137.17"
netmask: "27"
gateway: "173.234.137.30"
ipv6_ranges:
- first_ip: "2a0d:3003:b666:a00c:0002:0000:0000:0011"
last_ip: "2a0d:3003:b666:a00c:0002:0000:0000:fffc"
netmask: "64"
gateway: "2a0d:3003:b666:a00c::1"
volumes:
- path: "/opt/detee_vms/"
max_reservation_gb: 200
public_port_range:
start: 30000
end: 50000
max_ports_per_vm: 5

@ -1,13 +0,0 @@
uuid: "uuid-0002"
hostname: "ghe-vm-2"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: []
public_ipv4: true
public_ipv6: true
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

@ -1,13 +0,0 @@
uuid: "uuid-0004"
hostname: "ghe-vm-4"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: []
public_ipv4: false
public_ipv6: false
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

@ -1,13 +0,0 @@
uuid: "uuid-0003"
hostname: "ghe-vm-3"
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
extra_ports: []
public_ipv4: true
public_ipv6: false
disk_size_gb: 20
vcpus: 2
memory_mb: 2000
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"

26
scripts/install_daemon.sh Executable file

@ -0,0 +1,26 @@
#!/bin/bash
set -e
echo "Creating folders..."
mkdir -p /var/lib/detee/boot/
mkdir -p /etc/detee/daemon/vms/
mkdir -p /usr/local/bin/detee/
mkdir -p /opt/detee_vms/
echo "Installing qemu-system-x86..."
pacman -S qemu-system-x86 qemu-img --noconfirm
echo "Downloading detee-snp-daemon, systemd unit file and config..."
wget -O /etc/detee/daemon/sample_config.yaml https://registry.detee.ltd/daemon/config.yaml
wget -O /usr/local/bin/detee-snp-daemon https://registry.detee.ltd/daemon/detee-snp-daemon
chmod +x /usr/local/bin/detee-snp-daemon
wget -O /usr/local/bin/detee/start_qemu_vm.sh https://registry.detee.ltd/daemon/start_qemu_vm.sh
chmod +x /usr/local/bin/detee/start_qemu_vm.sh
wget -O /etc/systemd/system/detee-snp-daemon.service https://registry.detee.ltd/daemon/detee-snp-daemon.service
echo "Take a look at /etc/detee/daemon/sample_config.yaml"
echo "Modify config based on your setup and save it to /etc/detee/daemon/config.yaml"
echo "Press enter when done (this will attempt to start the daemon)"
read my_var
echo "Starting detee-snp-daemon..."
systemctl daemon-reload
systemctl start detee-snp-daemon.service

@ -24,6 +24,7 @@ add_nft_rules() {
nft add chain netdev deteemacvtap ${ifname}_ou "{ type filter hook egress device ${ifname} priority 0; policy accept; }"
# return if the rules already exist
nft list chain netdev deteemacvtap ${ifname}_in | grep ether && return 0
nft add rule netdev deteemacvtap ${ifname}_in ether type arp accept
nft add rule netdev deteemacvtap ${ifname}_in ether daddr != ${vtap_addr} drop
nft list chain netdev deteemacvtap ${ifname}_ou | grep ether && return 0
nft add rule netdev deteemacvtap ${ifname}_ou ether saddr != ${vtap_addr} drop
@ -87,7 +88,7 @@ while read -r interface; do
for port_pair in $NAT_PORT_FW; do
host_port="$( echo $port_pair | cut -d ':' -f1 )"
guest_port="$( echo $port_pair | cut -d ':' -f2 )"
ports+=",hostfwd=tcp::${host_port}-:${guest_port}"
ports+=",hostfwd=tcp::${host_port}-:${guest_port},hostfwd=udp::${host_port}-:${guest_port}"
done
qemu_device_params+=" -netdev user,id=natnic${ports}"
qemu_device_params+=" -device virtio-net-pci,netdev=natnic,romfile="

179
snp.proto

@ -1,179 +0,0 @@
syntax = "proto3";
package snp_proto;
message Empty {
}
message Pubkey {
string pubkey = 1;
}
message Contract {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 exposed_ports = 5;
string public_ipv4 = 6;
string public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_sha = 11;
string dtrfs_sha = 12;
string created_at = 13;
string updated_at = 14;
// total nanotoken cost per minute (for all units)
uint64 nano_per_minute = 15;
uint64 locked_nano = 16;
string collected_at = 17;
}
message MeasurementArgs {
// this will be IP:Port of the dtrfs API
// actually not a measurement arg, but needed for the injector
string dtrfs_api_endpoint = 1;
repeated uint32 exposed_ports = 2;
string ovmf_hash = 5;
// This is needed to allow the CLI to build the kernel params from known data.
// The CLI will use the kernel params to get the measurement.
repeated MeasurementIP ips = 6;
}
message MeasurementIP {
uint32 nic_index = 1;
string address = 2;
string mask = 3;
string gateway = 4;
}
message RegisterNodeReq {
string node_pubkey = 1;
string owner_pubkey = 2;
string main_ip = 3;
string country = 4;
string region = 5;
string city = 6;
// nanotokens per unit per minute
uint64 price = 7;
}
message NodeResources {
string node_pubkey = 1;
uint32 avail_ports = 2;
uint32 avail_ipv4 = 3;
uint32 avail_ipv6 = 4;
uint32 avail_vcpus = 5;
uint32 avail_memory_mb = 6;
uint32 avail_storage_gb = 7;
uint32 max_ports_per_vm = 8;
}
message NewVmReq {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 extra_ports = 5;
bool public_ipv4 = 6;
bool public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_url = 11;
string kernel_sha = 12;
string dtrfs_url = 13;
string dtrfs_sha = 14;
uint64 price_per_unit = 15;
uint64 locked_nano = 16;
}
message NewVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message UpdateVmReq {
string uuid = 1;
uint32 disk_size_gb = 2;
uint32 vcpus = 3;
uint32 memory_mb = 4;
string kernel_url = 5;
string kernel_sha = 6;
string dtrfs_url = 7;
string dtrfs_sha = 8;
}
message UpdateVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message DeleteVmReq {
string uuid = 1;
}
message BrainMessage {
oneof Msg {
NewVmReq new_vm_req = 1;
UpdateVmReq update_vm_req = 2;
DeleteVmReq delete_vm = 3;
}
}
message DaemonMessage {
oneof Msg {
Pubkey pubkey = 1;
NewVmResp new_vm_resp = 2;
UpdateVmResp update_vm_resp = 3;
NodeResources node_resources = 4;
}
}
service BrainDaemon {
rpc RegisterNode (RegisterNodeReq) returns (stream Contract);
rpc BrainMessages (Pubkey) returns (stream BrainMessage);
rpc DaemonMessages (stream DaemonMessage) returns (Empty);
}
message ListContractsReq {
string admin_pubkey = 1;
string node_pubkey = 2;
string uuid = 3;
}
message NodeFilters {
uint32 free_ports = 1;
bool offers_ipv4 = 2;
bool offers_ipv6 = 3;
uint32 vcpus = 4;
uint32 memory_mb = 5;
uint32 storage_gb = 6;
string country = 7;
string region = 8;
string city = 9;
string ip = 10;
}
message NodeListResp {
string node_pubkey = 1;
string country = 2;
string region = 3;
string city = 4;
string ip = 5; // required for latency test
uint32 server_rating = 6;
uint32 provider_rating = 7;
// nanotokens per unit per minute
uint64 price = 8;
}
service BrainCli {
rpc NewVm (NewVmReq) returns (NewVmResp);
rpc ListContracts (ListContractsReq) returns (stream Contract);
rpc ListNodes (NodeFilters) returns (stream NodeListResp);
rpc GetOneNode (NodeFilters) returns (NodeListResp);
rpc DeleteVm (DeleteVmReq) returns (Empty);
rpc UpdateVm (UpdateVmReq) returns (UpdateVmResp);
}

@ -44,6 +44,7 @@ pub enum InterfaceType {
#[derive(Deserialize, Debug)]
pub struct Config {
pub owner_wallet: String,
pub brain_url: String,
pub max_cores_per_vm: usize,
pub max_vcpu_reservation: usize,

@ -2,7 +2,8 @@ use anyhow::Result;
use ed25519_dalek::SigningKey;
use lazy_static::lazy_static;
use log::{info, warn};
use std::{fs::File, io::Write};
use sha2::{Digest, Sha256};
use std::{fs::File, io::Read, io::Write};
pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/";
pub(crate) const USED_RESOURCES: &str = "/etc/detee/daemon/used_resources.yaml";
@ -49,6 +50,12 @@ fn load_secret_key() -> Result<ed25519_dalek::SigningKey> {
))
}
pub fn sign_message(msg: &str) -> Result<String> {
use ed25519_dalek::Signer;
let key = load_secret_key()?;
Ok(bs58::encode(key.sign(msg.as_bytes()).to_bytes()).into_string())
}
pub fn get_public_key() -> String {
let pubkey = bs58::encode(load_secret_key().unwrap().verifying_key().to_bytes()).into_string();
log::info!("Loaded the following public key: {pubkey}");
@ -68,3 +75,18 @@ fn get_ip_info() -> anyhow::Result<IPInfo> {
info!("Got the following data from ipinfo.io: {body}");
Ok(serde_json::de::from_str(&body)?)
}
pub fn compute_sha256<P: AsRef<std::path::Path>>(path: P) -> Result<String> {
let mut file = File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 8192];
loop {
let bytes_read = file.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let result = hasher.finalize();
Ok(format!("{:x}", result))
}

@ -1,54 +1,66 @@
use crate::snp_proto::DaemonMessage;
use crate::global::*;
use crate::snp_proto::VmDaemonMessage;
use anyhow::Result;
use log::{debug, info, warn};
use snp_proto::{
brain_daemon_client::BrainDaemonClient, BrainMessage, Contract, Pubkey, RegisterNodeReq,
};
use snp_proto::{brain_vm_daemon_client::BrainVmDaemonClient, BrainVmMessage, VmContract, RegisterVmNodeReq};
use tokio::{
sync::mpsc::{Receiver, Sender},
task::JoinSet,
};
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
use tonic::transport::Channel;
use crate::global::*;
pub mod snp_proto {
tonic::include_proto!("snp_proto");
tonic::include_proto!("vm_proto");
}
impl From<snp_proto::NewVmResp> for snp_proto::DaemonMessage {
impl From<snp_proto::NewVmResp> for snp_proto::VmDaemonMessage {
fn from(value: snp_proto::NewVmResp) -> Self {
snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::NewVmResp(value)) }
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::NewVmResp(value)) }
}
}
impl From<snp_proto::UpdateVmResp> for snp_proto::DaemonMessage {
impl From<snp_proto::UpdateVmResp> for snp_proto::VmDaemonMessage {
fn from(value: snp_proto::UpdateVmResp) -> Self {
snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::UpdateVmResp(value)) }
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::UpdateVmResp(value)) }
}
}
impl From<snp_proto::NodeResources> for snp_proto::DaemonMessage {
fn from(value: snp_proto::NodeResources) -> Self {
snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::NodeResources(value)) }
impl From<snp_proto::VmNodeResources> for snp_proto::VmDaemonMessage {
fn from(value: snp_proto::VmNodeResources) -> Self {
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::VmNodeResources(value)) }
}
}
pub async fn register_node(config: &crate::config::Config) -> Result<Vec<Contract>> {
let mut client = BrainDaemonClient::connect(config.brain_url.clone()).await?;
pub async fn register_node(config: &crate::config::Config) -> Result<Vec<VmContract>> {
use tonic::metadata::AsciiMetadataValue;
use tonic::Request;
let mut client = BrainVmDaemonClient::connect(config.brain_url.clone()).await?;
debug!("Starting node registration...");
let ip_info = IP_INFO.clone();
let req = RegisterNodeReq {
let req = RegisterVmNodeReq {
node_pubkey: PUBLIC_KEY.clone(),
owner_pubkey: "IamTheOwnerOf".to_string() + &PUBLIC_KEY,
operator_wallet: config.owner_wallet.clone(),
main_ip: ip_info.ip,
country: ip_info.country,
region: ip_info.region,
city: ip_info.city,
price: config.price,
};
let pubkey = PUBLIC_KEY.clone();
let timestamp = chrono::Utc::now().to_rfc3339();
let signature = crate::global::sign_message(&format!("{timestamp}{req:?}"))?;
let timestamp: AsciiMetadataValue = timestamp.parse()?;
let pubkey: AsciiMetadataValue = pubkey.parse()?;
let signature: AsciiMetadataValue = signature.parse()?;
let mut req = Request::new(req);
req.metadata_mut().insert("timestamp", timestamp);
req.metadata_mut().insert("pubkey", pubkey);
req.metadata_mut().insert("request-signature", signature);
let mut contracts = Vec::new();
let mut grpc_stream = client.register_node(req).await?.into_inner();
let mut grpc_stream = client.register_vm_node(req).await?.into_inner();
while let Some(stream_update) = grpc_stream.next().await {
match stream_update {
Ok(node) => {
@ -64,13 +76,21 @@ pub async fn register_node(config: &crate::config::Config) -> Result<Vec<Contrac
Ok(contracts)
}
fn sign_stream_auth(contracts: Vec<String>) -> Result<snp_proto::DaemonStreamAuth> {
let pubkey = PUBLIC_KEY.clone();
let timestamp = chrono::Utc::now().to_rfc3339();
let signature =
crate::global::sign_message(&(timestamp.to_string() + &format!("{contracts:?}")))?;
Ok(snp_proto::DaemonStreamAuth { timestamp, pubkey, contracts, signature })
}
async fn receive_messages(
mut client: BrainDaemonClient<Channel>,
tx: Sender<BrainMessage>,
mut client: BrainVmDaemonClient<Channel>,
contracts: Vec<String>,
tx: Sender<BrainVmMessage>,
) -> Result<()> {
debug!("starting to listen for messages from brain");
let pubkey = PUBLIC_KEY.clone();
let mut grpc_stream = client.brain_messages(Pubkey { pubkey }).await?.into_inner();
let mut grpc_stream = client.brain_messages(sign_stream_auth(contracts)?).await?.into_inner();
while let Some(stream_update) = grpc_stream.next().await {
match stream_update {
Ok(msg) => {
@ -87,14 +107,16 @@ async fn receive_messages(
}
async fn send_messages(
mut client: BrainDaemonClient<Channel>,
rx: Receiver<DaemonMessage>,
tx: Sender<DaemonMessage>,
mut client: BrainVmDaemonClient<Channel>,
contracts: Vec<String>,
rx: Receiver<VmDaemonMessage>,
tx: Sender<VmDaemonMessage>,
) -> Result<()> {
debug!("starting daemon message stream to brain");
let pubkey = PUBLIC_KEY.clone();
let rx_stream = ReceiverStream::new(rx);
tx.send(DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::Pubkey(Pubkey { pubkey })) })
tx.send(VmDaemonMessage {
msg: Some(snp_proto::vm_daemon_message::Msg::Auth(sign_stream_auth(contracts)?)),
})
.await?;
client.daemon_messages(rx_stream).await?;
debug!("send_newvm_resp is about to exit");
@ -102,18 +124,24 @@ async fn send_messages(
}
pub struct ConnectionData {
pub contracts: Vec<String>,
pub brain_url: String,
pub brain_msg_tx: Sender<BrainMessage>,
pub daemon_msg_rx: Receiver<DaemonMessage>,
pub daemon_msg_tx: Sender<DaemonMessage>,
pub brain_msg_tx: Sender<BrainVmMessage>,
pub daemon_msg_rx: Receiver<VmDaemonMessage>,
pub daemon_msg_tx: Sender<VmDaemonMessage>,
}
pub async fn connect_and_run(cd: ConnectionData) -> Result<()> {
let client = BrainDaemonClient::connect(cd.brain_url).await?;
let client = BrainVmDaemonClient::connect(cd.brain_url).await?;
let mut streaming_tasks = JoinSet::new();
streaming_tasks.spawn(receive_messages(client.clone(), cd.brain_msg_tx));
streaming_tasks.spawn(send_messages(client.clone(), cd.daemon_msg_rx, cd.daemon_msg_tx));
streaming_tasks.spawn(receive_messages(client.clone(), cd.contracts.clone(), cd.brain_msg_tx));
streaming_tasks.spawn(send_messages(
client.clone(),
cd.contracts,
cd.daemon_msg_rx,
cd.daemon_msg_tx,
));
let task_output = streaming_tasks.join_next().await;
warn!("One stream exited: {task_output:?}");

@ -5,8 +5,9 @@ mod state;
use crate::global::*;
use crate::{config::Config, grpc::snp_proto};
use anyhow::Result;
use anyhow::{anyhow, Result};
use log::{debug, info, warn};
use std::{fs::File, path::Path};
use tokio::{
sync::mpsc::{Receiver, Sender},
time::{sleep, Duration},
@ -14,8 +15,8 @@ use tokio::{
#[allow(dead_code)]
struct VMHandler {
receiver: Receiver<snp_proto::BrainMessage>,
sender: Sender<snp_proto::DaemonMessage>,
receiver: Receiver<snp_proto::BrainVmMessage>,
sender: Sender<snp_proto::VmDaemonMessage>,
config: Config,
res: state::Resources,
}
@ -23,8 +24,8 @@ struct VMHandler {
#[allow(dead_code)]
impl VMHandler {
fn new(
receiver: Receiver<snp_proto::BrainMessage>,
sender: Sender<snp_proto::DaemonMessage>,
receiver: Receiver<snp_proto::BrainVmMessage>,
sender: Sender<snp_proto::VmDaemonMessage>,
) -> Self {
let config = match Config::load_from_disk(DAEMON_CONFIG_PATH) {
Ok(config) => config,
@ -33,9 +34,8 @@ impl VMHandler {
let res = match state::Resources::load_from_disk() {
Ok(res) => res,
Err(e) => {
warn!("Could not load resources from disk: {e:?}");
info!("Creating new resource calculator.");
state::Resources::new(&config.volumes)
log::error!("Could calculate resources: {e:?}");
std::process::exit(1);
}
};
Self { receiver, sender, config, res }
@ -60,15 +60,19 @@ impl VMHandler {
async fn send_node_resources(&mut self) {
let (avail_ipv4, avail_ipv6) = self.get_available_ips();
let mut avail_storage_gb = 0;
let mut total_gb_available = 0;
for volume in self.config.volumes.iter() {
avail_storage_gb += volume.max_reservation_gb;
if let Some(reservation) = self.res.reserved_storage.get(&volume.path) {
avail_storage_gb -= reservation;
let reservation: usize = match self.res.reserved_storage.get(&volume.path) {
Some(reserved) => *reserved,
None => 0 as usize,
};
let volume_gb_available = volume.max_reservation_gb - reservation;
if total_gb_available < volume_gb_available {
total_gb_available = volume_gb_available;
}
}
let avail_storage_gb = avail_storage_gb as u32;
let res = snp_proto::NodeResources {
let avail_storage_gb = total_gb_available as u32;
let res = snp_proto::VmNodeResources {
node_pubkey: PUBLIC_KEY.clone(),
avail_ports: (self.config.public_port_range.len() - self.res.reserved_ports.len())
as u32,
@ -140,8 +144,7 @@ impl VMHandler {
async fn handle_update_vm_req(&mut self, update_vm_req: snp_proto::UpdateVmReq) -> Result<()> {
debug!("Processing update vm request: {update_vm_req:?}");
let vm_id = update_vm_req.uuid.clone();
let content =
std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
let mut vm: state::VM = serde_yaml::from_str(&content)?;
match vm.update(update_vm_req.into(), &self.config, &mut self.res) {
Ok(_) => {
@ -170,26 +173,26 @@ impl VMHandler {
fn handle_delete_vm(&mut self, delete_vm_req: snp_proto::DeleteVmReq) -> Result<()> {
let vm_id = delete_vm_req.uuid;
let content =
std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
let vm: state::VM = serde_yaml::from_str(&content)?;
vm.delete(&mut self.res)?;
Ok(())
}
async fn run(mut self) {
sleep(Duration::from_millis(500)).await;
self.send_node_resources().await;
while let Some(brain_msg) = self.receiver.recv().await {
match brain_msg.msg {
Some(snp_proto::brain_message::Msg::NewVmReq(new_vm_req)) => {
Some(snp_proto::brain_vm_message::Msg::NewVmReq(new_vm_req)) => {
self.handle_new_vm_req(new_vm_req).await;
}
Some(snp_proto::brain_message::Msg::UpdateVmReq(update_vm_req)) => {
Some(snp_proto::brain_vm_message::Msg::UpdateVmReq(update_vm_req)) => {
if let Err(e) = self.handle_update_vm_req(update_vm_req).await {
log::error!("Could not update vm: {e:?}");
}
}
Some(snp_proto::brain_message::Msg::DeleteVm(delete_vm_req)) => {
Some(snp_proto::brain_vm_message::Msg::DeleteVm(delete_vm_req)) => {
let uuid = delete_vm_req.uuid.clone();
if let Err(e) = self.handle_delete_vm(delete_vm_req) {
log::error!("Could not delete vm {uuid}: {e:?}");
@ -202,16 +205,17 @@ impl VMHandler {
}
}
fn clear_deleted_contracts(&mut self, contracts: Vec<snp_proto::Contract>) {
fn clear_deleted_contracts(&mut self, contracts: Vec<snp_proto::VmContract>) {
for uuid in self.res.existing_vms.clone() {
if contracts.iter().find(|c| c.uuid == uuid).is_none() {
info!("VM {uuid} exists locally but not found in brain. Deleting...");
let content = match std::fs::read_to_string(
VM_CONFIG_DIR.to_string() + &uuid + ".yaml",
) {
let content =
match std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &uuid + ".yaml") {
Ok(content) => content,
Err(e) => {
log::error!("Could not find VM config for {uuid}. Cannot delete VM: {e:?}");
log::error!(
"Could not find VM config for {uuid}. Cannot delete VM: {e:?}"
);
continue;
}
};
@ -236,6 +240,13 @@ async fn main() {
env_logger::builder().filter_level(log::LevelFilter::Debug).init();
loop {
if std::env::var("DAEMON_AUTO_UPGRADE") != Ok("OFF".to_string()) {
// This upgrade procedure will get replaced in prod. We need this for the testnet.
if let Err(e) = download_and_replace_binary() {
log::error!("Failed to upgrade detee-snp-daemon to newer version: {e}");
}
}
let (brain_msg_tx, brain_msg_rx) = tokio::sync::mpsc::channel(6);
let (daemon_msg_tx, daemon_msg_rx) = tokio::sync::mpsc::channel(6);
@ -243,8 +254,12 @@ async fn main() {
let brain_url = vm_handler.config.brain_url.clone();
info!("Registering with the brain and getting back VM Contracts (if they exist).");
let mut contracts: Vec<String> = Vec::new();
match grpc::register_node(&vm_handler.config).await {
Ok(contracts) => vm_handler.clear_deleted_contracts(contracts),
Ok(c) => {
contracts.append(&mut c.iter().map(|c| c.uuid.clone()).collect());
vm_handler.clear_deleted_contracts(c)
}
Err(e) => log::error!("Could not get contracts from brain: {e:?}"),
};
@ -254,6 +269,7 @@ async fn main() {
info!("Connecting to brain...");
if let Err(e) = grpc::connect_and_run(grpc::ConnectionData {
contracts,
brain_url,
brain_msg_tx,
daemon_msg_rx,
@ -266,3 +282,26 @@ async fn main() {
sleep(Duration::from_secs(3)).await;
}
}
fn download_and_replace_binary() -> Result<()> {
use reqwest::blocking::get;
use std::os::unix::fs::PermissionsExt;
const TMP_DAEMON: &str = "/usr/local/bin/detee/new-daemon";
const BINARY: &str = "/usr/local/bin/detee-snp-daemon";
let response = get("https://registry.detee.ltd/daemon/detee-snp-daemon")?;
if !response.status().is_success() {
return Err(anyhow!("Failed to download file: {}", response.status()));
}
let mut tmp_file = File::create(Path::new(&TMP_DAEMON))?;
std::io::copy(&mut response.bytes()?.as_ref(), &mut tmp_file)?;
let new_hash = crate::global::compute_sha256(&TMP_DAEMON)?;
let old_hash = crate::global::compute_sha256(&BINARY)?;
log::debug!("Old binary hash: {old_hash}. New binary hash: {new_hash}");
if new_hash != old_hash {
std::fs::rename(BINARY, BINARY.to_string() + "_BACKUP")?;
std::fs::rename(TMP_DAEMON, BINARY)?;
std::fs::set_permissions(BINARY, std::fs::Permissions::from_mode(0o775))?;
std::process::exit(0);
}
Ok(())
}

@ -3,17 +3,15 @@ use crate::{config::Config, global::*, grpc::snp_proto};
use anyhow::{anyhow, Result};
use log::info;
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::{
collections::{HashMap, HashSet},
fs,
fs::{remove_file, File},
io::{Read, Write},
path::Path,
io::Write,
process::Command,
};
#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Default)]
pub struct Resources {
pub existing_vms: HashSet<String>,
// QEMU does not support MHz limiation
@ -36,9 +34,49 @@ impl Resources {
}
pub fn load_from_disk() -> Result<Self> {
let content = std::fs::read_to_string(USED_RESOURCES)?;
let mut res: Self = serde_yaml::from_str(&content)?;
let mut res = Self { ..Default::default() };
log::debug!("Reading VMs saved to disk to calculate used resources...");
for entry in fs::read_dir(VM_CONFIG_DIR)? {
let entry = entry?;
let path = entry.path();
if path.is_file() && path.to_string_lossy().ends_with(".yaml") {
log::info!("Found VM config: {:?}", path.to_str());
let content = std::fs::read_to_string(path)?;
let vm: VM = serde_yaml::from_str(&content)?;
res.existing_vms.insert(vm.uuid);
res.reserved_vcpus = res.reserved_vcpus.saturating_add(vm.vcpus);
res.reserved_memory = res.reserved_memory.saturating_add(vm.memory_mb);
for (port, _) in vm.fw_ports.iter() {
res.reserved_ports.insert(*port);
}
res.reserved_storage
.entry(vm.storage_dir.clone())
.and_modify(|gb| {
*gb = gb.saturating_add(vm.disk_size_gb);
})
.or_insert(vm.disk_size_gb);
for nic in vm.nics {
for ip in nic.ips {
if let Ok(ip_address) = ip.address.parse::<std::net::IpAddr>() {
if ip_address.is_ipv4() {
res.reserved_ipv4.insert(ip.address.clone());
}
if ip_address.is_ipv6() {
res.reserved_ipv6.insert(ip.address);
}
}
}
if let Some(vtap_name) = nic.if_config.vtap_name() {
res.reserved_if_names.insert(vtap_name);
}
}
}
}
res.scan_boot_files().unwrap();
res.save_to_disk()?;
Ok(res)
}
@ -71,7 +109,7 @@ impl Resources {
let mut volumes = config.volumes.clone();
for volume in volumes.iter_mut() {
if let Some(reservation) = self.reserved_storage.get(&volume.path) {
volume.max_reservation_gb -= reservation;
volume.max_reservation_gb = volume.max_reservation_gb.saturating_sub(*reservation);
}
}
volumes.sort_by_key(|v| v.max_reservation_gb);
@ -251,7 +289,9 @@ impl Resources {
}
fn free_vm_resources(&mut self, vm: &VM) {
self.existing_vms.remove(&vm.uuid);
if !self.existing_vms.remove(&vm.uuid) {
return;
}
self.reserved_vcpus = self.reserved_vcpus.saturating_sub(vm.vcpus);
self.reserved_memory = self.reserved_memory.saturating_sub(vm.memory_mb);
for nic in vm.nics.iter() {
@ -272,8 +312,12 @@ impl Resources {
for (host_port, _) in vm.fw_ports.iter() {
self.reserved_ports.remove(host_port);
}
self.reserved_storage.entry(vm.storage_dir.clone()).and_modify(|gb| *gb -= vm.disk_size_gb);
let _ = self.save_to_disk();
self.reserved_storage
.entry(vm.storage_dir.clone())
.and_modify(|gb| *gb = gb.saturating_sub(vm.disk_size_gb));
if let Err(e) = self.save_to_disk() {
log::error!("Could not save resources to disk: {e}");
}
}
}
@ -401,22 +445,14 @@ impl From<VM> for snp_proto::MeasurementArgs {
impl From<VM> for snp_proto::NewVmResp {
fn from(vm: VM) -> Self {
let uuid = vm.uuid.clone();
snp_proto::NewVmResp {
uuid,
args: Some(vm.into()),
error: "".to_string(),
}
snp_proto::NewVmResp { uuid, args: Some(vm.into()), error: "".to_string() }
}
}
impl From<VM> for snp_proto::UpdateVmResp {
fn from(vm: VM) -> Self {
let uuid = vm.uuid.clone();
snp_proto::UpdateVmResp {
uuid,
args: Some(vm.into()),
error: "".to_string(),
}
snp_proto::UpdateVmResp { uuid, args: Some(vm.into()), error: "".to_string() }
}
}
@ -628,20 +664,22 @@ impl VM {
config: &Config,
res: &mut Resources,
) -> Result<(), VMCreationErrors> {
if config.max_cores_per_vm < req.vcpus {
if req.vcpus > 0 && config.max_cores_per_vm < req.vcpus {
return Err(VMCreationErrors::TooManyCores);
}
if config.max_vcpu_reservation
if req.vcpus > 0
&& config.max_vcpu_reservation
< res.reserved_vcpus.saturating_sub(self.vcpus).saturating_add(req.vcpus)
{
return Err(VMCreationErrors::NotEnoughCPU);
}
if config.max_mem_reservation_mb
if req.memory_mb > 0
&& config.max_mem_reservation_mb
< res.reserved_memory.saturating_sub(self.memory_mb).saturating_add(req.memory_mb)
{
return Err(VMCreationErrors::NotEnoughMemory);
}
if req.disk_size_gb < self.disk_size_gb {
if req.disk_size_gb > 0 && req.disk_size_gb < self.disk_size_gb {
return Err(VMCreationErrors::DiskTooSmall);
}
@ -683,9 +721,15 @@ impl VM {
});
let _ = res.save_to_disk();
if req.memory_mb != 0 {
self.memory_mb = req.memory_mb;
}
if req.vcpus != 0 {
self.vcpus = req.vcpus;
}
if req.disk_size_gb != 0 {
self.disk_size_gb = req.disk_size_gb;
}
if let Err(e) = systemctl_stop_and_disable(&self.uuid) {
return Err(VMCreationErrors::HypervizorError(e.to_string()));
@ -811,9 +855,9 @@ impl VM {
vars += "\n";
vars += &format!(r#"export VCPUS="{}""#, self.vcpus);
vars += "\n";
vars += &format!(r#"export MEMORY="{}M""#, self.memory_mb);
vars += &format!(r#"export MEMORY="{}M""#, (self.memory_mb / 2 * 2));
vars += "\n";
vars += &format!(r#"export MAX_MEMORY="{}M""#, self.memory_mb + 256);
vars += &format!(r#"export MAX_MEMORY="{}M""#, (self.memory_mb / 2 * 2) + 256);
vars += "\n";
vars += &format!(r#"export DISK="{}""#, self.disk_path());
vars += "\n";
@ -996,7 +1040,7 @@ fn download_and_check_sha(url: &str, sha: &str) -> Result<()> {
}
let mut file = File::create(Path::new(&save_path))?;
copy(&mut response.bytes()?.as_ref(), &mut file)?;
match compute_sha256(&save_path) {
match crate::global::compute_sha256(&save_path) {
Ok(hash) => {
if hash != sha {
return Err(anyhow!(
@ -1010,18 +1054,3 @@ fn download_and_check_sha(url: &str, sha: &str) -> Result<()> {
}
Ok(())
}
fn compute_sha256<P: AsRef<Path>>(path: P) -> Result<String> {
let mut file = fs::File::open(path)?;
let mut hasher = Sha256::new();
let mut buffer = [0u8; 8192];
loop {
let bytes_read = file.read(&mut buffer)?;
if bytes_read == 0 {
break;
}
hasher.update(&buffer[..bytes_read]);
}
let result = hasher.finalize();
Ok(format!("{:x}", result))
}

@ -1,25 +0,0 @@
max_cores_per_vm: 4
max_vcpu_reservation: 8
max_mem_reservation_mb: 16384
network_interfaces:
- driver: "MACVTAP"
device: "eth0"
ipv4:
- subnet: "192.168.1.0/24"
gateway: "192.168.1.1"
reserved_addrs:
- "192.168.1.100"
- "192.168.1.101"
ipv6:
- subnet: "2001:db8::/32"
gateway: "2001:db8::1"
reserved_addrs:
- "2001:db8::1234"
- "2001:db8::5678"
volumes:
- path: "/mnt/storage"
max_reservation_gb: 200
public_port_range:
start: 8000
end: 9000
max_ports_per_vm: 5

@ -1,36 +0,0 @@
max_cores_per_vm: 16
max_vcpu_reservation: 32
max_mem_reservation_mb: 1265536
network_interfaces:
- driver: "Bridge"
device: "br0"
ipv4:
- subnet: "10.0.0.0/16"
gateway: "10.0.0.1"
reserved_addrs:
- "10.0.0.100"
- "10.0.0.101"
- "10.0.0.102"
ipv6: []
- driver: "IPVTAP"
device: "tap1"
ipv4:
- subnet: "172.16.0.0/20"
gateway: "172.16.0.1"
reserved_addrs:
- "172.16.0.10"
- "172.16.0.11"
ipv6:
- subnet: "2001:db8:abcd:1234::/64"
gateway: "2001:db8:abcd:1234::1"
reserved_addrs:
- "2001:db8:abcd:1234::dead"
- "2001:db8:abcd:1234::beef"
volumes:
- path: "/etc/detee/daemon/vms/"
max_reservation_gb: 500
public_port_range:
start: 10000
end: 11000
max_ports_per_vm: 20

@ -1,20 +0,0 @@
max_cores_per_vm: 12
max_vcpu_reservation: 24
max_mem_reservation_mb: 49152
network_interfaces:
- driver: "IPVTAP"
device: "tap0"
ipv4: []
ipv6:
- subnet: "2001:db8:abcd:1234::/64"
gateway: "2001:db8:abcd:1234::1"
reserved_addrs:
- "2001:db8:abcd:1234::dead"
- "2001:db8:abcd:1234::beef"
volumes:
- path: "/ipv6/volume"
max_reservation_gb: 600
public_port_range:
start: 15000
end: 16000
max_ports_per_vm: 10

@ -1,19 +0,0 @@
max_cores_per_vm: 2
max_vcpu_reservation: 4
max_mem_reservation_mb: 8192
network_interfaces:
- driver: "MACVTAP"
device: "eth0"
ipv4:
- subnet: "192.168.0.0/24"
gateway: "192.168.0.1"
reserved_addrs: []
ipv6: []
volumes:
- path: "/minimal/volume"
max_reservation_gb: 100
public_port_range:
start: 5000
end: 5100
max_ports_per_vm: 3

@ -1,22 +0,0 @@
max_cores_per_vm: 8
max_vcpu_reservation: 16
max_mem_reservation_mb: 32768
network_interfaces:
- driver: "Bridge"
device: "br1"
ipv4:
- subnet: "192.168.100.0/24"
gateway: "192.168.100.1"
reserved_addrs: []
ipv6:
- subnet: "2001:abcd::/48"
gateway: "2001:abcd::1"
reserved_addrs: []
volumes:
- path: "/network/volume"
max_reservation_gb: 750
public_port_range:
start: 6000
end: 7000
max_ports_per_vm: 8

@ -1,13 +0,0 @@
uuid: "123e4567-e89b-12d3-a456-426614174000"
hostname: "test-vm-01"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample"
extra_ports: [ ]
public_ipv4: true
public_ipv6: false
disk_size_gb: 50
vcpus: 4
memory_mb: 8192
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

@ -1,13 +0,0 @@
uuid: "987e6543-e21b-43d3-c321-654987210000"
hostname: "minimal-vm"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQAnotherExampleKey"
extra_ports: []
public_ipv4: false
public_ipv6: false
disk_size_gb: 10
vcpus: 1
memory_mb: 2048
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

@ -1,14 +0,0 @@
uuid: "246e1357-e98b-76d3-f345-129874650000"
hostname: "extensive-vm"
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample"
extra_ports: []
public_ipv4: true
public_ipv6: true
disk_size_gb: 35
vcpus: 2
memory_mb: 5000
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

@ -1,13 +0,0 @@
uuid: "DuTenPulaMeaCuUUIDulTauCuTot"
hostname: "testing-vm"
admin_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAKeyExampleForTesting"
extra_ports: [1234, 5678]
public_ipv4: false
public_ipv6: true
disk_size_gb: 25
vcpus: 2
memory_mb: 4096
kernel_url: "http://pb1n.de/?d25eec"
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
dtrfs_url: "http://pb1n.de/?e46db9"
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"

272
vm.proto Normal file

@ -0,0 +1,272 @@
syntax = "proto3";
package vm_proto;
message Empty {
}
message Pubkey {
string pubkey = 1;
}
message AccountBalance {
uint64 balance = 1;
uint64 tmp_locked = 2;
}
message VmContract {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 exposed_ports = 5;
string public_ipv4 = 6;
string public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_sha = 11;
string dtrfs_sha = 12;
string created_at = 13;
string updated_at = 14;
// total nanoLP cost per minute (for all units)
uint64 nano_per_minute = 15;
uint64 locked_nano = 16;
string collected_at = 17;
}
message MeasurementArgs {
// this will be IP:Port of the dtrfs API
// actually not a measurement arg, but needed for the injector
string dtrfs_api_endpoint = 1;
repeated uint32 exposed_ports = 2;
string ovmf_hash = 5;
// This is needed to allow the CLI to build the kernel params from known data.
// The CLI will use the kernel params to get the measurement.
repeated MeasurementIP ips = 6;
}
message MeasurementIP {
uint32 nic_index = 1;
string address = 2;
string mask = 3;
string gateway = 4;
}
// This should also include a block hash or similar, for auth
message RegisterVmNodeReq {
string node_pubkey = 1;
string operator_wallet = 2;
string main_ip = 3;
string country = 4;
string region = 5;
string city = 6;
// nanoLP per unit per minute
uint64 price = 7;
}
message VmNodeResources {
string node_pubkey = 1;
uint32 avail_ports = 2;
uint32 avail_ipv4 = 3;
uint32 avail_ipv6 = 4;
uint32 avail_vcpus = 5;
uint32 avail_memory_mb = 6;
uint32 avail_storage_gb = 7;
uint32 max_ports_per_vm = 8;
}
message NewVmReq {
string uuid = 1;
string hostname = 2;
string admin_pubkey = 3;
string node_pubkey = 4;
repeated uint32 extra_ports = 5;
bool public_ipv4 = 6;
bool public_ipv6 = 7;
uint32 disk_size_gb = 8;
uint32 vcpus = 9;
uint32 memory_mb = 10;
string kernel_url = 11;
string kernel_sha = 12;
string dtrfs_url = 13;
string dtrfs_sha = 14;
uint64 price_per_unit = 15;
uint64 locked_nano = 16;
}
message NewVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message UpdateVmReq {
string uuid = 1;
string admin_pubkey = 2;
uint32 disk_size_gb = 3;
uint32 vcpus = 4;
uint32 memory_mb = 5;
string kernel_url = 6;
string kernel_sha = 7;
string dtrfs_url = 8;
string dtrfs_sha = 9;
}
message UpdateVmResp {
string uuid = 1;
string error = 2;
MeasurementArgs args = 3;
}
message DeleteVmReq {
string uuid = 1;
string admin_pubkey = 2;
}
message BrainVmMessage {
oneof Msg {
NewVmReq new_vm_req = 1;
UpdateVmReq update_vm_req = 2;
DeleteVmReq delete_vm = 3;
}
}
message DaemonStreamAuth {
string timestamp = 1;
string pubkey = 2;
repeated string contracts = 3;
string signature = 4;
}
message VmDaemonMessage {
oneof Msg {
DaemonStreamAuth auth = 1;
NewVmResp new_vm_resp = 2;
UpdateVmResp update_vm_resp = 3;
VmNodeResources vm_node_resources = 4;
}
}
service BrainVmDaemon {
rpc RegisterVmNode (RegisterVmNodeReq) returns (stream VmContract);
rpc BrainMessages (DaemonStreamAuth) returns (stream BrainVmMessage);
rpc DaemonMessages (stream VmDaemonMessage) returns (Empty);
}
message ListVmContractsReq {
string admin_pubkey = 1;
string node_pubkey = 2;
string uuid = 3;
}
message VmNodeFilters {
uint32 free_ports = 1;
bool offers_ipv4 = 2;
bool offers_ipv6 = 3;
uint32 vcpus = 4;
uint32 memory_mb = 5;
uint32 storage_gb = 6;
string country = 7;
string region = 8;
string city = 9;
string ip = 10;
string node_pubkey = 11;
}
message VmNodeListResp {
string operator = 1;
string node_pubkey = 2;
string country = 3;
string region = 4;
string city = 5;
string ip = 6; // required for latency test
repeated string reports = 7; // TODO: this will become an enum
uint64 price = 8; // nanoLP per unit per minute
}
message ExtendVmReq {
string uuid = 1;
string admin_pubkey = 2;
uint64 locked_nano = 3;
}
message AirdropReq {
string pubkey = 1;
uint64 tokens = 2;
}
message SlashReq {
string pubkey = 1;
uint64 tokens = 2;
}
message Account {
string pubkey = 1;
uint64 balance = 2;
uint64 tmp_locked = 3;
}
message RegOperatorReq {
string pubkey = 1;
uint64 escrow = 2;
string email = 3;
}
message ListOperatorsResp {
string pubkey = 1;
uint64 escrow = 2;
string email = 3;
uint64 app_nodes = 4;
uint64 vm_nodes = 5;
uint64 reports = 6;
}
message InspectOperatorResp {
ListOperatorsResp operator = 1;
repeated VmNodeListResp nodes = 2;
}
message ReportNodeReq {
string admin_pubkey = 1;
string node_pubkey = 2;
string contract = 3;
string reason = 4;
}
message KickReq {
string operator_wallet = 1;
string contract_uuid = 2;
string reason = 3;
}
message BanUserReq {
string operator_wallet = 1;
string user_wallet = 2;
}
message KickResp {
uint64 nano_lp = 1;
}
service BrainCli {
rpc GetBalance (Pubkey) returns (AccountBalance);
rpc NewVm (NewVmReq) returns (NewVmResp);
rpc ListVmContracts (ListVmContractsReq) returns (stream VmContract);
rpc ListVmNodes (VmNodeFilters) returns (stream VmNodeListResp);
rpc GetOneVmNode (VmNodeFilters) returns (VmNodeListResp);
rpc DeleteVm (DeleteVmReq) returns (Empty);
rpc UpdateVm (UpdateVmReq) returns (UpdateVmResp);
rpc ExtendVm (ExtendVmReq) returns (Empty);
rpc ReportNode (ReportNodeReq) returns (Empty);
rpc ListOperators (Empty) returns (stream ListOperatorsResp);
rpc InspectOperator (Pubkey) returns (InspectOperatorResp);
rpc RegisterOperator (RegOperatorReq) returns (Empty);
rpc KickContract (KickReq) returns (KickResp);
rpc BanUser (BanUserReq) returns (Empty);
// admin commands
rpc Airdrop (AirdropReq) returns (Empty);
rpc Slash (SlashReq) returns (Empty);
rpc ListAllVmContracts (Empty) returns (stream VmContract);
rpc ListAccounts (Empty) returns (stream Account);
}