Compare commits
10 Commits
1a4cec421b
...
75b25ab7d3
Author | SHA1 | Date | |
---|---|---|---|
75b25ab7d3 | |||
76e832a3f6 | |||
4e6277aed7 | |||
4689ab82c7 | |||
3cca48680b | |||
8e8e1d1a99 | |||
76c88810a6 | |||
0ed21fdf98 | |||
f74d2887fb | |||
56f88aad3f |
71
Cargo.lock
generated
71
Cargo.lock
generated
@ -26,6 +26,21 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "android-tzdata"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0"
|
||||
|
||||
[[package]]
|
||||
name = "android_system_properties"
|
||||
version = "0.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.18"
|
||||
@ -257,6 +272,20 @@ version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.39"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825"
|
||||
dependencies = [
|
||||
"android-tzdata",
|
||||
"iana-time-zone",
|
||||
"js-sys",
|
||||
"num-traits",
|
||||
"wasm-bindgen",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "colorchoice"
|
||||
version = "1.0.3"
|
||||
@ -347,6 +376,7 @@ version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bs58",
|
||||
"chrono",
|
||||
"ed25519-dalek",
|
||||
"env_logger",
|
||||
"lazy_static",
|
||||
@ -766,6 +796,29 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.61"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220"
|
||||
dependencies = [
|
||||
"android_system_properties",
|
||||
"core-foundation-sys",
|
||||
"iana-time-zone-haiku",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"windows-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone-haiku"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "icu_collections"
|
||||
version = "1.5.0"
|
||||
@ -1053,6 +1106,15 @@ dependencies = [
|
||||
"tempfile",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "num-traits"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "object"
|
||||
version = "0.36.5"
|
||||
@ -2083,6 +2145,15 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.52.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
|
||||
dependencies = [
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-registry"
|
||||
version = "0.2.0"
|
||||
|
@ -22,6 +22,13 @@ tokio-stream = "0.1.17"
|
||||
tonic = "0.12"
|
||||
serde_json = "1.0.135"
|
||||
bs58 = "0.5.1"
|
||||
chrono = "0.4.39"
|
||||
|
||||
[build-dependencies]
|
||||
tonic-build = "0.12"
|
||||
|
||||
[profile.release]
|
||||
opt-level = "z"
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
strip = true
|
||||
|
2
build.rs
2
build.rs
@ -1,6 +1,6 @@
|
||||
fn main() {
|
||||
tonic_build::configure()
|
||||
.build_server(true)
|
||||
.compile_protos(&["snp.proto"], &["proto"])
|
||||
.compile_protos(&["vm.proto"], &["proto"])
|
||||
.unwrap_or_else(|e| panic!("Failed to compile protos {:?}", e));
|
||||
}
|
||||
|
@ -1,13 +0,0 @@
|
||||
uuid: "uuid-0001"
|
||||
hostname: "ghe-vm-1"
|
||||
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
|
||||
extra_ports: [80]
|
||||
public_ipv4: false
|
||||
public_ipv6: true
|
||||
disk_size_gb: 20
|
||||
vcpus: 2
|
||||
memory_mb: 2000
|
||||
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
|
||||
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
|
||||
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
|
||||
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"
|
@ -1,28 +0,0 @@
|
||||
brain_url: "http://164.92.249.180:31337"
|
||||
max_cores_per_vm: 4
|
||||
max_vcpu_reservation: 8
|
||||
max_mem_reservation_mb: 25000
|
||||
network_interfaces:
|
||||
- driver: "MACVTAP"
|
||||
device: "eno8303"
|
||||
ipv4_ranges:
|
||||
- first_ip: "173.234.136.154"
|
||||
last_ip: "173.234.136.155"
|
||||
netmask: "27"
|
||||
gateway: "173.234.136.158"
|
||||
- first_ip: "173.234.137.17"
|
||||
last_ip: "173.234.137.17"
|
||||
netmask: "27"
|
||||
gateway: "173.234.137.30"
|
||||
ipv6_ranges:
|
||||
- first_ip: "2a0d:3003:b666:a00c:0002:0000:0000:0011"
|
||||
last_ip: "2a0d:3003:b666:a00c:0002:0000:0000:fffc"
|
||||
netmask: "64"
|
||||
gateway: "2a0d:3003:b666:a00c::1"
|
||||
volumes:
|
||||
- path: "/opt/detee_vms/"
|
||||
max_reservation_gb: 200
|
||||
public_port_range:
|
||||
start: 30000
|
||||
end: 50000
|
||||
max_ports_per_vm: 5
|
@ -1,13 +0,0 @@
|
||||
uuid: "uuid-0002"
|
||||
hostname: "ghe-vm-2"
|
||||
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
|
||||
extra_ports: []
|
||||
public_ipv4: true
|
||||
public_ipv6: true
|
||||
disk_size_gb: 20
|
||||
vcpus: 2
|
||||
memory_mb: 2000
|
||||
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
|
||||
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
|
||||
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
|
||||
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"
|
@ -1,13 +0,0 @@
|
||||
uuid: "uuid-0004"
|
||||
hostname: "ghe-vm-4"
|
||||
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
|
||||
extra_ports: []
|
||||
public_ipv4: false
|
||||
public_ipv6: false
|
||||
disk_size_gb: 20
|
||||
vcpus: 2
|
||||
memory_mb: 2000
|
||||
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
|
||||
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
|
||||
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
|
||||
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"
|
@ -1,13 +0,0 @@
|
||||
uuid: "uuid-0003"
|
||||
hostname: "ghe-vm-3"
|
||||
admin_key: "MCowBQYDK2VwAyEAEoJ50VwJc7noWxylhioU2kk35MkO5as4U92UbP2A7xk="
|
||||
extra_ports: []
|
||||
public_ipv4: true
|
||||
public_ipv6: false
|
||||
disk_size_gb: 20
|
||||
vcpus: 2
|
||||
memory_mb: 2000
|
||||
kernel_url: "https://drive.google.com/uc?export=download&id=1bc2CmJjIBFSXRxTFQJy11uSobsazTs4n"
|
||||
kernel_sha: "203352667d403b437c856bec16809604e801e4f0cfabbf938daa8fda2fad0f84"
|
||||
dtrfs_url: "https://drive.google.com/uc?export=download&id=1WoAVb9VS0rlzmuEIwwzZMx7N7SsUQf6q"
|
||||
dtrfs_sha: "9fe9ae795a239a426a290f21dfa857182e3fe2fa556f8e8997ab35dfeb8fca24"
|
26
scripts/install_daemon.sh
Executable file
26
scripts/install_daemon.sh
Executable file
@ -0,0 +1,26 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
echo "Creating folders..."
|
||||
mkdir -p /var/lib/detee/boot/
|
||||
mkdir -p /etc/detee/daemon/vms/
|
||||
mkdir -p /usr/local/bin/detee/
|
||||
mkdir -p /opt/detee_vms/
|
||||
echo "Installing qemu-system-x86..."
|
||||
pacman -S qemu-system-x86 qemu-img --noconfirm
|
||||
|
||||
echo "Downloading detee-snp-daemon, systemd unit file and config..."
|
||||
wget -O /etc/detee/daemon/sample_config.yaml https://registry.detee.ltd/daemon/config.yaml
|
||||
wget -O /usr/local/bin/detee-snp-daemon https://registry.detee.ltd/daemon/detee-snp-daemon
|
||||
chmod +x /usr/local/bin/detee-snp-daemon
|
||||
wget -O /usr/local/bin/detee/start_qemu_vm.sh https://registry.detee.ltd/daemon/start_qemu_vm.sh
|
||||
chmod +x /usr/local/bin/detee/start_qemu_vm.sh
|
||||
wget -O /etc/systemd/system/detee-snp-daemon.service https://registry.detee.ltd/daemon/detee-snp-daemon.service
|
||||
|
||||
echo "Take a look at /etc/detee/daemon/sample_config.yaml"
|
||||
echo "Modify config based on your setup and save it to /etc/detee/daemon/config.yaml"
|
||||
echo "Press enter when done (this will attempt to start the daemon)"
|
||||
read my_var
|
||||
|
||||
echo "Starting detee-snp-daemon..."
|
||||
systemctl daemon-reload
|
||||
systemctl start detee-snp-daemon.service
|
@ -24,6 +24,7 @@ add_nft_rules() {
|
||||
nft add chain netdev deteemacvtap ${ifname}_ou "{ type filter hook egress device ${ifname} priority 0; policy accept; }"
|
||||
# return if the rules already exist
|
||||
nft list chain netdev deteemacvtap ${ifname}_in | grep ether && return 0
|
||||
nft add rule netdev deteemacvtap ${ifname}_in ether type arp accept
|
||||
nft add rule netdev deteemacvtap ${ifname}_in ether daddr != ${vtap_addr} drop
|
||||
nft list chain netdev deteemacvtap ${ifname}_ou | grep ether && return 0
|
||||
nft add rule netdev deteemacvtap ${ifname}_ou ether saddr != ${vtap_addr} drop
|
||||
@ -87,7 +88,7 @@ while read -r interface; do
|
||||
for port_pair in $NAT_PORT_FW; do
|
||||
host_port="$( echo $port_pair | cut -d ':' -f1 )"
|
||||
guest_port="$( echo $port_pair | cut -d ':' -f2 )"
|
||||
ports+=",hostfwd=tcp::${host_port}-:${guest_port}"
|
||||
ports+=",hostfwd=tcp::${host_port}-:${guest_port},hostfwd=udp::${host_port}-:${guest_port}"
|
||||
done
|
||||
qemu_device_params+=" -netdev user,id=natnic${ports}"
|
||||
qemu_device_params+=" -device virtio-net-pci,netdev=natnic,romfile="
|
||||
|
179
snp.proto
179
snp.proto
@ -1,179 +0,0 @@
|
||||
syntax = "proto3";
|
||||
package snp_proto;
|
||||
|
||||
message Empty {
|
||||
}
|
||||
|
||||
message Pubkey {
|
||||
string pubkey = 1;
|
||||
}
|
||||
|
||||
message Contract {
|
||||
string uuid = 1;
|
||||
string hostname = 2;
|
||||
string admin_pubkey = 3;
|
||||
string node_pubkey = 4;
|
||||
repeated uint32 exposed_ports = 5;
|
||||
string public_ipv4 = 6;
|
||||
string public_ipv6 = 7;
|
||||
uint32 disk_size_gb = 8;
|
||||
uint32 vcpus = 9;
|
||||
uint32 memory_mb = 10;
|
||||
string kernel_sha = 11;
|
||||
string dtrfs_sha = 12;
|
||||
string created_at = 13;
|
||||
string updated_at = 14;
|
||||
// total nanotoken cost per minute (for all units)
|
||||
uint64 nano_per_minute = 15;
|
||||
uint64 locked_nano = 16;
|
||||
string collected_at = 17;
|
||||
}
|
||||
|
||||
message MeasurementArgs {
|
||||
// this will be IP:Port of the dtrfs API
|
||||
// actually not a measurement arg, but needed for the injector
|
||||
string dtrfs_api_endpoint = 1;
|
||||
repeated uint32 exposed_ports = 2;
|
||||
string ovmf_hash = 5;
|
||||
// This is needed to allow the CLI to build the kernel params from known data.
|
||||
// The CLI will use the kernel params to get the measurement.
|
||||
repeated MeasurementIP ips = 6;
|
||||
}
|
||||
|
||||
message MeasurementIP {
|
||||
uint32 nic_index = 1;
|
||||
string address = 2;
|
||||
string mask = 3;
|
||||
string gateway = 4;
|
||||
}
|
||||
|
||||
message RegisterNodeReq {
|
||||
string node_pubkey = 1;
|
||||
string owner_pubkey = 2;
|
||||
string main_ip = 3;
|
||||
string country = 4;
|
||||
string region = 5;
|
||||
string city = 6;
|
||||
// nanotokens per unit per minute
|
||||
uint64 price = 7;
|
||||
}
|
||||
|
||||
message NodeResources {
|
||||
string node_pubkey = 1;
|
||||
uint32 avail_ports = 2;
|
||||
uint32 avail_ipv4 = 3;
|
||||
uint32 avail_ipv6 = 4;
|
||||
uint32 avail_vcpus = 5;
|
||||
uint32 avail_memory_mb = 6;
|
||||
uint32 avail_storage_gb = 7;
|
||||
uint32 max_ports_per_vm = 8;
|
||||
}
|
||||
|
||||
message NewVmReq {
|
||||
string uuid = 1;
|
||||
string hostname = 2;
|
||||
string admin_pubkey = 3;
|
||||
string node_pubkey = 4;
|
||||
repeated uint32 extra_ports = 5;
|
||||
bool public_ipv4 = 6;
|
||||
bool public_ipv6 = 7;
|
||||
uint32 disk_size_gb = 8;
|
||||
uint32 vcpus = 9;
|
||||
uint32 memory_mb = 10;
|
||||
string kernel_url = 11;
|
||||
string kernel_sha = 12;
|
||||
string dtrfs_url = 13;
|
||||
string dtrfs_sha = 14;
|
||||
uint64 price_per_unit = 15;
|
||||
uint64 locked_nano = 16;
|
||||
}
|
||||
|
||||
message NewVmResp {
|
||||
string uuid = 1;
|
||||
string error = 2;
|
||||
MeasurementArgs args = 3;
|
||||
}
|
||||
|
||||
message UpdateVmReq {
|
||||
string uuid = 1;
|
||||
uint32 disk_size_gb = 2;
|
||||
uint32 vcpus = 3;
|
||||
uint32 memory_mb = 4;
|
||||
string kernel_url = 5;
|
||||
string kernel_sha = 6;
|
||||
string dtrfs_url = 7;
|
||||
string dtrfs_sha = 8;
|
||||
}
|
||||
|
||||
message UpdateVmResp {
|
||||
string uuid = 1;
|
||||
string error = 2;
|
||||
MeasurementArgs args = 3;
|
||||
}
|
||||
|
||||
message DeleteVmReq {
|
||||
string uuid = 1;
|
||||
}
|
||||
|
||||
message BrainMessage {
|
||||
oneof Msg {
|
||||
NewVmReq new_vm_req = 1;
|
||||
UpdateVmReq update_vm_req = 2;
|
||||
DeleteVmReq delete_vm = 3;
|
||||
}
|
||||
}
|
||||
|
||||
message DaemonMessage {
|
||||
oneof Msg {
|
||||
Pubkey pubkey = 1;
|
||||
NewVmResp new_vm_resp = 2;
|
||||
UpdateVmResp update_vm_resp = 3;
|
||||
NodeResources node_resources = 4;
|
||||
}
|
||||
}
|
||||
|
||||
service BrainDaemon {
|
||||
rpc RegisterNode (RegisterNodeReq) returns (stream Contract);
|
||||
rpc BrainMessages (Pubkey) returns (stream BrainMessage);
|
||||
rpc DaemonMessages (stream DaemonMessage) returns (Empty);
|
||||
}
|
||||
|
||||
message ListContractsReq {
|
||||
string admin_pubkey = 1;
|
||||
string node_pubkey = 2;
|
||||
string uuid = 3;
|
||||
}
|
||||
|
||||
message NodeFilters {
|
||||
uint32 free_ports = 1;
|
||||
bool offers_ipv4 = 2;
|
||||
bool offers_ipv6 = 3;
|
||||
uint32 vcpus = 4;
|
||||
uint32 memory_mb = 5;
|
||||
uint32 storage_gb = 6;
|
||||
string country = 7;
|
||||
string region = 8;
|
||||
string city = 9;
|
||||
string ip = 10;
|
||||
}
|
||||
|
||||
message NodeListResp {
|
||||
string node_pubkey = 1;
|
||||
string country = 2;
|
||||
string region = 3;
|
||||
string city = 4;
|
||||
string ip = 5; // required for latency test
|
||||
uint32 server_rating = 6;
|
||||
uint32 provider_rating = 7;
|
||||
// nanotokens per unit per minute
|
||||
uint64 price = 8;
|
||||
}
|
||||
|
||||
service BrainCli {
|
||||
rpc NewVm (NewVmReq) returns (NewVmResp);
|
||||
rpc ListContracts (ListContractsReq) returns (stream Contract);
|
||||
rpc ListNodes (NodeFilters) returns (stream NodeListResp);
|
||||
rpc GetOneNode (NodeFilters) returns (NodeListResp);
|
||||
rpc DeleteVm (DeleteVmReq) returns (Empty);
|
||||
rpc UpdateVm (UpdateVmReq) returns (UpdateVmResp);
|
||||
}
|
@ -44,6 +44,7 @@ pub enum InterfaceType {
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct Config {
|
||||
pub owner_wallet: String,
|
||||
pub brain_url: String,
|
||||
pub max_cores_per_vm: usize,
|
||||
pub max_vcpu_reservation: usize,
|
||||
|
@ -2,7 +2,8 @@ use anyhow::Result;
|
||||
use ed25519_dalek::SigningKey;
|
||||
use lazy_static::lazy_static;
|
||||
use log::{info, warn};
|
||||
use std::{fs::File, io::Write};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{fs::File, io::Read, io::Write};
|
||||
|
||||
pub(crate) const VM_BOOT_DIR: &str = "/var/lib/detee/boot/";
|
||||
pub(crate) const USED_RESOURCES: &str = "/etc/detee/daemon/used_resources.yaml";
|
||||
@ -49,6 +50,12 @@ fn load_secret_key() -> Result<ed25519_dalek::SigningKey> {
|
||||
))
|
||||
}
|
||||
|
||||
pub fn sign_message(msg: &str) -> Result<String> {
|
||||
use ed25519_dalek::Signer;
|
||||
let key = load_secret_key()?;
|
||||
Ok(bs58::encode(key.sign(msg.as_bytes()).to_bytes()).into_string())
|
||||
}
|
||||
|
||||
pub fn get_public_key() -> String {
|
||||
let pubkey = bs58::encode(load_secret_key().unwrap().verifying_key().to_bytes()).into_string();
|
||||
log::info!("Loaded the following public key: {pubkey}");
|
||||
@ -68,3 +75,18 @@ fn get_ip_info() -> anyhow::Result<IPInfo> {
|
||||
info!("Got the following data from ipinfo.io: {body}");
|
||||
Ok(serde_json::de::from_str(&body)?)
|
||||
}
|
||||
|
||||
pub fn compute_sha256<P: AsRef<std::path::Path>>(path: P) -> Result<String> {
|
||||
let mut file = File::open(path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = [0u8; 8192];
|
||||
loop {
|
||||
let bytes_read = file.read(&mut buffer)?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..bytes_read]);
|
||||
}
|
||||
let result = hasher.finalize();
|
||||
Ok(format!("{:x}", result))
|
||||
}
|
||||
|
94
src/grpc.rs
94
src/grpc.rs
@ -1,54 +1,66 @@
|
||||
use crate::snp_proto::DaemonMessage;
|
||||
use crate::global::*;
|
||||
use crate::snp_proto::VmDaemonMessage;
|
||||
use anyhow::Result;
|
||||
use log::{debug, info, warn};
|
||||
use snp_proto::{
|
||||
brain_daemon_client::BrainDaemonClient, BrainMessage, Contract, Pubkey, RegisterNodeReq,
|
||||
};
|
||||
use snp_proto::{brain_vm_daemon_client::BrainVmDaemonClient, BrainVmMessage, VmContract, RegisterVmNodeReq};
|
||||
use tokio::{
|
||||
sync::mpsc::{Receiver, Sender},
|
||||
task::JoinSet,
|
||||
};
|
||||
use tokio_stream::{wrappers::ReceiverStream, StreamExt};
|
||||
use tonic::transport::Channel;
|
||||
use crate::global::*;
|
||||
|
||||
pub mod snp_proto {
|
||||
tonic::include_proto!("snp_proto");
|
||||
tonic::include_proto!("vm_proto");
|
||||
}
|
||||
|
||||
impl From<snp_proto::NewVmResp> for snp_proto::DaemonMessage {
|
||||
impl From<snp_proto::NewVmResp> for snp_proto::VmDaemonMessage {
|
||||
fn from(value: snp_proto::NewVmResp) -> Self {
|
||||
snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::NewVmResp(value)) }
|
||||
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::NewVmResp(value)) }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<snp_proto::UpdateVmResp> for snp_proto::DaemonMessage {
|
||||
impl From<snp_proto::UpdateVmResp> for snp_proto::VmDaemonMessage {
|
||||
fn from(value: snp_proto::UpdateVmResp) -> Self {
|
||||
snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::UpdateVmResp(value)) }
|
||||
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::UpdateVmResp(value)) }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<snp_proto::NodeResources> for snp_proto::DaemonMessage {
|
||||
fn from(value: snp_proto::NodeResources) -> Self {
|
||||
snp_proto::DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::NodeResources(value)) }
|
||||
impl From<snp_proto::VmNodeResources> for snp_proto::VmDaemonMessage {
|
||||
fn from(value: snp_proto::VmNodeResources) -> Self {
|
||||
snp_proto::VmDaemonMessage { msg: Some(snp_proto::vm_daemon_message::Msg::VmNodeResources(value)) }
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn register_node(config: &crate::config::Config) -> Result<Vec<Contract>> {
|
||||
let mut client = BrainDaemonClient::connect(config.brain_url.clone()).await?;
|
||||
pub async fn register_node(config: &crate::config::Config) -> Result<Vec<VmContract>> {
|
||||
use tonic::metadata::AsciiMetadataValue;
|
||||
use tonic::Request;
|
||||
let mut client = BrainVmDaemonClient::connect(config.brain_url.clone()).await?;
|
||||
debug!("Starting node registration...");
|
||||
let ip_info = IP_INFO.clone();
|
||||
let req = RegisterNodeReq {
|
||||
let req = RegisterVmNodeReq {
|
||||
node_pubkey: PUBLIC_KEY.clone(),
|
||||
owner_pubkey: "IamTheOwnerOf".to_string() + &PUBLIC_KEY,
|
||||
operator_wallet: config.owner_wallet.clone(),
|
||||
main_ip: ip_info.ip,
|
||||
country: ip_info.country,
|
||||
region: ip_info.region,
|
||||
city: ip_info.city,
|
||||
price: config.price,
|
||||
};
|
||||
|
||||
let pubkey = PUBLIC_KEY.clone();
|
||||
let timestamp = chrono::Utc::now().to_rfc3339();
|
||||
let signature = crate::global::sign_message(&format!("{timestamp}{req:?}"))?;
|
||||
let timestamp: AsciiMetadataValue = timestamp.parse()?;
|
||||
let pubkey: AsciiMetadataValue = pubkey.parse()?;
|
||||
let signature: AsciiMetadataValue = signature.parse()?;
|
||||
let mut req = Request::new(req);
|
||||
req.metadata_mut().insert("timestamp", timestamp);
|
||||
req.metadata_mut().insert("pubkey", pubkey);
|
||||
req.metadata_mut().insert("request-signature", signature);
|
||||
|
||||
let mut contracts = Vec::new();
|
||||
let mut grpc_stream = client.register_node(req).await?.into_inner();
|
||||
let mut grpc_stream = client.register_vm_node(req).await?.into_inner();
|
||||
while let Some(stream_update) = grpc_stream.next().await {
|
||||
match stream_update {
|
||||
Ok(node) => {
|
||||
@ -64,13 +76,21 @@ pub async fn register_node(config: &crate::config::Config) -> Result<Vec<Contrac
|
||||
Ok(contracts)
|
||||
}
|
||||
|
||||
fn sign_stream_auth(contracts: Vec<String>) -> Result<snp_proto::DaemonStreamAuth> {
|
||||
let pubkey = PUBLIC_KEY.clone();
|
||||
let timestamp = chrono::Utc::now().to_rfc3339();
|
||||
let signature =
|
||||
crate::global::sign_message(&(timestamp.to_string() + &format!("{contracts:?}")))?;
|
||||
Ok(snp_proto::DaemonStreamAuth { timestamp, pubkey, contracts, signature })
|
||||
}
|
||||
|
||||
async fn receive_messages(
|
||||
mut client: BrainDaemonClient<Channel>,
|
||||
tx: Sender<BrainMessage>,
|
||||
mut client: BrainVmDaemonClient<Channel>,
|
||||
contracts: Vec<String>,
|
||||
tx: Sender<BrainVmMessage>,
|
||||
) -> Result<()> {
|
||||
debug!("starting to listen for messages from brain");
|
||||
let pubkey = PUBLIC_KEY.clone();
|
||||
let mut grpc_stream = client.brain_messages(Pubkey { pubkey }).await?.into_inner();
|
||||
let mut grpc_stream = client.brain_messages(sign_stream_auth(contracts)?).await?.into_inner();
|
||||
while let Some(stream_update) = grpc_stream.next().await {
|
||||
match stream_update {
|
||||
Ok(msg) => {
|
||||
@ -87,14 +107,16 @@ async fn receive_messages(
|
||||
}
|
||||
|
||||
async fn send_messages(
|
||||
mut client: BrainDaemonClient<Channel>,
|
||||
rx: Receiver<DaemonMessage>,
|
||||
tx: Sender<DaemonMessage>,
|
||||
mut client: BrainVmDaemonClient<Channel>,
|
||||
contracts: Vec<String>,
|
||||
rx: Receiver<VmDaemonMessage>,
|
||||
tx: Sender<VmDaemonMessage>,
|
||||
) -> Result<()> {
|
||||
debug!("starting daemon message stream to brain");
|
||||
let pubkey = PUBLIC_KEY.clone();
|
||||
let rx_stream = ReceiverStream::new(rx);
|
||||
tx.send(DaemonMessage { msg: Some(snp_proto::daemon_message::Msg::Pubkey(Pubkey { pubkey })) })
|
||||
tx.send(VmDaemonMessage {
|
||||
msg: Some(snp_proto::vm_daemon_message::Msg::Auth(sign_stream_auth(contracts)?)),
|
||||
})
|
||||
.await?;
|
||||
client.daemon_messages(rx_stream).await?;
|
||||
debug!("send_newvm_resp is about to exit");
|
||||
@ -102,18 +124,24 @@ async fn send_messages(
|
||||
}
|
||||
|
||||
pub struct ConnectionData {
|
||||
pub contracts: Vec<String>,
|
||||
pub brain_url: String,
|
||||
pub brain_msg_tx: Sender<BrainMessage>,
|
||||
pub daemon_msg_rx: Receiver<DaemonMessage>,
|
||||
pub daemon_msg_tx: Sender<DaemonMessage>,
|
||||
pub brain_msg_tx: Sender<BrainVmMessage>,
|
||||
pub daemon_msg_rx: Receiver<VmDaemonMessage>,
|
||||
pub daemon_msg_tx: Sender<VmDaemonMessage>,
|
||||
}
|
||||
|
||||
pub async fn connect_and_run(cd: ConnectionData) -> Result<()> {
|
||||
let client = BrainDaemonClient::connect(cd.brain_url).await?;
|
||||
let client = BrainVmDaemonClient::connect(cd.brain_url).await?;
|
||||
let mut streaming_tasks = JoinSet::new();
|
||||
|
||||
streaming_tasks.spawn(receive_messages(client.clone(), cd.brain_msg_tx));
|
||||
streaming_tasks.spawn(send_messages(client.clone(), cd.daemon_msg_rx, cd.daemon_msg_tx));
|
||||
streaming_tasks.spawn(receive_messages(client.clone(), cd.contracts.clone(), cd.brain_msg_tx));
|
||||
streaming_tasks.spawn(send_messages(
|
||||
client.clone(),
|
||||
cd.contracts,
|
||||
cd.daemon_msg_rx,
|
||||
cd.daemon_msg_tx,
|
||||
));
|
||||
|
||||
let task_output = streaming_tasks.join_next().await;
|
||||
warn!("One stream exited: {task_output:?}");
|
||||
|
93
src/main.rs
93
src/main.rs
@ -5,8 +5,9 @@ mod state;
|
||||
|
||||
use crate::global::*;
|
||||
use crate::{config::Config, grpc::snp_proto};
|
||||
use anyhow::Result;
|
||||
use anyhow::{anyhow, Result};
|
||||
use log::{debug, info, warn};
|
||||
use std::{fs::File, path::Path};
|
||||
use tokio::{
|
||||
sync::mpsc::{Receiver, Sender},
|
||||
time::{sleep, Duration},
|
||||
@ -14,8 +15,8 @@ use tokio::{
|
||||
|
||||
#[allow(dead_code)]
|
||||
struct VMHandler {
|
||||
receiver: Receiver<snp_proto::BrainMessage>,
|
||||
sender: Sender<snp_proto::DaemonMessage>,
|
||||
receiver: Receiver<snp_proto::BrainVmMessage>,
|
||||
sender: Sender<snp_proto::VmDaemonMessage>,
|
||||
config: Config,
|
||||
res: state::Resources,
|
||||
}
|
||||
@ -23,8 +24,8 @@ struct VMHandler {
|
||||
#[allow(dead_code)]
|
||||
impl VMHandler {
|
||||
fn new(
|
||||
receiver: Receiver<snp_proto::BrainMessage>,
|
||||
sender: Sender<snp_proto::DaemonMessage>,
|
||||
receiver: Receiver<snp_proto::BrainVmMessage>,
|
||||
sender: Sender<snp_proto::VmDaemonMessage>,
|
||||
) -> Self {
|
||||
let config = match Config::load_from_disk(DAEMON_CONFIG_PATH) {
|
||||
Ok(config) => config,
|
||||
@ -33,9 +34,8 @@ impl VMHandler {
|
||||
let res = match state::Resources::load_from_disk() {
|
||||
Ok(res) => res,
|
||||
Err(e) => {
|
||||
warn!("Could not load resources from disk: {e:?}");
|
||||
info!("Creating new resource calculator.");
|
||||
state::Resources::new(&config.volumes)
|
||||
log::error!("Could calculate resources: {e:?}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
};
|
||||
Self { receiver, sender, config, res }
|
||||
@ -60,15 +60,19 @@ impl VMHandler {
|
||||
|
||||
async fn send_node_resources(&mut self) {
|
||||
let (avail_ipv4, avail_ipv6) = self.get_available_ips();
|
||||
let mut avail_storage_gb = 0;
|
||||
let mut total_gb_available = 0;
|
||||
for volume in self.config.volumes.iter() {
|
||||
avail_storage_gb += volume.max_reservation_gb;
|
||||
if let Some(reservation) = self.res.reserved_storage.get(&volume.path) {
|
||||
avail_storage_gb -= reservation;
|
||||
let reservation: usize = match self.res.reserved_storage.get(&volume.path) {
|
||||
Some(reserved) => *reserved,
|
||||
None => 0 as usize,
|
||||
};
|
||||
let volume_gb_available = volume.max_reservation_gb - reservation;
|
||||
if total_gb_available < volume_gb_available {
|
||||
total_gb_available = volume_gb_available;
|
||||
}
|
||||
}
|
||||
let avail_storage_gb = avail_storage_gb as u32;
|
||||
let res = snp_proto::NodeResources {
|
||||
let avail_storage_gb = total_gb_available as u32;
|
||||
let res = snp_proto::VmNodeResources {
|
||||
node_pubkey: PUBLIC_KEY.clone(),
|
||||
avail_ports: (self.config.public_port_range.len() - self.res.reserved_ports.len())
|
||||
as u32,
|
||||
@ -140,8 +144,7 @@ impl VMHandler {
|
||||
async fn handle_update_vm_req(&mut self, update_vm_req: snp_proto::UpdateVmReq) -> Result<()> {
|
||||
debug!("Processing update vm request: {update_vm_req:?}");
|
||||
let vm_id = update_vm_req.uuid.clone();
|
||||
let content =
|
||||
std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
|
||||
let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
|
||||
let mut vm: state::VM = serde_yaml::from_str(&content)?;
|
||||
match vm.update(update_vm_req.into(), &self.config, &mut self.res) {
|
||||
Ok(_) => {
|
||||
@ -170,26 +173,26 @@ impl VMHandler {
|
||||
|
||||
fn handle_delete_vm(&mut self, delete_vm_req: snp_proto::DeleteVmReq) -> Result<()> {
|
||||
let vm_id = delete_vm_req.uuid;
|
||||
let content =
|
||||
std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
|
||||
let content = std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &vm_id + ".yaml")?;
|
||||
let vm: state::VM = serde_yaml::from_str(&content)?;
|
||||
vm.delete(&mut self.res)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn run(mut self) {
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
self.send_node_resources().await;
|
||||
while let Some(brain_msg) = self.receiver.recv().await {
|
||||
match brain_msg.msg {
|
||||
Some(snp_proto::brain_message::Msg::NewVmReq(new_vm_req)) => {
|
||||
Some(snp_proto::brain_vm_message::Msg::NewVmReq(new_vm_req)) => {
|
||||
self.handle_new_vm_req(new_vm_req).await;
|
||||
}
|
||||
Some(snp_proto::brain_message::Msg::UpdateVmReq(update_vm_req)) => {
|
||||
Some(snp_proto::brain_vm_message::Msg::UpdateVmReq(update_vm_req)) => {
|
||||
if let Err(e) = self.handle_update_vm_req(update_vm_req).await {
|
||||
log::error!("Could not update vm: {e:?}");
|
||||
}
|
||||
}
|
||||
Some(snp_proto::brain_message::Msg::DeleteVm(delete_vm_req)) => {
|
||||
Some(snp_proto::brain_vm_message::Msg::DeleteVm(delete_vm_req)) => {
|
||||
let uuid = delete_vm_req.uuid.clone();
|
||||
if let Err(e) = self.handle_delete_vm(delete_vm_req) {
|
||||
log::error!("Could not delete vm {uuid}: {e:?}");
|
||||
@ -202,16 +205,17 @@ impl VMHandler {
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_deleted_contracts(&mut self, contracts: Vec<snp_proto::Contract>) {
|
||||
fn clear_deleted_contracts(&mut self, contracts: Vec<snp_proto::VmContract>) {
|
||||
for uuid in self.res.existing_vms.clone() {
|
||||
if contracts.iter().find(|c| c.uuid == uuid).is_none() {
|
||||
info!("VM {uuid} exists locally but not found in brain. Deleting...");
|
||||
let content = match std::fs::read_to_string(
|
||||
VM_CONFIG_DIR.to_string() + &uuid + ".yaml",
|
||||
) {
|
||||
let content =
|
||||
match std::fs::read_to_string(VM_CONFIG_DIR.to_string() + &uuid + ".yaml") {
|
||||
Ok(content) => content,
|
||||
Err(e) => {
|
||||
log::error!("Could not find VM config for {uuid}. Cannot delete VM: {e:?}");
|
||||
log::error!(
|
||||
"Could not find VM config for {uuid}. Cannot delete VM: {e:?}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
@ -236,6 +240,13 @@ async fn main() {
|
||||
env_logger::builder().filter_level(log::LevelFilter::Debug).init();
|
||||
|
||||
loop {
|
||||
if std::env::var("DAEMON_AUTO_UPGRADE") != Ok("OFF".to_string()) {
|
||||
// This upgrade procedure will get replaced in prod. We need this for the testnet.
|
||||
if let Err(e) = download_and_replace_binary() {
|
||||
log::error!("Failed to upgrade detee-snp-daemon to newer version: {e}");
|
||||
}
|
||||
}
|
||||
|
||||
let (brain_msg_tx, brain_msg_rx) = tokio::sync::mpsc::channel(6);
|
||||
let (daemon_msg_tx, daemon_msg_rx) = tokio::sync::mpsc::channel(6);
|
||||
|
||||
@ -243,8 +254,12 @@ async fn main() {
|
||||
let brain_url = vm_handler.config.brain_url.clone();
|
||||
|
||||
info!("Registering with the brain and getting back VM Contracts (if they exist).");
|
||||
let mut contracts: Vec<String> = Vec::new();
|
||||
match grpc::register_node(&vm_handler.config).await {
|
||||
Ok(contracts) => vm_handler.clear_deleted_contracts(contracts),
|
||||
Ok(c) => {
|
||||
contracts.append(&mut c.iter().map(|c| c.uuid.clone()).collect());
|
||||
vm_handler.clear_deleted_contracts(c)
|
||||
}
|
||||
Err(e) => log::error!("Could not get contracts from brain: {e:?}"),
|
||||
};
|
||||
|
||||
@ -254,6 +269,7 @@ async fn main() {
|
||||
|
||||
info!("Connecting to brain...");
|
||||
if let Err(e) = grpc::connect_and_run(grpc::ConnectionData {
|
||||
contracts,
|
||||
brain_url,
|
||||
brain_msg_tx,
|
||||
daemon_msg_rx,
|
||||
@ -266,3 +282,26 @@ async fn main() {
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
}
|
||||
}
|
||||
|
||||
fn download_and_replace_binary() -> Result<()> {
|
||||
use reqwest::blocking::get;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
const TMP_DAEMON: &str = "/usr/local/bin/detee/new-daemon";
|
||||
const BINARY: &str = "/usr/local/bin/detee-snp-daemon";
|
||||
let response = get("https://registry.detee.ltd/daemon/detee-snp-daemon")?;
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow!("Failed to download file: {}", response.status()));
|
||||
}
|
||||
let mut tmp_file = File::create(Path::new(&TMP_DAEMON))?;
|
||||
std::io::copy(&mut response.bytes()?.as_ref(), &mut tmp_file)?;
|
||||
let new_hash = crate::global::compute_sha256(&TMP_DAEMON)?;
|
||||
let old_hash = crate::global::compute_sha256(&BINARY)?;
|
||||
log::debug!("Old binary hash: {old_hash}. New binary hash: {new_hash}");
|
||||
if new_hash != old_hash {
|
||||
std::fs::rename(BINARY, BINARY.to_string() + "_BACKUP")?;
|
||||
std::fs::rename(TMP_DAEMON, BINARY)?;
|
||||
std::fs::set_permissions(BINARY, std::fs::Permissions::from_mode(0o775))?;
|
||||
std::process::exit(0);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
113
src/state.rs
113
src/state.rs
@ -3,17 +3,15 @@ use crate::{config::Config, global::*, grpc::snp_proto};
|
||||
use anyhow::{anyhow, Result};
|
||||
use log::info;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
fs,
|
||||
fs::{remove_file, File},
|
||||
io::{Read, Write},
|
||||
path::Path,
|
||||
io::Write,
|
||||
process::Command,
|
||||
};
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[derive(Serialize, Deserialize, Debug, Default)]
|
||||
pub struct Resources {
|
||||
pub existing_vms: HashSet<String>,
|
||||
// QEMU does not support MHz limiation
|
||||
@ -36,9 +34,49 @@ impl Resources {
|
||||
}
|
||||
|
||||
pub fn load_from_disk() -> Result<Self> {
|
||||
let content = std::fs::read_to_string(USED_RESOURCES)?;
|
||||
let mut res: Self = serde_yaml::from_str(&content)?;
|
||||
let mut res = Self { ..Default::default() };
|
||||
|
||||
log::debug!("Reading VMs saved to disk to calculate used resources...");
|
||||
for entry in fs::read_dir(VM_CONFIG_DIR)? {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
if path.is_file() && path.to_string_lossy().ends_with(".yaml") {
|
||||
log::info!("Found VM config: {:?}", path.to_str());
|
||||
let content = std::fs::read_to_string(path)?;
|
||||
let vm: VM = serde_yaml::from_str(&content)?;
|
||||
res.existing_vms.insert(vm.uuid);
|
||||
res.reserved_vcpus = res.reserved_vcpus.saturating_add(vm.vcpus);
|
||||
res.reserved_memory = res.reserved_memory.saturating_add(vm.memory_mb);
|
||||
for (port, _) in vm.fw_ports.iter() {
|
||||
res.reserved_ports.insert(*port);
|
||||
}
|
||||
res.reserved_storage
|
||||
.entry(vm.storage_dir.clone())
|
||||
.and_modify(|gb| {
|
||||
*gb = gb.saturating_add(vm.disk_size_gb);
|
||||
})
|
||||
.or_insert(vm.disk_size_gb);
|
||||
for nic in vm.nics {
|
||||
for ip in nic.ips {
|
||||
if let Ok(ip_address) = ip.address.parse::<std::net::IpAddr>() {
|
||||
if ip_address.is_ipv4() {
|
||||
res.reserved_ipv4.insert(ip.address.clone());
|
||||
}
|
||||
if ip_address.is_ipv6() {
|
||||
res.reserved_ipv6.insert(ip.address);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(vtap_name) = nic.if_config.vtap_name() {
|
||||
res.reserved_if_names.insert(vtap_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
res.scan_boot_files().unwrap();
|
||||
|
||||
res.save_to_disk()?;
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
@ -71,7 +109,7 @@ impl Resources {
|
||||
let mut volumes = config.volumes.clone();
|
||||
for volume in volumes.iter_mut() {
|
||||
if let Some(reservation) = self.reserved_storage.get(&volume.path) {
|
||||
volume.max_reservation_gb -= reservation;
|
||||
volume.max_reservation_gb = volume.max_reservation_gb.saturating_sub(*reservation);
|
||||
}
|
||||
}
|
||||
volumes.sort_by_key(|v| v.max_reservation_gb);
|
||||
@ -251,7 +289,9 @@ impl Resources {
|
||||
}
|
||||
|
||||
fn free_vm_resources(&mut self, vm: &VM) {
|
||||
self.existing_vms.remove(&vm.uuid);
|
||||
if !self.existing_vms.remove(&vm.uuid) {
|
||||
return;
|
||||
}
|
||||
self.reserved_vcpus = self.reserved_vcpus.saturating_sub(vm.vcpus);
|
||||
self.reserved_memory = self.reserved_memory.saturating_sub(vm.memory_mb);
|
||||
for nic in vm.nics.iter() {
|
||||
@ -272,8 +312,12 @@ impl Resources {
|
||||
for (host_port, _) in vm.fw_ports.iter() {
|
||||
self.reserved_ports.remove(host_port);
|
||||
}
|
||||
self.reserved_storage.entry(vm.storage_dir.clone()).and_modify(|gb| *gb -= vm.disk_size_gb);
|
||||
let _ = self.save_to_disk();
|
||||
self.reserved_storage
|
||||
.entry(vm.storage_dir.clone())
|
||||
.and_modify(|gb| *gb = gb.saturating_sub(vm.disk_size_gb));
|
||||
if let Err(e) = self.save_to_disk() {
|
||||
log::error!("Could not save resources to disk: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -401,22 +445,14 @@ impl From<VM> for snp_proto::MeasurementArgs {
|
||||
impl From<VM> for snp_proto::NewVmResp {
|
||||
fn from(vm: VM) -> Self {
|
||||
let uuid = vm.uuid.clone();
|
||||
snp_proto::NewVmResp {
|
||||
uuid,
|
||||
args: Some(vm.into()),
|
||||
error: "".to_string(),
|
||||
}
|
||||
snp_proto::NewVmResp { uuid, args: Some(vm.into()), error: "".to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
impl From<VM> for snp_proto::UpdateVmResp {
|
||||
fn from(vm: VM) -> Self {
|
||||
let uuid = vm.uuid.clone();
|
||||
snp_proto::UpdateVmResp {
|
||||
uuid,
|
||||
args: Some(vm.into()),
|
||||
error: "".to_string(),
|
||||
}
|
||||
snp_proto::UpdateVmResp { uuid, args: Some(vm.into()), error: "".to_string() }
|
||||
}
|
||||
}
|
||||
|
||||
@ -628,20 +664,22 @@ impl VM {
|
||||
config: &Config,
|
||||
res: &mut Resources,
|
||||
) -> Result<(), VMCreationErrors> {
|
||||
if config.max_cores_per_vm < req.vcpus {
|
||||
if req.vcpus > 0 && config.max_cores_per_vm < req.vcpus {
|
||||
return Err(VMCreationErrors::TooManyCores);
|
||||
}
|
||||
if config.max_vcpu_reservation
|
||||
if req.vcpus > 0
|
||||
&& config.max_vcpu_reservation
|
||||
< res.reserved_vcpus.saturating_sub(self.vcpus).saturating_add(req.vcpus)
|
||||
{
|
||||
return Err(VMCreationErrors::NotEnoughCPU);
|
||||
}
|
||||
if config.max_mem_reservation_mb
|
||||
if req.memory_mb > 0
|
||||
&& config.max_mem_reservation_mb
|
||||
< res.reserved_memory.saturating_sub(self.memory_mb).saturating_add(req.memory_mb)
|
||||
{
|
||||
return Err(VMCreationErrors::NotEnoughMemory);
|
||||
}
|
||||
if req.disk_size_gb < self.disk_size_gb {
|
||||
if req.disk_size_gb > 0 && req.disk_size_gb < self.disk_size_gb {
|
||||
return Err(VMCreationErrors::DiskTooSmall);
|
||||
}
|
||||
|
||||
@ -683,9 +721,15 @@ impl VM {
|
||||
});
|
||||
let _ = res.save_to_disk();
|
||||
|
||||
if req.memory_mb != 0 {
|
||||
self.memory_mb = req.memory_mb;
|
||||
}
|
||||
if req.vcpus != 0 {
|
||||
self.vcpus = req.vcpus;
|
||||
}
|
||||
if req.disk_size_gb != 0 {
|
||||
self.disk_size_gb = req.disk_size_gb;
|
||||
}
|
||||
|
||||
if let Err(e) = systemctl_stop_and_disable(&self.uuid) {
|
||||
return Err(VMCreationErrors::HypervizorError(e.to_string()));
|
||||
@ -811,9 +855,9 @@ impl VM {
|
||||
vars += "\n";
|
||||
vars += &format!(r#"export VCPUS="{}""#, self.vcpus);
|
||||
vars += "\n";
|
||||
vars += &format!(r#"export MEMORY="{}M""#, self.memory_mb);
|
||||
vars += &format!(r#"export MEMORY="{}M""#, (self.memory_mb / 2 * 2));
|
||||
vars += "\n";
|
||||
vars += &format!(r#"export MAX_MEMORY="{}M""#, self.memory_mb + 256);
|
||||
vars += &format!(r#"export MAX_MEMORY="{}M""#, (self.memory_mb / 2 * 2) + 256);
|
||||
vars += "\n";
|
||||
vars += &format!(r#"export DISK="{}""#, self.disk_path());
|
||||
vars += "\n";
|
||||
@ -996,7 +1040,7 @@ fn download_and_check_sha(url: &str, sha: &str) -> Result<()> {
|
||||
}
|
||||
let mut file = File::create(Path::new(&save_path))?;
|
||||
copy(&mut response.bytes()?.as_ref(), &mut file)?;
|
||||
match compute_sha256(&save_path) {
|
||||
match crate::global::compute_sha256(&save_path) {
|
||||
Ok(hash) => {
|
||||
if hash != sha {
|
||||
return Err(anyhow!(
|
||||
@ -1010,18 +1054,3 @@ fn download_and_check_sha(url: &str, sha: &str) -> Result<()> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn compute_sha256<P: AsRef<Path>>(path: P) -> Result<String> {
|
||||
let mut file = fs::File::open(path)?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buffer = [0u8; 8192];
|
||||
loop {
|
||||
let bytes_read = file.read(&mut buffer)?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buffer[..bytes_read]);
|
||||
}
|
||||
let result = hasher.finalize();
|
||||
Ok(format!("{:x}", result))
|
||||
}
|
||||
|
@ -1,25 +0,0 @@
|
||||
max_cores_per_vm: 4
|
||||
max_vcpu_reservation: 8
|
||||
max_mem_reservation_mb: 16384
|
||||
network_interfaces:
|
||||
- driver: "MACVTAP"
|
||||
device: "eth0"
|
||||
ipv4:
|
||||
- subnet: "192.168.1.0/24"
|
||||
gateway: "192.168.1.1"
|
||||
reserved_addrs:
|
||||
- "192.168.1.100"
|
||||
- "192.168.1.101"
|
||||
ipv6:
|
||||
- subnet: "2001:db8::/32"
|
||||
gateway: "2001:db8::1"
|
||||
reserved_addrs:
|
||||
- "2001:db8::1234"
|
||||
- "2001:db8::5678"
|
||||
volumes:
|
||||
- path: "/mnt/storage"
|
||||
max_reservation_gb: 200
|
||||
public_port_range:
|
||||
start: 8000
|
||||
end: 9000
|
||||
max_ports_per_vm: 5
|
@ -1,36 +0,0 @@
|
||||
max_cores_per_vm: 16
|
||||
max_vcpu_reservation: 32
|
||||
max_mem_reservation_mb: 1265536
|
||||
network_interfaces:
|
||||
- driver: "Bridge"
|
||||
device: "br0"
|
||||
ipv4:
|
||||
- subnet: "10.0.0.0/16"
|
||||
gateway: "10.0.0.1"
|
||||
reserved_addrs:
|
||||
- "10.0.0.100"
|
||||
- "10.0.0.101"
|
||||
- "10.0.0.102"
|
||||
ipv6: []
|
||||
- driver: "IPVTAP"
|
||||
device: "tap1"
|
||||
ipv4:
|
||||
- subnet: "172.16.0.0/20"
|
||||
gateway: "172.16.0.1"
|
||||
reserved_addrs:
|
||||
- "172.16.0.10"
|
||||
- "172.16.0.11"
|
||||
ipv6:
|
||||
- subnet: "2001:db8:abcd:1234::/64"
|
||||
gateway: "2001:db8:abcd:1234::1"
|
||||
reserved_addrs:
|
||||
- "2001:db8:abcd:1234::dead"
|
||||
- "2001:db8:abcd:1234::beef"
|
||||
volumes:
|
||||
- path: "/etc/detee/daemon/vms/"
|
||||
max_reservation_gb: 500
|
||||
public_port_range:
|
||||
start: 10000
|
||||
end: 11000
|
||||
max_ports_per_vm: 20
|
||||
|
@ -1,20 +0,0 @@
|
||||
max_cores_per_vm: 12
|
||||
max_vcpu_reservation: 24
|
||||
max_mem_reservation_mb: 49152
|
||||
network_interfaces:
|
||||
- driver: "IPVTAP"
|
||||
device: "tap0"
|
||||
ipv4: []
|
||||
ipv6:
|
||||
- subnet: "2001:db8:abcd:1234::/64"
|
||||
gateway: "2001:db8:abcd:1234::1"
|
||||
reserved_addrs:
|
||||
- "2001:db8:abcd:1234::dead"
|
||||
- "2001:db8:abcd:1234::beef"
|
||||
volumes:
|
||||
- path: "/ipv6/volume"
|
||||
max_reservation_gb: 600
|
||||
public_port_range:
|
||||
start: 15000
|
||||
end: 16000
|
||||
max_ports_per_vm: 10
|
@ -1,19 +0,0 @@
|
||||
max_cores_per_vm: 2
|
||||
max_vcpu_reservation: 4
|
||||
max_mem_reservation_mb: 8192
|
||||
network_interfaces:
|
||||
- driver: "MACVTAP"
|
||||
device: "eth0"
|
||||
ipv4:
|
||||
- subnet: "192.168.0.0/24"
|
||||
gateway: "192.168.0.1"
|
||||
reserved_addrs: []
|
||||
ipv6: []
|
||||
volumes:
|
||||
- path: "/minimal/volume"
|
||||
max_reservation_gb: 100
|
||||
public_port_range:
|
||||
start: 5000
|
||||
end: 5100
|
||||
max_ports_per_vm: 3
|
||||
|
@ -1,22 +0,0 @@
|
||||
max_cores_per_vm: 8
|
||||
max_vcpu_reservation: 16
|
||||
max_mem_reservation_mb: 32768
|
||||
network_interfaces:
|
||||
- driver: "Bridge"
|
||||
device: "br1"
|
||||
ipv4:
|
||||
- subnet: "192.168.100.0/24"
|
||||
gateway: "192.168.100.1"
|
||||
reserved_addrs: []
|
||||
ipv6:
|
||||
- subnet: "2001:abcd::/48"
|
||||
gateway: "2001:abcd::1"
|
||||
reserved_addrs: []
|
||||
volumes:
|
||||
- path: "/network/volume"
|
||||
max_reservation_gb: 750
|
||||
public_port_range:
|
||||
start: 6000
|
||||
end: 7000
|
||||
max_ports_per_vm: 8
|
||||
|
@ -1,13 +0,0 @@
|
||||
uuid: "123e4567-e89b-12d3-a456-426614174000"
|
||||
hostname: "test-vm-01"
|
||||
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEArandomkeyexample"
|
||||
extra_ports: [ ]
|
||||
public_ipv4: true
|
||||
public_ipv6: false
|
||||
disk_size_gb: 50
|
||||
vcpus: 4
|
||||
memory_mb: 8192
|
||||
kernel_url: "http://pb1n.de/?d25eec"
|
||||
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
|
||||
dtrfs_url: "http://pb1n.de/?e46db9"
|
||||
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"
|
@ -1,13 +0,0 @@
|
||||
uuid: "987e6543-e21b-43d3-c321-654987210000"
|
||||
hostname: "minimal-vm"
|
||||
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQAnotherExampleKey"
|
||||
extra_ports: []
|
||||
public_ipv4: false
|
||||
public_ipv6: false
|
||||
disk_size_gb: 10
|
||||
vcpus: 1
|
||||
memory_mb: 2048
|
||||
kernel_url: "http://pb1n.de/?d25eec"
|
||||
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
|
||||
dtrfs_url: "http://pb1n.de/?e46db9"
|
||||
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"
|
@ -1,14 +0,0 @@
|
||||
uuid: "246e1357-e98b-76d3-f345-129874650000"
|
||||
hostname: "extensive-vm"
|
||||
admin_key: "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEExtendedKeyExample"
|
||||
extra_ports: []
|
||||
public_ipv4: true
|
||||
public_ipv6: true
|
||||
disk_size_gb: 35
|
||||
vcpus: 2
|
||||
memory_mb: 5000
|
||||
kernel_url: "http://pb1n.de/?d25eec"
|
||||
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
|
||||
dtrfs_url: "http://pb1n.de/?e46db9"
|
||||
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"
|
||||
|
@ -1,13 +0,0 @@
|
||||
uuid: "DuTenPulaMeaCuUUIDulTauCuTot"
|
||||
hostname: "testing-vm"
|
||||
admin_key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAAKeyExampleForTesting"
|
||||
extra_ports: [1234, 5678]
|
||||
public_ipv4: false
|
||||
public_ipv6: true
|
||||
disk_size_gb: 25
|
||||
vcpus: 2
|
||||
memory_mb: 4096
|
||||
kernel_url: "http://pb1n.de/?d25eec"
|
||||
kernel_sha: "be29dfef7157bfe860e94e96dcfab2318c5006e92e8d846a3ad7aa804b3b994e"
|
||||
dtrfs_url: "http://pb1n.de/?e46db9"
|
||||
dtrfs_sha: "62e7362c9350d60698cae6eed302562a2b41bec1d248889baad302da19c3bb47"
|
272
vm.proto
Normal file
272
vm.proto
Normal file
@ -0,0 +1,272 @@
|
||||
syntax = "proto3";
|
||||
package vm_proto;
|
||||
|
||||
message Empty {
|
||||
}
|
||||
|
||||
message Pubkey {
|
||||
string pubkey = 1;
|
||||
}
|
||||
|
||||
message AccountBalance {
|
||||
uint64 balance = 1;
|
||||
uint64 tmp_locked = 2;
|
||||
}
|
||||
|
||||
message VmContract {
|
||||
string uuid = 1;
|
||||
string hostname = 2;
|
||||
string admin_pubkey = 3;
|
||||
string node_pubkey = 4;
|
||||
repeated uint32 exposed_ports = 5;
|
||||
string public_ipv4 = 6;
|
||||
string public_ipv6 = 7;
|
||||
uint32 disk_size_gb = 8;
|
||||
uint32 vcpus = 9;
|
||||
uint32 memory_mb = 10;
|
||||
string kernel_sha = 11;
|
||||
string dtrfs_sha = 12;
|
||||
string created_at = 13;
|
||||
string updated_at = 14;
|
||||
// total nanoLP cost per minute (for all units)
|
||||
uint64 nano_per_minute = 15;
|
||||
uint64 locked_nano = 16;
|
||||
string collected_at = 17;
|
||||
}
|
||||
|
||||
message MeasurementArgs {
|
||||
// this will be IP:Port of the dtrfs API
|
||||
// actually not a measurement arg, but needed for the injector
|
||||
string dtrfs_api_endpoint = 1;
|
||||
repeated uint32 exposed_ports = 2;
|
||||
string ovmf_hash = 5;
|
||||
// This is needed to allow the CLI to build the kernel params from known data.
|
||||
// The CLI will use the kernel params to get the measurement.
|
||||
repeated MeasurementIP ips = 6;
|
||||
}
|
||||
|
||||
message MeasurementIP {
|
||||
uint32 nic_index = 1;
|
||||
string address = 2;
|
||||
string mask = 3;
|
||||
string gateway = 4;
|
||||
}
|
||||
|
||||
// This should also include a block hash or similar, for auth
|
||||
message RegisterVmNodeReq {
|
||||
string node_pubkey = 1;
|
||||
string operator_wallet = 2;
|
||||
string main_ip = 3;
|
||||
string country = 4;
|
||||
string region = 5;
|
||||
string city = 6;
|
||||
// nanoLP per unit per minute
|
||||
uint64 price = 7;
|
||||
}
|
||||
|
||||
message VmNodeResources {
|
||||
string node_pubkey = 1;
|
||||
uint32 avail_ports = 2;
|
||||
uint32 avail_ipv4 = 3;
|
||||
uint32 avail_ipv6 = 4;
|
||||
uint32 avail_vcpus = 5;
|
||||
uint32 avail_memory_mb = 6;
|
||||
uint32 avail_storage_gb = 7;
|
||||
uint32 max_ports_per_vm = 8;
|
||||
}
|
||||
|
||||
message NewVmReq {
|
||||
string uuid = 1;
|
||||
string hostname = 2;
|
||||
string admin_pubkey = 3;
|
||||
string node_pubkey = 4;
|
||||
repeated uint32 extra_ports = 5;
|
||||
bool public_ipv4 = 6;
|
||||
bool public_ipv6 = 7;
|
||||
uint32 disk_size_gb = 8;
|
||||
uint32 vcpus = 9;
|
||||
uint32 memory_mb = 10;
|
||||
string kernel_url = 11;
|
||||
string kernel_sha = 12;
|
||||
string dtrfs_url = 13;
|
||||
string dtrfs_sha = 14;
|
||||
uint64 price_per_unit = 15;
|
||||
uint64 locked_nano = 16;
|
||||
}
|
||||
|
||||
message NewVmResp {
|
||||
string uuid = 1;
|
||||
string error = 2;
|
||||
MeasurementArgs args = 3;
|
||||
}
|
||||
|
||||
message UpdateVmReq {
|
||||
string uuid = 1;
|
||||
string admin_pubkey = 2;
|
||||
uint32 disk_size_gb = 3;
|
||||
uint32 vcpus = 4;
|
||||
uint32 memory_mb = 5;
|
||||
string kernel_url = 6;
|
||||
string kernel_sha = 7;
|
||||
string dtrfs_url = 8;
|
||||
string dtrfs_sha = 9;
|
||||
}
|
||||
|
||||
message UpdateVmResp {
|
||||
string uuid = 1;
|
||||
string error = 2;
|
||||
MeasurementArgs args = 3;
|
||||
}
|
||||
|
||||
message DeleteVmReq {
|
||||
string uuid = 1;
|
||||
string admin_pubkey = 2;
|
||||
}
|
||||
|
||||
message BrainVmMessage {
|
||||
oneof Msg {
|
||||
NewVmReq new_vm_req = 1;
|
||||
UpdateVmReq update_vm_req = 2;
|
||||
DeleteVmReq delete_vm = 3;
|
||||
}
|
||||
}
|
||||
|
||||
message DaemonStreamAuth {
|
||||
string timestamp = 1;
|
||||
string pubkey = 2;
|
||||
repeated string contracts = 3;
|
||||
string signature = 4;
|
||||
}
|
||||
|
||||
message VmDaemonMessage {
|
||||
oneof Msg {
|
||||
DaemonStreamAuth auth = 1;
|
||||
NewVmResp new_vm_resp = 2;
|
||||
UpdateVmResp update_vm_resp = 3;
|
||||
VmNodeResources vm_node_resources = 4;
|
||||
}
|
||||
}
|
||||
|
||||
service BrainVmDaemon {
|
||||
rpc RegisterVmNode (RegisterVmNodeReq) returns (stream VmContract);
|
||||
rpc BrainMessages (DaemonStreamAuth) returns (stream BrainVmMessage);
|
||||
rpc DaemonMessages (stream VmDaemonMessage) returns (Empty);
|
||||
}
|
||||
|
||||
message ListVmContractsReq {
|
||||
string admin_pubkey = 1;
|
||||
string node_pubkey = 2;
|
||||
string uuid = 3;
|
||||
}
|
||||
|
||||
message VmNodeFilters {
|
||||
uint32 free_ports = 1;
|
||||
bool offers_ipv4 = 2;
|
||||
bool offers_ipv6 = 3;
|
||||
uint32 vcpus = 4;
|
||||
uint32 memory_mb = 5;
|
||||
uint32 storage_gb = 6;
|
||||
string country = 7;
|
||||
string region = 8;
|
||||
string city = 9;
|
||||
string ip = 10;
|
||||
string node_pubkey = 11;
|
||||
}
|
||||
|
||||
message VmNodeListResp {
|
||||
string operator = 1;
|
||||
string node_pubkey = 2;
|
||||
string country = 3;
|
||||
string region = 4;
|
||||
string city = 5;
|
||||
string ip = 6; // required for latency test
|
||||
repeated string reports = 7; // TODO: this will become an enum
|
||||
uint64 price = 8; // nanoLP per unit per minute
|
||||
}
|
||||
|
||||
message ExtendVmReq {
|
||||
string uuid = 1;
|
||||
string admin_pubkey = 2;
|
||||
uint64 locked_nano = 3;
|
||||
}
|
||||
|
||||
message AirdropReq {
|
||||
string pubkey = 1;
|
||||
uint64 tokens = 2;
|
||||
}
|
||||
|
||||
message SlashReq {
|
||||
string pubkey = 1;
|
||||
uint64 tokens = 2;
|
||||
}
|
||||
|
||||
message Account {
|
||||
string pubkey = 1;
|
||||
uint64 balance = 2;
|
||||
uint64 tmp_locked = 3;
|
||||
}
|
||||
|
||||
message RegOperatorReq {
|
||||
string pubkey = 1;
|
||||
uint64 escrow = 2;
|
||||
string email = 3;
|
||||
}
|
||||
|
||||
message ListOperatorsResp {
|
||||
string pubkey = 1;
|
||||
uint64 escrow = 2;
|
||||
string email = 3;
|
||||
uint64 app_nodes = 4;
|
||||
uint64 vm_nodes = 5;
|
||||
uint64 reports = 6;
|
||||
}
|
||||
|
||||
message InspectOperatorResp {
|
||||
ListOperatorsResp operator = 1;
|
||||
repeated VmNodeListResp nodes = 2;
|
||||
}
|
||||
|
||||
message ReportNodeReq {
|
||||
string admin_pubkey = 1;
|
||||
string node_pubkey = 2;
|
||||
string contract = 3;
|
||||
string reason = 4;
|
||||
}
|
||||
|
||||
message KickReq {
|
||||
string operator_wallet = 1;
|
||||
string contract_uuid = 2;
|
||||
string reason = 3;
|
||||
}
|
||||
|
||||
message BanUserReq {
|
||||
string operator_wallet = 1;
|
||||
string user_wallet = 2;
|
||||
}
|
||||
|
||||
message KickResp {
|
||||
uint64 nano_lp = 1;
|
||||
}
|
||||
|
||||
|
||||
service BrainCli {
|
||||
rpc GetBalance (Pubkey) returns (AccountBalance);
|
||||
rpc NewVm (NewVmReq) returns (NewVmResp);
|
||||
rpc ListVmContracts (ListVmContractsReq) returns (stream VmContract);
|
||||
rpc ListVmNodes (VmNodeFilters) returns (stream VmNodeListResp);
|
||||
rpc GetOneVmNode (VmNodeFilters) returns (VmNodeListResp);
|
||||
rpc DeleteVm (DeleteVmReq) returns (Empty);
|
||||
rpc UpdateVm (UpdateVmReq) returns (UpdateVmResp);
|
||||
rpc ExtendVm (ExtendVmReq) returns (Empty);
|
||||
rpc ReportNode (ReportNodeReq) returns (Empty);
|
||||
rpc ListOperators (Empty) returns (stream ListOperatorsResp);
|
||||
rpc InspectOperator (Pubkey) returns (InspectOperatorResp);
|
||||
rpc RegisterOperator (RegOperatorReq) returns (Empty);
|
||||
rpc KickContract (KickReq) returns (KickResp);
|
||||
rpc BanUser (BanUserReq) returns (Empty);
|
||||
// admin commands
|
||||
rpc Airdrop (AirdropReq) returns (Empty);
|
||||
rpc Slash (SlashReq) returns (Empty);
|
||||
rpc ListAllVmContracts (Empty) returns (stream VmContract);
|
||||
rpc ListAccounts (Empty) returns (stream Account);
|
||||
}
|
Loading…
Reference in New Issue
Block a user