diff --git a/Cargo.lock b/Cargo.lock index 87e6b57..e138dc7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1011,7 +1011,7 @@ dependencies = [ [[package]] name = "detee-shared" version = "0.1.0" -source = "git+ssh://git@gitea.detee.cloud/testnet/proto?branch=credits_app#01e93d3a2e4502c0e8e72026e8a1c55810961815" +source = "git+ssh://git@gitea.detee.cloud/testnet/proto?branch=offers#4753a17fa29393b3f99b6dfcdcec48d935e6ebd9" dependencies = [ "bincode 2.0.1", "prost", diff --git a/Cargo.toml b/Cargo.toml index c117b51..cc29262 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,7 +15,7 @@ serde_yaml = "0.9.34" surrealdb = "2.2.2" tokio = { version = "1.44.2", features = ["macros", "rt-multi-thread"] } tonic = { version = "0.12", features = ["tls"] } -detee-shared = { git = "ssh://git@gitea.detee.cloud/testnet/proto", branch = "credits_app" } +detee-shared = { git = "ssh://git@gitea.detee.cloud/testnet/proto", branch = "offers" } ed25519-dalek = "2.1.1" bs58 = "0.5.1" tokio-stream = "0.1.17" diff --git a/src/bin/migration0.rs b/src/bin/migration1.rs similarity index 89% rename from src/bin/migration0.rs rename to src/bin/migration1.rs index f3d8a2f..19209f8 100644 --- a/src/bin/migration0.rs +++ b/src/bin/migration1.rs @@ -9,7 +9,7 @@ use surreal_brain::{db, old_brain}; #[tokio::main] async fn main() -> Result<(), Box> { dotenv().ok(); - let old_brain_data = old_brain::BrainData::load_from_disk()?; + // let old_brain_data = old_brain::BrainData::load_from_disk()?; // println!("{}", serde_yaml::to_string(&old_brain_data)?); let db_url = std::env::var("DB_URL").expect("DB_URL not set in .env"); @@ -21,7 +21,7 @@ async fn main() -> Result<(), Box> { let db = db::db_connection(&db_url, &db_user, &db_pass, &db_ns, &db_name).await.unwrap(); env_logger::builder().filter_level(log::LevelFilter::Trace); - db::migration0(&db, &old_brain_data).await?; + db::migration1(&db).await?; Ok(()) } diff --git a/src/constants.rs b/src/constants.rs index e41996d..aff54ea 100644 --- a/src/constants.rs +++ b/src/constants.rs @@ -8,7 +8,9 @@ pub const CERT_PATH: &str = "/etc/detee/brain/brain-crt.pem"; pub const CERT_KEY_PATH: &str = "/etc/detee/brain/brain-key.pem"; pub const CONFIG_PATH: &str = "/etc/detee/brain/config.ini"; -pub const DB_SCHEMA_FILES: [&str; 2] = ["surql/tables.sql", "surql/functions.sql"]; +pub const MIGRATION_0_SCHEMA: [&str; 2] = ["surql/migration0/tables.sql", "surql/migration0/functions.sql"]; +pub const MIGRATION_1_PATCH: &str = "surql/migration1/patch_tables.sql"; +pub const MIGRATION_1_SCHEMA: &str = "surql/migration1/final_tables.sql"; pub static ADMIN_ACCOUNTS: LazyLock> = LazyLock::new(|| { let default_admin_keys = vec![ diff --git a/src/db/mod.rs b/src/db/mod.rs index b6a3f42..71bab47 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -5,8 +5,8 @@ pub mod general; pub mod vm; use crate::constants::{ - APP_NODE, DB_SCHEMA_FILES, DEFAULT_ENDPOINT, DELETED_APP, DELETED_VM, NEW_APP_REQ, NEW_VM_REQ, - UPDATE_VM_REQ, + APP_NODE, DEFAULT_ENDPOINT, DELETED_APP, DELETED_VM, MIGRATION_0_SCHEMA, MIGRATION_1_PATCH, + MIGRATION_1_SCHEMA, NEW_APP_REQ, NEW_VM_REQ, UPDATE_VM_REQ, }; use crate::old_brain; use prelude::*; @@ -77,34 +77,18 @@ pub async fn db_connection( Ok(db_connection) } -pub async fn migration0( - db: &Surreal, - old_data: &old_brain::BrainData, -) -> Result<(), Error> { - let accounts: Vec = old_data.into(); - let vm_nodes: Vec = old_data.into(); - let app_nodes: Vec = old_data.into(); - let active_vm: Vec = old_data.into(); - let active_app: Vec = old_data.into(); +// TODO: figure if we can inspect DB schema in the DB with the schema from migration0, +// and execute migration1_patch only if it is needed. - for schema_data in DB_SCHEMA_FILES.map(|path| (std::fs::read_to_string(path), path)) { - let schema_file = schema_data.1; - println!("Loading schema from {schema_file}"); - let schema = schema_data.0?; - db.query(schema).await?; - } - - println!("Inserting accounts..."); - let _: Vec = db.insert(()).content(accounts).await?; - println!("Inserting vm nodes..."); - let _: Vec = db.insert(()).content(vm_nodes).await?; - println!("Inserting app nodes..."); - let _: Vec = db.insert(()).content(app_nodes).await.unwrap(); - println!("Inserting active vm contracts..."); - let _: Vec = db.insert(()).relation(active_vm).await?; - println!("Inserting app contracts..."); - let _: Vec = db.insert(()).relation(active_app).await?; +pub async fn migration1_patch(db: &Surreal) -> Result<(), Error> { + let patch_instructions = std::fs::read_to_string(MIGRATION_1_PATCH)?; + db.query(patch_instructions).await?; + Ok(()) +} +pub async fn migration1_tables(db: &Surreal) -> Result<(), Error> { + let patch_instructions = std::fs::read_to_string(MIGRATION_1_SCHEMA)?; + db.query(patch_instructions).await?; Ok(()) } diff --git a/src/old_brain.rs b/src/old_brain.rs deleted file mode 100644 index d4dc2ee..0000000 --- a/src/old_brain.rs +++ /dev/null @@ -1,135 +0,0 @@ -// SPDX-License-Identifier: Apache-2.0 - -// TODO: delete this file after migration0 gets executed - -use chrono::Utc; -use dashmap::DashMap; -use serde::{Deserialize, Serialize}; -use std::collections::{HashMap, HashSet}; - -use crate::constants::OLD_BRAIN_DATA_PATH; - -#[derive(Clone, Default, Serialize, Deserialize, Debug)] -pub struct AccountData { - pub balance: u64, - pub tmp_locked: u64, - // holds reasons why VMs of this account got kicked - pub kicked_for: Vec, - pub last_kick: chrono::DateTime, - // holds accounts that banned this account - pub banned_by: HashSet, -} - -#[derive(Clone, Default, Serialize, Deserialize)] -pub struct OperatorData { - pub escrow: u64, - pub email: String, - pub banned_users: HashSet, - pub vm_nodes: HashSet, - pub app_nodes: HashSet, -} - -#[derive(Eq, PartialEq, Clone, Debug, Default, Serialize, Deserialize)] -pub struct VmNode { - pub public_key: String, - pub operator_wallet: String, - pub country: String, - pub region: String, - pub city: String, - pub ip: String, - pub avail_mem_mb: u32, - pub avail_vcpus: u32, - pub avail_storage_gbs: u32, - pub avail_ipv4: u32, - pub avail_ipv6: u32, - pub avail_ports: u32, - pub max_ports_per_vm: u32, - // nanoLP per unit per minute - pub price: u64, - // 1st String is user wallet and 2nd String is report message - pub reports: HashMap, - pub offline_minutes: u64, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct VmContract { - pub uuid: String, - pub hostname: String, - pub admin_pubkey: String, - pub node_pubkey: String, - pub exposed_ports: Vec, - pub public_ipv4: String, - pub public_ipv6: String, - pub disk_size_gb: u32, - pub vcpus: u32, - pub memory_mb: u32, - pub kernel_sha: String, - pub dtrfs_sha: String, - pub created_at: chrono::DateTime, - pub updated_at: chrono::DateTime, - // recommended value is 20000 - /// price per unit per minute - pub price_per_unit: u64, - pub locked_nano: u64, - pub collected_at: chrono::DateTime, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct AppContract { - pub uuid: String, - pub package_url: String, - pub admin_pubkey: String, - pub node_pubkey: String, - pub mapped_ports: Vec<(u16, u16)>, - pub host_ipv4: String, - pub disk_size_mb: u32, - pub vcpus: u32, - pub memory_mb: u32, - pub created_at: chrono::DateTime, - pub updated_at: chrono::DateTime, - // price per unit per minute - // recommended value is 20000 - pub price_per_unit: u64, - pub locked_nano: u64, - pub collected_at: chrono::DateTime, - pub hratls_pubkey: String, - pub public_package_mr_enclave: Option>, - pub app_name: String, -} - -#[derive(Eq, Hash, PartialEq, Clone, Debug, Default, Serialize, Deserialize)] -pub struct AppNode { - pub node_pubkey: String, - pub operator_wallet: String, - pub country: String, - pub region: String, - pub city: String, - pub ip: String, - pub avail_mem_mb: u32, - pub avail_vcpus: u32, - pub avail_storage_mb: u32, - pub avail_no_of_port: u32, - pub max_ports_per_app: u32, - // nanotokens per unit per minute - pub price: u64, - pub offline_minutes: u64, -} - -#[derive(Default, Serialize, Deserialize)] -pub struct BrainData { - pub accounts: DashMap, - pub operators: DashMap, - pub vm_nodes: Vec, - pub vm_contracts: Vec, - - pub app_nodes: Vec, - pub app_contracts: Vec, -} - -impl BrainData { - pub fn load_from_disk() -> Result> { - let content = std::fs::read_to_string(OLD_BRAIN_DATA_PATH)?; - let data: Self = serde_yaml::from_str(&content)?; - Ok(data) - } -} diff --git a/surql/functions.sql b/surql/migration0/functions.sql similarity index 100% rename from surql/functions.sql rename to surql/migration0/functions.sql diff --git a/surql/tables.sql b/surql/migration0/tables.sql similarity index 100% rename from surql/tables.sql rename to surql/migration0/tables.sql diff --git a/surql/migration1/final_tables.sql b/surql/migration1/final_tables.sql new file mode 100644 index 0000000..0e9559a --- /dev/null +++ b/surql/migration1/final_tables.sql @@ -0,0 +1,158 @@ +-- SPDX-License-Identifier: Apache-2.0 + +DEFINE TABLE account SCHEMAFULL; +DEFINE FIELD balance ON TABLE account TYPE int DEFAULT 0; +DEFINE FIELD tmp_locked ON TABLE account TYPE int DEFAULT 0; +DEFINE FIELD escrow ON TABLE account TYPE int DEFAULT 0; +DEFINE FIELD email ON TABLE account TYPE string DEFAULT ""; + +DEFINE TABLE vm_node SCHEMAFULL; +DEFINE FIELD operator ON TABLE vm_node TYPE record; +DEFINE FIELD pub_sub_node ON TABLE vm_node TYPE string default "127.0.0.1:31337"; +DEFINE FIELD country ON TABLE vm_node TYPE string; +DEFINE FIELD region ON TABLE vm_node TYPE string; +DEFINE FIELD city ON TABLE vm_node TYPE string; +DEFINE FIELD ip ON TABLE vm_node TYPE string; +DEFINE FIELD avail_ipv4 ON TABLE vm_node TYPE int; +DEFINE FIELD avail_ipv6 ON TABLE vm_node TYPE int; +DEFINE FIELD avail_ports ON TABLE vm_node TYPE int; +DEFINE FIELD max_ports_per_vm ON TABLE vm_node TYPE int; +DEFINE FIELD connected_at ON TABLE vm_node TYPE datetime; +DEFINE FIELD disconnected_at ON TABLE vm_node TYPE datetime; + +DEFINE TABLE vm_node_offer SCHEMAFULL; +DEFINE FIELD vm_node ON TABLE vm_node_offer TYPE record; +DEFINE FIELD price ON TABLE vm_node_offer TYPE int; +DEFINE FIELD vcpus ON TABLE vm_node_offer TYPE int; +DEFINE FIELD memory_mib ON TABLE vm_node_offer TYPE int; +DEFINE FIELD disk_mib ON TABLE vm_node_offer TYPE int; + +DEFINE TABLE new_vm_req TYPE RELATION FROM account TO vm_node SCHEMAFULL; +DEFINE FIELD hostname ON TABLE new_vm_req TYPE string; +DEFINE FIELD extra_ports ON TABLE new_vm_req TYPE array; +DEFINE FIELD public_ipv4 ON TABLE new_vm_req TYPE bool; +DEFINE FIELD public_ipv6 ON TABLE new_vm_req TYPE bool; +DEFINE FIELD disk_size_mib ON TABLE new_vm_req TYPE int; +DEFINE FIELD vcpus ON TABLE new_vm_req TYPE int; +DEFINE FIELD memory_mib ON TABLE new_vm_req TYPE int; +DEFINE FIELD dtrfs_sha ON TABLE new_vm_req TYPE string; +DEFINE FIELD dtrfs_url ON TABLE new_vm_req TYPE string; +DEFINE FIELD kernel_sha ON TABLE new_vm_req TYPE string; +DEFINE FIELD kernel_url ON TABLE new_vm_req TYPE string; +DEFINE FIELD created_at ON TABLE new_vm_req TYPE datetime; +DEFINE FIELD price_per_unit ON TABLE new_vm_req TYPE int; +DEFINE FIELD locked_nano ON TABLE new_vm_req TYPE int; +DEFINE FIELD error ON TABLE new_vm_req TYPE string; + +DEFINE TABLE active_vm TYPE RELATION FROM account TO vm_node SCHEMAFULL; +DEFINE FIELD hostname ON TABLE active_vm TYPE string; +DEFINE FIELD mapped_ports ON TABLE active_vm TYPE array<[int, int]>; +DEFINE FIELD public_ipv4 ON TABLE active_vm TYPE string; +DEFINE FIELD public_ipv6 ON TABLE active_vm TYPE string; +DEFINE FIELD disk_size_mib ON TABLE active_vm TYPE int; +DEFINE FIELD vcpus ON TABLE active_vm TYPE int; +DEFINE FIELD memory_mib ON TABLE active_vm TYPE int; +DEFINE FIELD dtrfs_sha ON TABLE active_vm TYPE string; +DEFINE FIELD kernel_sha ON TABLE active_vm TYPE string; +DEFINE FIELD created_at ON TABLE active_vm TYPE datetime; +DEFINE FIELD price_per_unit ON TABLE active_vm TYPE int; +DEFINE FIELD locked_nano ON TABLE active_vm TYPE int; +DEFINE FIELD collected_at ON TABLE active_vm TYPE datetime; + +DEFINE TABLE update_vm_req TYPE RELATION FROM account TO vm_node SCHEMAFULL; +DEFINE FIELD vcpus ON TABLE update_vm_req TYPE int; +DEFINE FIELD memory_mib ON TABLE update_vm_req TYPE int; +DEFINE FIELD disk_size_mib ON TABLE update_vm_req TYPE int; +DEFINE FIELD dtrfs_sha ON TABLE update_vm_req TYPE string; +DEFINE FIELD dtrfs_url ON TABLE update_vm_req TYPE string; +DEFINE FIELD kernel_sha ON TABLE update_vm_req TYPE string; +DEFINE FIELD kernel_url ON TABLE update_vm_req TYPE string; +DEFINE FIELD created_at ON TABLE update_vm_req TYPE datetime; +DEFINE FIELD error ON TABLE update_vm_req TYPE string; + +DEFINE TABLE deleted_vm TYPE RELATION FROM account TO vm_node SCHEMAFULL; +DEFINE FIELD hostname ON TABLE deleted_vm TYPE string; +DEFINE FIELD mapped_ports ON TABLE deleted_vm TYPE array<[int, int]>; +DEFINE FIELD public_ipv4 ON TABLE deleted_vm TYPE string; +DEFINE FIELD public_ipv6 ON TABLE deleted_vm TYPE string; +DEFINE FIELD disk_size_mib ON TABLE deleted_vm TYPE int; +DEFINE FIELD vcpus ON TABLE deleted_vm TYPE int; +DEFINE FIELD memory_mib ON TABLE deleted_vm TYPE int; +DEFINE FIELD dtrfs_sha ON TABLE deleted_vm TYPE string; +DEFINE FIELD kernel_sha ON TABLE deleted_vm TYPE string; +DEFINE FIELD created_at ON TABLE deleted_vm TYPE datetime; +DEFINE FIELD deleted_at ON TABLE deleted_vm TYPE datetime DEFAULT time::now(); +DEFINE FIELD price_per_unit ON TABLE deleted_vm TYPE int; + +DEFINE TABLE app_node SCHEMAFULL; +DEFINE FIELD operator ON TABLE app_node TYPE record; +DEFINE FIELD pub_sub_node ON TABLE app_node TYPE string default "127.0.0.1:31337"; +DEFINE FIELD country ON TABLE app_node TYPE string; +DEFINE FIELD region ON TABLE app_node TYPE string; +DEFINE FIELD city ON TABLE app_node TYPE string; +DEFINE FIELD ip ON TABLE app_node TYPE string; +DEFINE FIELD avail_mem_mib ON TABLE app_node TYPE int; +DEFINE FIELD avail_vcpus ON TABLE app_node TYPE int; +DEFINE FIELD avail_storage_mib ON TABLE app_node TYPE int; +DEFINE FIELD avail_ports ON TABLE app_node TYPE int; +DEFINE FIELD max_ports_per_app ON TABLE app_node TYPE int; +DEFINE FIELD price ON TABLE app_node TYPE int; +DEFINE FIELD connected_at ON TABLE app_node TYPE datetime; +DEFINE FIELD disconnected_at ON TABLE app_node TYPE datetime; + +DEFINE TABLE new_app_req Type RELATION FROM account to app_node SCHEMAFULL; +DEFINE FIELD app_name ON TABLE new_app_req TYPE string; +DEFINE FIELD package_url ON TABLE new_app_req TYPE string; +DEFINE FIELD mr_enclave ON TABLE new_app_req TYPE string; +DEFINE FIELD hratls_pubkey ON TABLE new_app_req TYPE string; +DEFINE FIELD ports ON TABLE new_app_req TYPE array; +DEFINE FIELD memory_mib ON TABLE new_app_req TYPE int; +DEFINE FIELD vcpus ON TABLE new_app_req TYPE int; +DEFINE FIELD disk_size_mib ON TABLE new_app_req TYPE int; +DEFINE FIELD locked_nano ON TABLE new_app_req TYPE int; +DEFINE FIELD price_per_unit ON TABLE new_app_req TYPE int; +DEFINE FIELD error ON TABLE new_app_req TYPE string; +DEFINE FIELD created_at ON TABLE new_app_req TYPE datetime; + +DEFINE TABLE active_app TYPE RELATION FROM account TO app_node SCHEMAFULL; +DEFINE FIELD app_name ON TABLE active_app TYPE string; +DEFINE FIELD mapped_ports ON TABLE active_app TYPE array<[int, int]>; +DEFINE FIELD host_ipv4 ON TABLE active_app TYPE string; +DEFINE FIELD vcpus ON TABLE active_app TYPE int; +DEFINE FIELD memory_mib ON TABLE active_app TYPE int; +DEFINE FIELD disk_size_mib ON TABLE active_app TYPE int; +DEFINE FIELD created_at ON TABLE active_app TYPE datetime; +DEFINE FIELD price_per_unit ON TABLE active_app TYPE int; +DEFINE FIELD locked_nano ON TABLE active_app TYPE int; +DEFINE FIELD collected_at ON TABLE active_app TYPE datetime; +DEFINE FIELD mr_enclave ON TABLE active_app TYPE string; +DEFINE FIELD package_url ON TABLE active_app TYPE string; +DEFINE FIELD hratls_pubkey ON TABLE active_app TYPE string; + +DEFINE TABLE deleted_app TYPE RELATION FROM account TO app_node SCHEMAFULL; +DEFINE FIELD app_name ON TABLE deleted_app TYPE string; +DEFINE FIELD mapped_ports ON TABLE deleted_app TYPE array<[int, int]>; +DEFINE FIELD host_ipv4 ON TABLE deleted_app TYPE string; +DEFINE FIELD vcpus ON TABLE deleted_app TYPE int; +DEFINE FIELD memory_mib ON TABLE deleted_app TYPE int; +DEFINE FIELD disk_size_mib ON TABLE deleted_app TYPE int; +DEFINE FIELD created_at ON TABLE deleted_app TYPE datetime; +DEFINE FIELD deleted_at ON TABLE deleted_app TYPE datetime DEFAULT time::now(); +DEFINE FIELD price_per_unit ON TABLE deleted_app TYPE int; +DEFINE FIELD mr_enclave ON TABLE deleted_app TYPE string; +DEFINE FIELD package_url ON TABLE deleted_app TYPE string; +DEFINE FIELD hratls_pubkey ON TABLE deleted_app TYPE string; + +DEFINE TABLE ban TYPE RELATION FROM account TO account; +DEFINE FIELD created_at ON TABLE ban TYPE datetime DEFAULT time::now();; + +DEFINE TABLE kick TYPE RELATION FROM account TO account; +DEFINE FIELD created_at ON TABLE kick TYPE datetime; +DEFINE FIELD reason ON TABLE kick TYPE string; +DEFINE FIELD contract ON TABLE kick TYPE record; +DEFINE FIELD node ON TABLE kick TYPE record; + +DEFINE TABLE report TYPE RELATION FROM account TO vm_node|app_node; +DEFINE FIELD created_at ON TABLE report TYPE datetime; +DEFINE FIELD reason ON TABLE report TYPE string; +DEFINE FIELD contract_id ON TABLE report TYPE string; diff --git a/surql/migration1/patch_tables.sql b/surql/migration1/patch_tables.sql new file mode 100644 index 0000000..5d26e06 --- /dev/null +++ b/surql/migration1/patch_tables.sql @@ -0,0 +1,13 @@ +-- SPDX-License-Identifier: Apache-2.0 + +REMOVE FIELD price ON TABLE vm_node; +REMOVE FIELD avail_vcpus ON TABLE vm_node; +REMOVE FIELD avail_storage_mib ON TABLE vm_node; +REMOVE FIELD avail_mem_mib ON TABLE vm_node; + +DEFINE TABLE vm_node_offer SCHEMAFULL; +DEFINE FIELD vm_node ON TABLE vm_node_offer TYPE record; +DEFINE FIELD price ON TABLE vm_node_offer TYPE int; +DEFINE FIELD vcpus ON TABLE vm_node_offer TYPE int; +DEFINE FIELD memory_mib ON TABLE vm_node_offer TYPE int; +DEFINE FIELD disk_mib ON TABLE vm_node_offer TYPE int; diff --git a/tests/common/prepare_test_env.rs b/tests/common/prepare_test_env.rs index af3a42c..fa6b260 100644 --- a/tests/common/prepare_test_env.rs +++ b/tests/common/prepare_test_env.rs @@ -9,7 +9,7 @@ use detee_shared::vm_proto::brain_vm_daemon_server::BrainVmDaemonServer; use dotenv::dotenv; use hyper_util::rt::TokioIo; use std::sync::Arc; -use surreal_brain::constants::DB_SCHEMA_FILES; +use surreal_brain::constants::MIGRATION_0_SCHEMA; use surreal_brain::grpc::app::{AppCliServer, AppDaemonServer}; use surreal_brain::grpc::general::GeneralCliServer; use surreal_brain::grpc::vm::{VmCliServer, VmDaemonServer}; @@ -39,7 +39,7 @@ pub async fn prepare_test_db() -> Result> { serde_yaml::from_str(&raw_mock_data)?; db.query(format!("REMOVE DATABASE {db_name}")).await?; - for schema in DB_SCHEMA_FILES.map(std::fs::read_to_string) { + for schema in MIGRATION_0_SCHEMA.map(std::fs::read_to_string) { db.query(schema?).await?; } surreal_brain::db::migration0(&db, &mock_data).await?;