testnet upgrade

Signed-off-by: Valentyn Faychuk <valy@detee.ltd>

Cargo.toml

@@ -42,3 +42,6 @@ tonic-build = "0.12"
 [patch.crates-io.curve25519-dalek]
 git = "https://github.com/anza-xyz/curve25519-dalek.git"
 rev = "b500cdc2a920cd5bff9e2dd974d7b97349d61464"
+
+[features]
+test = []

docker/challenge.sh

@@ -8,4 +8,10 @@ else
   echo $INIT_NODES | tr ' ' '\n' > detee_challenge_nodes
 fi
 
+if [ -z "${NODE_IP}" ]; then
+  echo "The NODE_IP environment variable is not set."
+else
+  echo $NODE_IP > detee_node_ip
+fi
+
 occlum run /bin/hacker-challenge

scripts/build-container.sh

@@ -14,7 +14,11 @@ fi
 
 cd docker
 cp ../challenge_instance/challenge.tar.gz challenge.tar.gz
+if [ -n "$TEST" ]; then
+    docker build -f challenge.Dockerfile -t detee/hacker-challenge:test .
+else
     docker build -f challenge.Dockerfile -t detee/hacker-challenge:latest .
+fi
 cd ..
 
 echo "Container named 'hacker-challenge' built"

scripts/package.sh

@@ -20,8 +20,13 @@ if [ "$prerequisites" == "--prep" ]; then
     (cd occlum/tools/toolchains/utils_lib && ./build.sh)
 fi
 
-echo "Building the hacker-challenge signed bundle"
+if [ -n "$TEST" ]; then
+    echo "Building the hacker-challenge with test feature"
+    occlum-cargo build --release --features test
+else
+    echo "Building the hacker-challenge"
     occlum-cargo build --release
+fi
 strip target/x86_64-unknown-linux-musl/release/hacker-challenge
 sha256sum target/x86_64-unknown-linux-musl/release/hacker-challenge
 

scripts/testnet.sh

@@ -31,45 +31,67 @@ function build_mint_sol_tool() {
     cd ..
 }
 
-source "${script_dir}/build-container.sh"
+TEST=1 ./${script_dir}/build-container.sh
-build_mint_sol_tool
+[ -e "${script_dir}/mint_sol" ] || build_mint_sol_tool
 
-# Cleanup old containers and run the network root
+# Cleanup old containers and the network
-docker ps -a | grep 'hacker-challenge' | awk '{ print $NF }' | xargs docker rm -f || true
+echo "Creating the network and the root node"
+docker ps -a | grep 'dthc' | awk '{ print $NF }' | xargs docker rm -f || true
+docker network inspect dthc > /dev/null 2>&1 \
+    || docker network create --subnet=172.18.0.0/24 dthc \
+    || true
 
-echo "Waiting for the network root to start"
+echo "Waiting for the root node to start"
-docker run --device /dev/sgx/enclave \
+# 172.18.0.1 is for the network gateway
+docker run --name dthc-root \
+           --network dthc -d \
+           --ip 172.18.0.2 \
+           --env NODE_IP="172.18.0.2" \
            --device /dev/sgx/provision \
-           --name "hacker-challenge" \
+           --device /dev/sgx/enclave \
-           -d detee/hacker-challenge:latest
+           detee/hacker-challenge:test
 while true; do
     echo -n "." && sleep 1
-    docker logs hacker-challenge | grep -q "SOL" && echo && break
+    docker logs dthc-root | grep -q "SOL" && echo && break
 done
 
 echo "Sending SOL to the root and waiting for the mint"
-address=$(docker logs hacker-challenge | grep 'SOL' | awk '{ print $NF }')
+address=$(docker logs dthc-root | grep 'SOL' | awk '{ print $NF }')
 "${script_dir}"/mint_sol "${address}"
 while true; do
     echo -n "." && sleep 1
-    docker logs hacker-challenge | grep -q "Mint created" && echo && break
+    docker logs dthc-root | grep -q "Mint created" && echo && break
 done
 
 echo "Creating the cluster"
-for p in {31311..31320}; do
+for n in {1..20}; do
-    docker run --device /dev/sgx/enclave \
+    #init_nodes=$(docker inspect dthc-root --format '{{ .NetworkSettings.Networks.dthc.IPAddress }}')
+    node_ip="172.18.0.$((2 + n))"
+    node_port=$((31300 + n))
+    node_volume="/tmp/dthc${n}"
+
+    docker run --name dthc${n} \
+               --network dthc -d \
+               --ip ${node_ip} \
+               --env NODE_IP="${node_ip}" \
+               --env INIT_NODES="172.18.0.2 172.18.0.3 172.18.0.4" \
+               --volume ${node_volume}:/challenge/main \
+               --publish ${node_port}:31372 \
                --device /dev/sgx/provision \
-               --env INIT_NODES="172.17.0.2 172.17.0.3 172.17.0.4" \
+               --device /dev/sgx/enclave \
-               -v "/tmp/hacker-challenge${p}:/challenge/main" \
+               detee/hacker-challenge:test
-               --name "hacker-challenge${p}" -p "${p}:31372" \
-               -d detee/hacker-challenge:latest
 done
 sleep 15 # Wait for the cluster to start
 
 echo "Running the test mint"
-for p in {31311..31320}; do
+for n in {1..20}; do
-    curl -X POST "127.0.0.1:${p}/mint" \
+    node_port=$((31300 + n))
+    curl -X POST "127.0.0.1:${node_port}/mint" \
          --json '{"wallet": "EZT16iP1SQVUFf1AJN6oiE5BZPnyBUqaKDkZ4oZRsvhR"}' \
          --connect-timeout 5 2> /dev/null
     echo ""
 done
+
+# curl 127.0.0.1:31303/metrics
+# curl -X POST 127.0.0.1:31303/mint -d '{"wallet": "EZT16iP1SQVUFf1AJN6oiE5BZPnyBUqaKDkZ4oZRsvhR"}' -H 'Content-Type: application/json'
+# docker run --network dthc -d --name dthc-3 --ip 172.18.0.3 --env NODE_IP='172.18.0.3' --env INIT_NODES='172.18.0.2' -v /tmp/dthc3:/challenge/main -p 31303:31372 --device /dev/sgx/provision --device /dev/sgx/enclave detee/hacker-challenge:test

src/datastore.rs

@@ -26,7 +26,7 @@ pub struct NodeInfo {
 
 impl NodeInfo {
     pub fn is_newer_than(&self, older_self: &Self) -> bool {
-        self.keepalive >= older_self.keepalive
+        self.keepalive > older_self.keepalive
     }
 
     pub fn to_json(&self) -> String {

src/main.rs

@@ -12,8 +12,8 @@ use datastore::State;
 use detee_sgx::{InstanceMeasurement, RaTlsConfig};
 use std::{
     fs::File,
+    io::Error,
     io::{BufRead, BufReader},
-    io::{Error, ErrorKind},
     sync::Arc,
 };
 use tokio::{
@@ -26,9 +26,22 @@ const INIT_NODES_FILE: &str = "/host/detee_challenge_nodes";
 const KEYS_FILE: &str = "/host/main/TRY_TO_HACK_THIS";
 const MAX_CONNECTIONS: usize = 3;
 
+#[cfg(feature = "test")]
+async fn resolve_my_ip() -> Result<String, Error> {
+    let node_ip = File::open("/host/detee_node_ip")?;
+    let mut reader = BufReader::new(node_ip);
+    let mut ip = String::new();
+    reader.read_line(&mut ip)?;
+    if ip.ends_with('\n') {
+        ip.pop();
+    }
+    Ok(ip)
+}
+
+#[cfg(not(feature = "test"))]
 async fn resolve_my_ip() -> Result<String, Error> {
     let err = "Can't resolve my external IP, try again";
-    let ip = public_ip::addr_v4().await.ok_or(Error::new(ErrorKind::Other, err))?;
+    let ip = public_ip::addr_v4().await.ok_or(Error::new(std::io::ErrorKind::Other, err))?;
     Ok(format!("{}", ip))
 }
 
@@ -108,8 +121,9 @@ async fn main() {
     env_logger::init_from_env(env_logger::Env::default().default_filter_or("warn"));
     let ratls_config = RaTlsConfig::new()
         .allow_instance_measurement(InstanceMeasurement::new().with_current_mrenclave().unwrap());
-
     let my_ip = resolve_my_ip().await.unwrap();
+    println!("Starting on IP {}", my_ip);
+
     let state = Arc::new(State::new(my_ip.clone()));
     let sol_client = Arc::new(get_sol_client(state.clone(), ratls_config.clone()).await);