From 986d1d2e445b65dd6fb5bcc88fb127e2b587880c Mon Sep 17 00:00:00 2001
From: "Tate, Hongliang Tian" <tate.thl@antfin.com>
Date: Mon, 20 Jan 2020 07:40:52 +0000
Subject: [PATCH] Fix ioctl SGX_IOC_GET_EPID_GROUP_ID for EBUSY error

SGX SDK's sgx_init_quote may return SGX_ERROR_BUSY, which is previously not
handled. The implementation of ioctl for /dev/sgx is now fixed to handle this
error.
---
 .../attestation/sgx_attestation_agent.rs      |  9 +++-
 src/pal/src/ocalls/attestation.c              |  3 ++
 test/ioctl/main.c                             | 41 ++++++++++++++-----
 3 files changed, 41 insertions(+), 12 deletions(-)

diff --git a/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs b/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs
index a2dd06ea..3bcc35f0 100644
--- a/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs
+++ b/src/libos/src/fs/dev_sgx/attestation/sgx_attestation_agent.rs
@@ -74,8 +74,13 @@ impl InnerAgent {
             );
             assert!(status == sgx_status_t::SGX_SUCCESS);
 
-            if (retval != sgx_status_t::SGX_SUCCESS) {
-                return_errno!(EINVAL, "occlum_ocall_sgx_init_quote failed");
+            if retval != sgx_status_t::SGX_SUCCESS {
+                match retval {
+                    sgx_status_t::SGX_ERROR_BUSY => {
+                        return_errno!(EBUSY, "occlum_ocall_sgx_init_quote is temporarily busy")
+                    }
+                    _ => return_errno!(EINVAL, "occlum_ocall_sgx_init_quote failed"),
+                }
             }
         }
 
diff --git a/src/pal/src/ocalls/attestation.c b/src/pal/src/ocalls/attestation.c
index c602514c..31fdb226 100644
--- a/src/pal/src/ocalls/attestation.c
+++ b/src/pal/src/ocalls/attestation.c
@@ -5,6 +5,9 @@ sgx_status_t occlum_ocall_sgx_init_quote(
     sgx_target_info_t* target_info,
     sgx_epid_group_id_t* epid_group_id)
 {
+    // Intel's manual:
+    // It's suggested that the caller should wait (typically several seconds
+    // to ten of seconds) and retry this API if SGX_ERROR_BUSY is returned.
     return sgx_init_quote(target_info, epid_group_id);
 }
 
diff --git a/test/ioctl/main.c b/test/ioctl/main.c
index 3324a1d7..059e7957 100644
--- a/test/ioctl/main.c
+++ b/test/ioctl/main.c
@@ -44,6 +44,9 @@ typedef struct {
 #define SGXIOC_GET_EPID_GROUP_ID _IOR('s', 1, sgx_epid_group_id_t)
 #define SGXIOC_GEN_QUOTE         _IOWR('s', 2, sgxioc_gen_quote_arg_t)
 
+// The max number of retries if ioctl returns EBUSY
+#define IOCTL_MAX_RETRIES       20
+
 typedef int(*sgx_ioctl_test_body_t)(int sgx_fd);
 
 static int do_SGXIOC_IS_EDDM_SUPPORTED(int sgx_fd) {
@@ -57,10 +60,25 @@ static int do_SGXIOC_IS_EDDM_SUPPORTED(int sgx_fd) {
     return 0;
 }
 
+
 static int do_SGXIOC_GET_EPID_GROUP_ID(int sgx_fd) {
-    sgx_epid_group_id_t epid_group_id = { 0 };
-    if (ioctl(sgx_fd, SGXIOC_GET_EPID_GROUP_ID, &epid_group_id) < 0) {
-        THROW_ERROR("failed to ioctl /dev/sgx");
+    int nretries = 0;
+    while (nretries < IOCTL_MAX_RETRIES) {
+        sgx_epid_group_id_t epid_group_id = { 0 };
+        int ret = ioctl(sgx_fd, SGXIOC_GET_EPID_GROUP_ID, &epid_group_id);
+        if (ret == 0) {
+            break;
+        }
+        else if (errno != EBUSY) {
+            THROW_ERROR("failed to ioctl /dev/sgx");
+        }
+
+        printf("WARN: /dev/sgx is temporarily busy. Try again after 1 second.");
+        sleep(1);
+        nretries++;
+    }
+    if (nretries == IOCTL_MAX_RETRIES) {
+        THROW_ERROR("failed to ioctl /dev/sgx due to timeout");
     }
     return 0;
 }
@@ -77,19 +95,22 @@ static int do_SGXIOC_GEN_QUOTE(int sgx_fd) {
         .quote_buf_len = sizeof(quote_buf),             // input
         .quote = { .as_buf = (uint8_t*) quote_buf }     // output
     };
-
-    while (1) {
+    int nretries = 0;
+    while (nretries < IOCTL_MAX_RETRIES) {
         int ret = ioctl(sgx_fd, SGXIOC_GEN_QUOTE, &gen_quote_arg);
         if (ret == 0) {
             break;
         }
-        else if (errno == EAGAIN) {
-            printf("WARN: /dev/sgx is temporarily busy. Try again after 1 second.");
-            sleep(1);
-        }
-        else {
+        else if (errno != EBUSY) {
             THROW_ERROR("failed to ioctl /dev/sgx");
         }
+
+        printf("WARN: /dev/sgx is temporarily busy. Try again after 1 second.");
+        sleep(1);
+        nretries++;
+    }
+    if (nretries == IOCTL_MAX_RETRIES) {
+        THROW_ERROR("failed to ioctl /dev/sgx due to timeout");
     }
 
     sgx_quote_t* quote = (sgx_quote_t*)quote_buf;