Avoid locking for checking if a process has been forced to exit

It turns out taking a lock in every system call is a significant
performance bottleneck. In light of this finding, we replace a mutex in
a critical path of system call with an atomic boolean.

src/libos/src/process/do_exit.rs

@@ -23,8 +23,8 @@ pub fn do_exit(status: i32) {
 /// A thread may be forced to exit for two reasons: 1) a fatal signal; 2)
 /// exit_group syscall.
 pub fn handle_force_exit() {
-    if let Some(term_status) = current!().process().is_forced_exit() {
+    if current!().process().is_forced_to_exit() {
-        exit_thread(term_status);
+        exit_thread(current!().process().term_status().unwrap());
     }
 }
 

src/libos/src/process/mod.rs

@@ -25,7 +25,7 @@ pub use self::do_spawn::do_spawn_without_exec;
 pub use self::process::{Process, ProcessFilter, ProcessStatus, IDLE};
 pub use self::syscalls::*;
 pub use self::task::Task;
-pub use self::term_status::TermStatus;
+pub use self::term_status::{ForcedExitStatus, TermStatus};
 pub use self::thread::{Thread, ThreadStatus};
 
 mod do_arch_prctl;

src/libos/src/process/process/builder.rs

@@ -1,7 +1,8 @@
 use super::super::task::Task;
 use super::super::thread::{ThreadBuilder, ThreadId};
 use super::super::{
-    FileTableRef, FsViewRef, ProcessRef, ProcessVMRef, ResourceLimitsRef, SchedAgentRef,
+    FileTableRef, ForcedExitStatus, FsViewRef, ProcessRef, ProcessVMRef, ResourceLimitsRef,
+    SchedAgentRef,
 };
 use super::{Process, ProcessInner};
 use crate::prelude::*;
@@ -96,7 +97,7 @@ impl ProcessBuilder {
             let inner = SgxMutex::new(ProcessInner::new());
             let sig_dispositions = SgxRwLock::new(SigDispositions::new());
             let sig_queues = SgxMutex::new(SigQueues::new());
-            let forced_exit = SgxRwLock::new(None);
+            let forced_exit_status = ForcedExitStatus::new();
             Arc::new(Process {
                 pid,
                 exec_path,
@@ -104,7 +105,7 @@ impl ProcessBuilder {
                 inner,
                 sig_dispositions,
                 sig_queues,
-                forced_exit,
+                forced_exit_status,
             })
         };
 

src/libos/src/process/process/mod.rs

@@ -1,7 +1,7 @@
 use std::fmt;
 
 use super::wait::WaitQueue;
-use super::{ProcessRef, TermStatus, ThreadRef};
+use super::{ForcedExitStatus, ProcessRef, TermStatus, ThreadRef};
 use crate::prelude::*;
 use crate::signal::{SigDispositions, SigNum, SigQueues};
 
@@ -21,7 +21,7 @@ pub struct Process {
     // Signal
     sig_dispositions: SgxRwLock<SigDispositions>,
     sig_queues: SgxMutex<SigQueues>,
-    forced_exit: SgxRwLock<Option<TermStatus>>,
+    forced_exit_status: ForcedExitStatus,
 }
 
 #[derive(Debug, PartialEq, Clone, Copy)]
@@ -109,9 +109,13 @@ impl Process {
         &self.sig_dispositions
     }
 
+    pub fn term_status(&self) -> Option<TermStatus> {
+        self.forced_exit_status.term_status()
+    }
+
     /// Check whether the process has been forced to exit.
-    pub fn is_forced_exit(&self) -> Option<TermStatus> {
+    pub fn is_forced_to_exit(&self) -> bool {
-        *self.forced_exit.read().unwrap()
+        self.forced_exit_status.is_forced_to_exit()
     }
 
     /// Force a process to exit.
@@ -122,8 +126,7 @@ impl Process {
     ///
     /// A process may be forced to exit many times, but only the first time counts.
     pub fn force_exit(&self, term_status: TermStatus) {
-        let mut forced_exit = self.forced_exit.write().unwrap();
+        self.forced_exit_status.force_exit(term_status);
-        forced_exit.get_or_insert(term_status);
     }
 
     /// Get the internal representation of the process.

src/libos/src/process/term_status.rs

@@ -1,6 +1,37 @@
 //! The termination status of a process or thread.
 
 use crate::signal::SigNum;
+use sgx_tstd::sync::SgxMutex;
+use std::sync::atomic::{AtomicBool, Ordering};
+
+pub struct ForcedExitStatus {
+    exited: AtomicBool,
+    status: SgxMutex<Option<TermStatus>>,
+}
+
+impl ForcedExitStatus {
+    pub fn new() -> Self {
+        Self {
+            exited: AtomicBool::new(false),
+            status: SgxMutex::new(None),
+        }
+    }
+
+    pub fn is_forced_to_exit(&self) -> bool {
+        self.exited.load(Ordering::SeqCst)
+    }
+
+    pub fn force_exit(&self, status: TermStatus) {
+        let mut old_status = self.status.lock().unwrap();
+        // set the bool after getting the status lock
+        self.exited.store(true, Ordering::SeqCst);
+        old_status.get_or_insert(status);
+    }
+
+    pub fn term_status(&self) -> Option<TermStatus> {
+        *self.status.lock().unwrap()
+    }
+}
 
 // TODO: support core dump
 #[derive(Debug, Copy, Clone, PartialEq)]

src/libos/src/process/thread/mod.rs

@@ -3,8 +3,8 @@ use std::ptr::NonNull;
 
 use super::task::Task;
 use super::{
-    FileTableRef, FsViewRef, ProcessRef, ProcessVM, ProcessVMRef, ResourceLimitsRef, SchedAgentRef,
+    FileTableRef, ForcedExitStatus, FsViewRef, ProcessRef, ProcessVM, ProcessVMRef,
-    TermStatus, ThreadRef,
+    ResourceLimitsRef, SchedAgentRef, TermStatus, ThreadRef,
 };
 use crate::prelude::*;
 use crate::signal::{SigQueues, SigSet, SigStack};

src/libos/src/signal/do_sigreturn.rs

@@ -52,7 +52,7 @@ pub fn deliver_signal(cpu_context: &mut CpuContext) {
     let thread = current!();
     let process = thread.process();
 
-    if process.is_forced_exit().is_none() {
+    if !process.is_forced_to_exit() {
         do_deliver_signal(&thread, &process, cpu_context);
     }