Merge pull request #70 for networking system calls

deps/sefs

@@ -1 +1 @@
-Subproject commit 166616e5ade1a5c929f705fd1564ef0ea337ba72
+Subproject commit 2865c419b3d36a8b0e7ef843b115cb1ecb3176f8

src/Enclave.edl

@@ -4,11 +4,14 @@ enclave {
     from "sgx_tstdc.edl" import *;
     from "sgx_tstd.edl" import *;
     from "sgx_tprotected_fs.edl" import *;
+    from "sgx_net.edl" import *;
 
     trusted {
         /* define ECALLs here. */
         public int libos_boot([in, string] const char* executable_path, [user_check] const char** argv);
         public int libos_run(void);
+        /* This is only for debug usage */
+        public int dummy_ecall(void);
     };
 
     untrusted {

src/libos/Cargo.toml

@@ -14,7 +14,10 @@ rcore-fs = { path = "../../deps/sefs/rcore-fs" }
 rcore-fs-sefs = { path = "../../deps/sefs/rcore-fs-sefs" }
 
 [features]
-default = []
+default = ["integrity_only_opt", "sgx_file_cache"]
+syscall_timing = []     # Timing for each syscall. But it has cost from more ocall.
+integrity_only_opt = [] # Clear bss only. It should be disabled if checking memory reads.
+sgx_file_cache = []     # Cache SgxFile objects. Invalidation is unimplemented.
 
 [target.'cfg(not(target_env = "sgx"))'.dependencies]
 xmas-elf = { path = "../../deps/xmas-elf" }

src/libos/Enclave_config.xml

@@ -3,7 +3,7 @@
   <ProdID>0</ProdID>
   <ISVSVN>0</ISVSVN>
   <StackMaxSize>0x100000</StackMaxSize>
-  <HeapMaxSize>0x1000000</HeapMaxSize>
+  <HeapMaxSize>0x2000000</HeapMaxSize>
   <TCSNum>8</TCSNum>
   <TCSPolicy>1</TCSPolicy>
   <DisableDebug>0</DisableDebug>

src/libos/src/entry.rs

@@ -35,6 +35,10 @@ pub extern "C" fn libos_run() -> i32 {
     .unwrap_or(EXIT_STATUS_INTERNAL_ERROR)
 }
 
+#[no_mangle]
+pub extern "C" fn dummy_ecall() -> i32 {
+    0
+}
 // Use 127 as a special value to indicate internal error from libos, not from
 // user programs, although it is completely ok for a user program to return 127.
 const EXIT_STATUS_INTERNAL_ERROR: i32 = 127;
@@ -64,7 +68,7 @@ fn parse_arguments(
 
 // TODO: make sure do_boot can only be called once
 fn do_boot(path_str: &str, argv: &Vec<CString>) -> Result<(), Error> {
-    info!("boot: path: {:?}, argv: {:?}", path_str, argv);
+    //    info!("boot: path: {:?}, argv: {:?}", path_str, argv);
     util::mpx_util::mpx_enable()?;
 
     let envp = std::vec::Vec::new();
@@ -78,5 +82,11 @@ fn do_boot(path_str: &str, argv: &Vec<CString>) -> Result<(), Error> {
 // TODO: make sure do_run() cannot be called after do_boot()
 fn do_run() -> Result<i32, Error> {
     let exit_status = process::run_task()?;
+
+    // sync file system
+    // TODO: only sync when all processes exit
+    use rcore_fs::vfs::FileSystem;
+    crate::fs::ROOT_INODE.fs().sync()?;
+
     Ok(exit_status)
 }

src/libos/src/errno.rs

@@ -22,6 +22,15 @@ impl convert::From<(Errno, &'static str)> for Error {
     }
 }
 
+impl convert::From<std::io::Error> for Error {
+    fn from(info: std::io::Error) -> Error {
+        Error::new(
+            Errno::from_errno(info.raw_os_error().unwrap()),
+            "std::io::Error",
+        )
+    }
+}
+
 impl error::Error for Error {
     fn description(&self) -> &str {
         self.desc
@@ -39,6 +48,7 @@ impl fmt::Display for Error {
 }
 
 #[derive(Clone, Copy, Debug, PartialEq)]
+#[repr(u8)]
 pub enum Errno {
     EUNDEF = 0,
     EPERM = 1,
@@ -80,12 +90,112 @@ pub enum Errno {
     ENOLCK = 37,
     ENOSYS = 38,
     ENOTEMPTY = 39,
+    ELOOP = 40,
+    EWOULDBLOCK = 41,
+    ENOMSG = 42,
+    EIDRM = 43,
+    ECHRNG = 44,
+    EL2NSYNC = 45,
+    EL3HLT = 46,
+    EL3RST = 47,
+    ELNRNG = 48,
+    EUNATCH = 49,
+    ENOCSI = 50,
+    EL2HLT = 51,
+    EBADE = 52,
+    EBADR = 53,
+    EXFULL = 54,
+    ENOANO = 55,
+    EBADRQC = 56,
+    EBADSLT = 57,
+    EDEADLOCK = 58,
+    EBFONT = 59,
+    ENOSTR = 60,
+    ENODATA = 61,
+    ETIME = 62,
+    ENOSR = 63,
+    ENONET = 64,
+    ENOPKG = 65,
+    EREMOTE = 66,
+    ENOLINK = 67,
+    EADV = 68,
+    ESRMNT = 69,
+    ECOMM = 70,
+    EPROTO = 71,
+    EMULTIHOP = 72,
+    EDOTDOT = 73,
+    EBADMSG = 74,
+    EOVERFLOW = 75,
+    ENOTUNIQ = 76,
+    EBADFD = 77,
+    EREMCHG = 78,
+    ELIBACC = 79,
+    ELIBBAD = 80,
+    ELIBSCN = 81,
+    ELIBMAX = 82,
+    ELIBEXEC = 83,
+    EILSEQ = 84,
+    ERESTART = 85,
+    ESTRPIPE = 86,
+    EUSERS = 87,
+    ENOTSOCK = 88,
+    EDESTADDRREQ = 89,
+    EMSGSIZE = 90,
+    EPROTOTYPE = 91,
+    ENOPROTOOPT = 92,
+    EPROTONOSUPPORT = 93,
+    ESOCKTNOSUPPORT = 94,
+    EOPNOTSUPP = 95,
+    EPFNOSUPPORT = 96,
+    EAFNOSUPPORT = 97,
+    EADDRINUSE = 98,
+    EADDRNOTAVAIL = 99,
+    ENETDOWN = 100,
+    ENETUNREACH = 101,
+    ENETRESET = 102,
+    ECONNABORTED = 103,
+    ECONNRESET = 104,
+    ENOBUFS = 105,
+    EISCONN = 106,
+    ENOTCONN = 107,
+    ESHUTDOWN = 108,
+    ETOOMANYREFS = 109,
+    ETIMEDOUT = 110,
+    ECONNREFUSED = 111,
+    EHOSTDOWN = 112,
+    EHOSTUNREACH = 113,
+    EALREADY = 114,
+    EINPROGRESS = 115,
+    ESTALE = 116,
+    EUCLEAN = 117,
+    ENOTNAM = 118,
+    ENAVAIL = 119,
+    EISNAM = 120,
+    EREMOTEIO = 121,
+    EDQUOT = 122,
+    ENOMEDIUM = 123,
+    EMEDIUMTYPE = 124,
+    ECANCELED = 125,
+    ENOKEY = 126,
+    EKEYEXPIRED = 127,
+    EKEYREVOKED = 128,
+    EKEYREJECTED = 129,
+    EOWNERDEAD = 130,
+    ENOTRECOVERABLE = 131,
+    ERFKILL = 132,
+    EHWPOISON = 133,
 }
 
 impl Errno {
     pub fn as_retval(&self) -> i32 {
         -(*self as i32)
     }
+    pub fn from_errno(mut errno: i32) -> Self {
+        if errno < 0 || errno > 133 {
+            errno = 0;
+        }
+        unsafe { core::mem::transmute(errno as u8) }
+    }
 }
 
 impl fmt::Display for Errno {

src/libos/src/fs/access.rs

@@ -17,7 +17,6 @@ impl AccessModes {
     }
 }
 
-
 bitflags! {
     pub struct AccessFlags : u32 {
         const AT_SYMLINK_NOFOLLOW = 0x100;
@@ -31,10 +30,18 @@ impl AccessFlags {
     }
 }
 
+pub const AT_FDCWD: i32 = -100;
 
-pub const AT_FDCWD : i32 = -100;
-
-pub fn do_faccessat(dirfd: Option<FileDesc>, path: &str, mode: AccessModes, flags: AccessFlags) -> Result<(), Error> {
+pub fn do_faccessat(
+    dirfd: Option<FileDesc>,
+    path: &str,
+    mode: AccessModes,
+    flags: AccessFlags,
+) -> Result<(), Error> {
+    info!(
+        "faccessat: dirfd: {:?}, path: {:?}, mode: {:?}, flags: {:?}",
+        dirfd, path, mode, flags
+    );
     match dirfd {
         // TODO: handle dirfd
         Some(dirfd) => errno!(ENOSYS, "cannot accept dirfd"),
@@ -43,6 +50,7 @@ pub fn do_faccessat(dirfd: Option<FileDesc>, path: &str, mode: AccessModes, flag
 }
 
 pub fn do_access(path: &str, mode: AccessModes) -> Result<(), Error> {
+    info!("access: path: {:?}, mode: {:?}", path, mode);
     let current_ref = process::get_current();
     let mut current = current_ref.lock().unwrap();
     let inode = current.lookup_inode(path)?;

src/libos/src/fs/file.rs

@@ -4,7 +4,7 @@ use std::borrow::BorrowMut;
 use std::fmt;
 use std::io::SeekFrom;
 
-pub trait File: Debug + Sync + Send {
+pub trait File: Debug + Sync + Send + Any {
     fn read(&self, buf: &mut [u8]) -> Result<usize, Error>;
     fn write(&self, buf: &[u8]) -> Result<usize, Error>;
     fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<usize, Error>;
@@ -17,6 +17,7 @@ pub trait File: Debug + Sync + Send {
     fn sync_all(&self) -> Result<(), Error>;
     fn sync_data(&self) -> Result<(), Error>;
     fn read_entry(&self) -> Result<String, Error>;
+    fn as_any(&self) -> &Any;
 }
 
 pub type FileRef = Arc<Box<File>>;
@@ -35,7 +36,7 @@ impl SgxFile {
         is_append: bool,
     ) -> Result<SgxFile, Error> {
         if !is_readable && !is_writable {
-            return Err(Error::new(Errno::EINVAL, "Invalid permissions"));
+            return errno!(EINVAL, "Invalid permissions");
         }
 
         Ok(SgxFile {
@@ -114,6 +115,10 @@ impl File for SgxFile {
     fn read_entry(&self) -> Result<String, Error> {
         unimplemented!()
     }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
 }
 
 #[derive(Clone)]
@@ -130,7 +135,7 @@ struct SgxFileInner {
 impl SgxFileInner {
     pub fn write(&mut self, buf: &[u8]) -> Result<usize, Error> {
         if !self.is_writable {
-            return Err(Error::new(Errno::EINVAL, "File not writable"));
+            return errno!(EINVAL, "File not writable");
         }
 
         let mut file_guard = self.file.lock().unwrap();
@@ -158,7 +163,7 @@ impl SgxFileInner {
 
     pub fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
         if !self.is_readable {
-            return Err(Error::new(Errno::EINVAL, "File not readable"));
+            return errno!(EINVAL, "File not readable");
         }
 
         let mut file_guard = self.file.lock().unwrap();
@@ -191,7 +196,7 @@ impl SgxFileInner {
                     let backward_offset = (-relative_offset) as usize;
                     if self.pos < backward_offset {
                         // underflow
-                        return Err(Error::new(Errno::EINVAL, "Invalid seek position"));
+                        return errno!(EINVAL, "Invalid seek position");
                     }
                     SeekFrom::Start((self.pos - backward_offset) as u64)
                 }
@@ -207,7 +212,7 @@ impl SgxFileInner {
 
     pub fn writev(&mut self, bufs: &[&[u8]]) -> Result<usize, Error> {
         if !self.is_writable {
-            return Err(Error::new(Errno::EINVAL, "File not writable"));
+            return errno!(EINVAL, "File not writable");
         }
 
         let mut file_guard = self.file.lock().unwrap();
@@ -233,7 +238,7 @@ impl SgxFileInner {
                 Err(e) => {
                     match total_bytes {
                         // a complete failure
-                        0 => return Err(Error::new(Errno::EINVAL, "Failed to write")),
+                        0 => return errno!(EINVAL, "Failed to write"),
                         // a partially failure
                         _ => break,
                     }
@@ -247,7 +252,7 @@ impl SgxFileInner {
 
     fn readv(&mut self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
         if !self.is_readable {
-            return Err(Error::new(Errno::EINVAL, "File not readable"));
+            return errno!(EINVAL, "File not readable");
         }
 
         let mut file_guard = self.file.lock().unwrap();
@@ -269,7 +274,7 @@ impl SgxFileInner {
                 Err(e) => {
                     match total_bytes {
                         // a complete failure
-                        0 => return Err(Error::new(Errno::EINVAL, "Failed to write")),
+                        0 => return errno!(EINVAL, "Failed to write"),
                         // a partially failure
                         _ => break,
                     }
@@ -305,7 +310,7 @@ impl StdoutFile {
 
 impl File for StdoutFile {
     fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
-        Err(Error::new(Errno::EBADF, "Stdout does not support read"))
+        errno!(EBADF, "Stdout does not support read")
     }
 
     fn write(&self, buf: &[u8]) -> Result<usize, Error> {
@@ -318,16 +323,16 @@ impl File for StdoutFile {
         Ok(write_len)
     }
 
-    fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
-        unimplemented!()
+    fn read_at(&self, _offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
+        self.read(buf)
     }
 
-    fn write_at(&self, offset: usize, buf: &[u8]) -> Result<usize, Error> {
-        unimplemented!()
+    fn write_at(&self, _offset: usize, buf: &[u8]) -> Result<usize, Error> {
+        self.write(buf)
     }
 
     fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
-        Err(Error::new(Errno::EBADF, "Stdout does not support read"))
+        errno!(EBADF, "Stdout does not support read")
     }
 
     fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
@@ -344,7 +349,7 @@ impl File for StdoutFile {
                 Err(e) => {
                     match total_bytes {
                         // a complete failure
-                        0 => return Err(Error::new(Errno::EINVAL, "Failed to write")),
+                        0 => return errno!(EINVAL, "Failed to write"),
                         // a partially failure
                         _ => break,
                     }
@@ -355,27 +360,46 @@ impl File for StdoutFile {
     }
 
     fn seek(&self, seek_pos: SeekFrom) -> Result<off_t, Error> {
-        Err(Error::new(Errno::ESPIPE, "Stdout does not support seek"))
+        errno!(ESPIPE, "Stdout does not support seek")
     }
 
     fn metadata(&self) -> Result<Metadata, Error> {
-        unimplemented!()
+        Ok(Metadata {
+            dev: 0,
+            inode: 0,
+            size: 0,
+            blk_size: 0,
+            blocks: 0,
+            atime: Timespec { sec: 0, nsec: 0 },
+            mtime: Timespec { sec: 0, nsec: 0 },
+            ctime: Timespec { sec: 0, nsec: 0 },
+            type_: FileType::CharDevice,
+            mode: 0,
+            nlinks: 0,
+            uid: 0,
+            gid: 0,
+        })
     }
 
-    fn set_len(&self, len: u64) -> Result<(), Error> {
-        unimplemented!()
+    fn set_len(&self, _len: u64) -> Result<(), Error> {
+        errno!(EINVAL, "Stdout does not support set_len")
     }
 
     fn sync_all(&self) -> Result<(), Error> {
-        unimplemented!()
+        self.sync_data()
     }
 
     fn sync_data(&self) -> Result<(), Error> {
-        unimplemented!()
+        self.inner.lock().flush()?;
+        Ok(())
     }
 
     fn read_entry(&self) -> Result<String, Error> {
-        unimplemented!()
+        errno!(ENOTDIR, "Stdout does not support read_entry")
+    }
+
+    fn as_any(&self) -> &Any {
+        self
     }
 }
 
@@ -412,7 +436,7 @@ impl File for StdinFile {
     }
 
     fn write(&self, buf: &[u8]) -> Result<usize, Error> {
-        Err(Error::new(Errno::EBADF, "Stdin does not support write"))
+        errno!(EBADF, "Stdin does not support write")
     }
 
     fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
@@ -437,7 +461,7 @@ impl File for StdinFile {
                 Err(e) => {
                     match total_bytes {
                         // a complete failure
-                        0 => return Err(Error::new(Errno::EINVAL, "Failed to write")),
+                        0 => return errno!(EINVAL, "Failed to write"),
                         // a partially failure
                         _ => break,
                     }
@@ -448,31 +472,49 @@ impl File for StdinFile {
     }
 
     fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
-        Err(Error::new(Errno::EBADF, "Stdin does not support write"))
+        errno!(EBADF, "Stdin does not support write")
     }
 
     fn seek(&self, pos: SeekFrom) -> Result<off_t, Error> {
-        Err(Error::new(Errno::ESPIPE, "Stdin does not support seek"))
+        errno!(ESPIPE, "Stdin does not support seek")
     }
 
     fn metadata(&self) -> Result<Metadata, Error> {
-        unimplemented!()
+        Ok(Metadata {
+            dev: 0,
+            inode: 0,
+            size: 0,
+            blk_size: 0,
+            blocks: 0,
+            atime: Timespec { sec: 0, nsec: 0 },
+            mtime: Timespec { sec: 0, nsec: 0 },
+            ctime: Timespec { sec: 0, nsec: 0 },
+            type_: FileType::CharDevice,
+            mode: 0,
+            nlinks: 0,
+            uid: 0,
+            gid: 0,
+        })
     }
 
-    fn set_len(&self, len: u64) -> Result<(), Error> {
-        unimplemented!()
+    fn set_len(&self, _len: u64) -> Result<(), Error> {
+        errno!(EINVAL, "Stdin does not support set_len")
     }
 
     fn sync_all(&self) -> Result<(), Error> {
-        unimplemented!()
+        self.sync_data()
     }
 
     fn sync_data(&self) -> Result<(), Error> {
-        unimplemented!()
+        Ok(())
     }
 
     fn read_entry(&self) -> Result<String, Error> {
-        unimplemented!()
+        errno!(ENOTDIR, "Stdin does not support read_entry")
+    }
+
+    fn as_any(&self) -> &Any {
+        self
     }
 }
 

src/libos/src/fs/file_table.rs

@@ -4,7 +4,7 @@ use std;
 
 pub type FileDesc = u32;
 
-#[derive(Debug, Default)]
+#[derive(Debug, Default, Clone)]
 #[repr(C)]
 pub struct FileTable {
     table: Vec<Option<FileTableEntry>>,
@@ -19,7 +19,12 @@ impl FileTable {
         }
     }
 
-    pub fn dup(&mut self, fd: FileDesc, min_fd: FileDesc, close_on_spawn: bool) -> Result<FileDesc, Error> {
+    pub fn dup(
+        &mut self,
+        fd: FileDesc,
+        min_fd: FileDesc,
+        close_on_spawn: bool,
+    ) -> Result<FileDesc, Error> {
         let file_ref = self.get(fd)?;
 
         let min_fd = min_fd as usize;
@@ -34,11 +39,13 @@ impl FileTable {
                 }
             }
 
-            table.iter()
+            table
+                .iter()
                 .enumerate()
                 .skip(min_fd as usize)
                 .find(|&(idx, opt)| opt.is_none())
-                .unwrap().0
+                .unwrap()
+                .0
         } as FileDesc;
 
         self.put_at(min_free_fd, file_ref, close_on_spawn);
@@ -50,7 +57,8 @@ impl FileTable {
         let mut table = &mut self.table;
 
         let min_free_fd = if self.num_fds < table.len() {
-            table.iter()
+            table
+                .iter()
                 .enumerate()
                 .find(|&(idx, opt)| opt.is_none())
                 .unwrap()
@@ -123,30 +131,19 @@ impl FileTable {
             None => errno!(EBADF, "Invalid file descriptor"),
         }
     }
-}
 
-impl Clone for FileTable {
-    fn clone(&self) -> FileTable {
-        // Only clone file descriptors that are not close-on-spawn
-        let mut num_cloned_fds = 0;
-        let cloned_table = self
-            .table
-            .iter()
-            .map(|entry| match entry {
-                Some(file_table_entry) => match file_table_entry.close_on_spawn {
-                    false => {
-                        num_cloned_fds += 1;
-                        Some(file_table_entry.clone())
+    /// Remove file descriptors that are close-on-spawn
+    pub fn close_on_spawn(&mut self) {
+        for entry in self.table.iter_mut() {
+            let need_close = if let Some(entry) = entry {
+                entry.close_on_spawn
+            } else {
+                false
+            };
+            if need_close {
+                *entry = None;
+                self.num_fds -= 1;
             }
-                    true => None,
-                },
-                None => None,
-            })
-            .collect();
-
-        FileTable {
-            table: cloned_table,
-            num_fds: num_cloned_fds,
         }
     }
 }

src/libos/src/fs/inode_file.rs

@@ -2,8 +2,8 @@ use rcore_fs::vfs::{FileSystem, FsError, INode};
 use rcore_fs_sefs::SEFS;
 use std::fmt;
 
-use super::*;
 use super::sgx_impl::SgxStorage;
+use super::*;
 
 lazy_static! {
     /// The root of file system
@@ -32,7 +32,7 @@ pub struct OpenOptions {
 impl File for INodeFile {
     fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
         if !self.options.read {
-            return Err(Error::new(Errno::EBADF, "File not readable"));
+            return errno!(EBADF, "File not readable");
         }
         let mut offset = self.offset.lock().unwrap();
         let len = self.inode.read_at(*offset, buf)?;
@@ -42,7 +42,7 @@ impl File for INodeFile {
 
     fn write(&self, buf: &[u8]) -> Result<usize, Error> {
         if !self.options.write {
-            return Err(Error::new(Errno::EBADF, "File not writable"));
+            return errno!(EBADF, "File not writable");
         }
         let mut offset = self.offset.lock().unwrap();
         if self.options.append {
@@ -56,7 +56,7 @@ impl File for INodeFile {
 
     fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
         if !self.options.read {
-            return Err(Error::new(Errno::EBADF, "File not readable"));
+            return errno!(EBADF, "File not readable");
         }
         let len = self.inode.read_at(offset, buf)?;
         Ok(len)
@@ -64,7 +64,7 @@ impl File for INodeFile {
 
     fn write_at(&self, offset: usize, buf: &[u8]) -> Result<usize, Error> {
         if !self.options.write {
-            return Err(Error::new(Errno::EBADF, "File not writable"));
+            return errno!(EBADF, "File not writable");
         }
         let len = self.inode.write_at(offset, buf)?;
         Ok(len)
@@ -72,7 +72,7 @@ impl File for INodeFile {
 
     fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
         if !self.options.read {
-            return Err(Error::new(Errno::EBADF, "File not readable"));
+            return errno!(EBADF, "File not readable");
         }
         let mut offset = self.offset.lock().unwrap();
         let mut total_len = 0;
@@ -91,7 +91,7 @@ impl File for INodeFile {
 
     fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
         if !self.options.write {
-            return Err(Error::new(Errno::EBADF, "File not writable"));
+            return errno!(EBADF, "File not writable");
         }
         let mut offset = self.offset.lock().unwrap();
         if self.options.append {
@@ -129,7 +129,7 @@ impl File for INodeFile {
 
     fn set_len(&self, len: u64) -> Result<(), Error> {
         if !self.options.write {
-            return Err(Error::new(EBADF, "File not writable. Can't set len."));
+            return errno!(EBADF, "File not writable. Can't set len.");
         }
         self.inode.resize(len as usize)?;
         Ok(())
@@ -147,13 +147,17 @@ impl File for INodeFile {
 
     fn read_entry(&self) -> Result<String, Error> {
         if !self.options.read {
-            return Err(Error::new(EBADF, "File not readable. Can't read entry."));
+            return errno!(EBADF, "File not readable. Can't read entry.");
         }
         let mut offset = self.offset.lock().unwrap();
         let name = self.inode.get_entry(*offset)?;
         *offset += 1;
         Ok(name)
     }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
 }
 
 impl INodeFile {

src/libos/src/fs/io_multiplexing.rs

@@ -0,0 +1,380 @@
+use super::*;
+use std::any::Any;
+use std::collections::btree_map::BTreeMap;
+use std::fmt;
+use std::sync::atomic::spin_loop_hint;
+use std::vec::Vec;
+
+/// Forward to host `poll`
+/// (sgx_libc doesn't have `select`)
+pub fn do_select(
+    nfds: usize,
+    readfds: &mut libc::fd_set,
+    writefds: &mut libc::fd_set,
+    exceptfds: &mut libc::fd_set,
+    timeout: Option<libc::timeval>,
+) -> Result<usize, Error> {
+    info!("select: nfds: {}", nfds);
+    // convert libos fd to Linux fd
+    let mut host_to_libos_fd = [0; libc::FD_SETSIZE];
+    let mut polls = Vec::<libc::pollfd>::new();
+
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_table_ref = proc.get_files().lock().unwrap();
+
+    for fd in 0..nfds {
+        let (r, w, e) = (
+            readfds.is_set(fd),
+            writefds.is_set(fd),
+            exceptfds.is_set(fd),
+        );
+        if !(r || w || e) {
+            continue;
+        }
+        if let Ok(socket) = file_table_ref.get(fd as FileDesc)?.as_unix_socket() {
+            warn!("select unix socket is unimplemented, spin for read");
+            readfds.clear();
+            writefds.clear();
+            exceptfds.clear();
+
+            // FIXME: spin poll until can read (hack for php)
+            while r && socket.poll()?.0 == false {
+                spin_loop_hint();
+            }
+
+            let (rr, ww, ee) = socket.poll()?;
+            if r && rr {
+                readfds.set(fd);
+            }
+            if w && ww {
+                writefds.set(fd);
+            }
+            if e && ee {
+                writefds.set(fd);
+            }
+            return Ok(1);
+        }
+        let host_fd = file_table_ref.get(fd as FileDesc)?.as_socket()?.fd();
+
+        host_to_libos_fd[host_fd as usize] = fd;
+        let mut events = 0;
+        if r {
+            events |= libc::POLLIN;
+        }
+        if w {
+            events |= libc::POLLOUT;
+        }
+        if e {
+            events |= libc::POLLERR;
+        }
+
+        polls.push(libc::pollfd {
+            fd: host_fd as c_int,
+            events,
+            revents: 0,
+        });
+    }
+
+    let timeout = match timeout {
+        None => -1,
+        Some(tv) => (tv.tv_sec * 1000 + tv.tv_usec / 1000) as i32,
+    };
+
+    let ret = try_libc!(libc::ocall::poll(
+        polls.as_mut_ptr(),
+        polls.len() as u64,
+        timeout
+    ));
+
+    // convert fd back and write fdset
+    readfds.clear();
+    writefds.clear();
+    exceptfds.clear();
+
+    for poll in polls.iter() {
+        let fd = host_to_libos_fd[poll.fd as usize];
+        if poll.revents & libc::POLLIN != 0 {
+            readfds.set(fd);
+        }
+        if poll.revents & libc::POLLOUT != 0 {
+            writefds.set(fd);
+        }
+        if poll.revents & libc::POLLERR != 0 {
+            exceptfds.set(fd);
+        }
+    }
+
+    Ok(ret as usize)
+}
+
+pub fn do_poll(polls: &mut [libc::pollfd], timeout: c_int) -> Result<usize, Error> {
+    info!(
+        "poll: {:?}, timeout: {}",
+        polls.iter().map(|p| p.fd).collect::<Vec<_>>(),
+        timeout
+    );
+
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+
+    // convert libos fd to Linux fd
+    for poll in polls.iter_mut() {
+        let file_ref = proc.get_files().lock().unwrap().get(poll.fd as FileDesc)?;
+        if let Ok(socket) = file_ref.as_socket() {
+            poll.fd = socket.fd();
+        } else if let Ok(socket) = file_ref.as_unix_socket() {
+            // FIXME: spin poll until can read (hack for php)
+            while (poll.events & libc::POLLIN) != 0 && socket.poll()?.0 == false {
+                spin_loop_hint();
+            }
+
+            let (r, w, e) = socket.poll()?;
+            if r {
+                poll.revents |= libc::POLLIN;
+            }
+            if w {
+                poll.revents |= libc::POLLOUT;
+            }
+            if e {
+                poll.revents |= libc::POLLERR;
+            }
+            poll.revents &= poll.events;
+            warn!("poll unix socket is unimplemented, spin for read");
+            return Ok(1);
+        } else {
+            return errno!(EBADF, "not a socket");
+        }
+    }
+    let ret = try_libc!(libc::ocall::poll(
+        polls.as_mut_ptr(),
+        polls.len() as u64,
+        timeout
+    ));
+    // recover fd ?
+    Ok(ret as usize)
+}
+
+pub fn do_epoll_create1(flags: c_int) -> Result<FileDesc, Error> {
+    info!("epoll_create1: flags: {}", flags);
+
+    let epoll = EpollFile::new()?;
+    let file_ref: Arc<Box<File>> = Arc::new(Box::new(epoll));
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let fd = {
+        let close_on_spawn = flags & libc::EPOLL_CLOEXEC != 0;
+        proc.get_files()
+            .lock()
+            .unwrap()
+            .put(file_ref, close_on_spawn)
+    };
+    Ok(fd)
+}
+
+pub fn do_epoll_ctl(
+    epfd: FileDesc,
+    op: c_int,
+    fd: FileDesc,
+    event: *const libc::epoll_event,
+) -> Result<(), Error> {
+    info!("epoll_ctl: epfd: {}, op: {:?}, fd: {}", epfd, op, fd);
+
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let mut file_table_ref = proc.get_files().lock().unwrap();
+    let mut file_ref = file_table_ref.get(epfd)?;
+    let mut epoll = file_ref.as_epoll()?.inner.lock().unwrap();
+
+    let host_fd = file_table_ref.get(fd)?.as_socket()?.fd() as FileDesc;
+    epoll.ctl(op, host_fd, event)?;
+
+    Ok(())
+}
+
+pub fn do_epoll_wait(
+    epfd: FileDesc,
+    events: &mut [libc::epoll_event],
+    timeout: c_int,
+) -> Result<usize, Error> {
+    info!(
+        "epoll_wait: epfd: {}, len: {:?}, timeout: {}",
+        epfd,
+        events.len(),
+        timeout
+    );
+
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let mut file_ref = proc.get_files().lock().unwrap().get(epfd)?;
+    let mut epoll = file_ref.as_epoll()?.inner.lock().unwrap();
+
+    let count = epoll.wait(events, timeout)?;
+    Ok(count)
+}
+
+/// Safe methods for `libc::fd_set`
+trait FdSetExt {
+    fn set(&mut self, fd: usize);
+    fn clear(&mut self);
+    fn is_set(&mut self, fd: usize) -> bool;
+}
+
+impl FdSetExt for libc::fd_set {
+    fn set(&mut self, fd: usize) {
+        assert!(fd < libc::FD_SETSIZE);
+        unsafe {
+            libc::FD_SET(fd as c_int, self);
+        }
+    }
+
+    fn clear(&mut self) {
+        unsafe {
+            libc::FD_ZERO(self);
+        }
+    }
+
+    fn is_set(&mut self, fd: usize) -> bool {
+        assert!(fd < libc::FD_SETSIZE);
+        unsafe { libc::FD_ISSET(fd as c_int, self) }
+    }
+}
+
+pub struct EpollFile {
+    inner: SgxMutex<EpollFileInner>,
+}
+
+impl EpollFile {
+    pub fn new() -> Result<Self, Error> {
+        Ok(Self {
+            inner: SgxMutex::new(EpollFileInner::new()?),
+        })
+    }
+}
+
+struct EpollFileInner {
+    epoll_fd: c_int,
+}
+
+// FIXME: What if a Linux fd is closed but still in an epoll?
+impl EpollFileInner {
+    /// Create a new Linux epoll file descriptor
+    pub fn new() -> Result<Self, Error> {
+        let ret = try_libc!(libc::ocall::epoll_create1(0));
+        Ok(EpollFileInner { epoll_fd: ret })
+    }
+
+    pub fn ctl(
+        &mut self,
+        op: c_int,
+        host_fd: FileDesc,
+        event: *const libc::epoll_event,
+    ) -> Result<(), Error> {
+        let ret = try_libc!(libc::ocall::epoll_ctl(
+            self.epoll_fd,
+            op,
+            host_fd as c_int,
+            event as *mut _
+        ));
+        Ok(())
+    }
+
+    /// Wait for an I/O event on the epoll.
+    /// Returns the number of file descriptors ready for the requested I/O.
+    pub fn wait(
+        &mut self,
+        events: &mut [libc::epoll_event],
+        timeout: c_int,
+    ) -> Result<usize, Error> {
+        let ret = try_libc!(libc::ocall::epoll_wait(
+            self.epoll_fd,
+            events.as_mut_ptr(),
+            events.len() as c_int,
+            timeout,
+        ));
+        Ok(ret as usize)
+    }
+}
+
+impl Drop for EpollFileInner {
+    fn drop(&mut self) {
+        unsafe {
+            libc::ocall::close(self.epoll_fd);
+        }
+    }
+}
+
+impl File for EpollFile {
+    fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn write(&self, buf: &[u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn write_at(&self, offset: usize, buf: &[u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn seek(&self, pos: SeekFrom) -> Result<off_t, Error> {
+        errno!(ESPIPE, "Epoll does not support seek")
+    }
+
+    fn metadata(&self) -> Result<Metadata, Error> {
+        unimplemented!()
+    }
+
+    fn set_len(&self, len: u64) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_all(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_data(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn read_entry(&self) -> Result<String, Error> {
+        unimplemented!()
+    }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
+}
+
+impl Debug for EpollFile {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        let inner = self.inner.lock().unwrap();
+        f.debug_struct("EpollFile")
+            .field("epoll_fd", &inner.epoll_fd)
+            .finish()
+    }
+}
+
+pub trait AsEpoll {
+    fn as_epoll(&self) -> Result<&EpollFile, Error>;
+}
+
+impl AsEpoll for FileRef {
+    fn as_epoll(&self) -> Result<&EpollFile, Error> {
+        self.as_any()
+            .downcast_ref::<EpollFile>()
+            .ok_or(Error::new(Errno::EBADF, "not a epoll"))
+    }
+}

src/libos/src/fs/mod.rs

@@ -1,31 +1,34 @@
-use {process, std};
 use prelude::*;
 use process::Process;
 use rcore_fs::vfs::{FileType, FsError, INode, Metadata, Timespec};
 use std::sgxfs as fs_impl;
+use {process, std};
 
 use super::*;
 
+pub use self::access::{do_access, do_faccessat, AccessFlags, AccessModes, AT_FDCWD};
 pub use self::file::{File, FileRef, SgxFile, StdinFile, StdoutFile};
 pub use self::file_table::{FileDesc, FileTable};
-pub use self::inode_file::{INodeExt, INodeFile, ROOT_INODE};
 use self::inode_file::OpenOptions;
+pub use self::inode_file::{INodeExt, INodeFile, ROOT_INODE};
+pub use self::io_multiplexing::*;
+use self::null::NullFile;
 pub use self::pipe::Pipe;
-pub use self::access::{AccessModes, AccessFlags, AT_FDCWD, do_access, do_faccessat};
+pub use self::socket_file::{AsSocket, SocketFile};
+pub use self::unix_socket::{AsUnixSocket, UnixSocketFile};
+use std::any::Any;
+use std::mem::uninitialized;
 
+mod access;
 mod file;
 mod file_table;
 mod inode_file;
+mod io_multiplexing;
+mod null;
 mod pipe;
 mod sgx_impl;
-mod access;
-
-// TODO: use the type defined in Rust libc.
-//
-// However, off_t is defined as u64 in the current Rust SGX SDK, which is
-// wrong (see issue https://github.com/baidu/rust-sgx-sdk/issues/46)
-#[allow(non_camel_case_types)]
-pub type off_t = i64;
+mod socket_file;
+mod unix_socket;
 
 pub fn do_open(path: &str, flags: u32, mode: u32) -> Result<FileDesc, Error> {
     let flags = OpenFlags::from_bits_truncate(flags);
@@ -37,28 +40,15 @@ pub fn do_open(path: &str, flags: u32, mode: u32) -> Result<FileDesc, Error> {
     let current_ref = process::get_current();
     let mut proc = current_ref.lock().unwrap();
 
-    let inode = if flags.contains(OpenFlags::CREATE) {
-        let (dir_path, file_name) = split_path(&path);
-        let dir_inode = proc.lookup_inode(dir_path)?;
-        match dir_inode.find(file_name) {
-            Ok(file_inode) => {
-                if flags.contains(OpenFlags::EXCLUSIVE) {
-                    return Err(Error::new(EEXIST, "file exists"));
-                }
-                file_inode
-            }
-            Err(FsError::EntryNotFound) => dir_inode.create(file_name, FileType::File, mode)?,
-            Err(e) => return Err(Error::from(e)),
-        }
-    } else {
-        proc.lookup_inode(&path)?
-    };
-
-    let file_ref: Arc<Box<File>> = Arc::new(Box::new(INodeFile::open(inode, flags.to_options())?));
+    let file = proc.open_file(path, flags, mode)?;
+    let file_ref: Arc<Box<File>> = Arc::new(file);
 
     let fd = {
         let close_on_spawn = flags.contains(OpenFlags::CLOEXEC);
-        proc.get_files().lock().unwrap().put(file_ref, close_on_spawn)
+        proc.get_files()
+            .lock()
+            .unwrap()
+            .put(file_ref, close_on_spawn)
     };
     Ok(fd)
 }
@@ -190,7 +180,7 @@ pub fn do_getdents64(fd: FileDesc, buf: &mut [u8]) -> Result<usize, Error> {
     let file_ref = current_process.get_files().lock().unwrap().get(fd)?;
     let info = file_ref.metadata()?;
     if info.type_ != FileType::Dir {
-        return Err(Error::new(ENOTDIR, ""));
+        return errno!(ENOTDIR, "");
     }
     let mut writer = unsafe { DirentBufWriter::new(buf) };
     loop {
@@ -208,6 +198,7 @@ pub fn do_getdents64(fd: FileDesc, buf: &mut [u8]) -> Result<usize, Error> {
 }
 
 pub fn do_close(fd: FileDesc) -> Result<(), Error> {
+    info!("close: fd: {}", fd);
     let current_ref = process::get_current();
     let current_process = current_ref.lock().unwrap();
     let file_table_ref = current_process.get_files();
@@ -217,6 +208,7 @@ pub fn do_close(fd: FileDesc) -> Result<(), Error> {
 }
 
 pub fn do_pipe2(flags: u32) -> Result<[FileDesc; 2], Error> {
+    info!("pipe2: flags: {:#x}", flags);
     let flags = OpenFlags::from_bits_truncate(flags);
     let current_ref = process::get_current();
     let current = current_ref.lock().unwrap();
@@ -227,6 +219,7 @@ pub fn do_pipe2(flags: u32) -> Result<[FileDesc; 2], Error> {
     let close_on_spawn = flags.contains(OpenFlags::CLOEXEC);
     let reader_fd = file_table.put(Arc::new(Box::new(pipe.reader)), close_on_spawn);
     let writer_fd = file_table.put(Arc::new(Box::new(pipe.writer)), close_on_spawn);
+    info!("pipe2: reader_fd: {}, writer_fd: {}", reader_fd, writer_fd);
     Ok([reader_fd, writer_fd])
 }
 
@@ -281,7 +274,7 @@ pub fn do_chdir(path: &str) -> Result<(), Error> {
     let inode = current_process.lookup_inode(path)?;
     let info = inode.metadata()?;
     if info.type_ != FileType::Dir {
-        return Err(Error::new(ENOTDIR, ""));
+        return errno!(ENOTDIR, "");
     }
     current_process.change_cwd(path);
     Ok(())
@@ -309,7 +302,7 @@ pub fn do_mkdir(path: &str, mode: usize) -> Result<(), Error> {
     let (dir_path, file_name) = split_path(&path);
     let inode = current_process.lookup_inode(dir_path)?;
     if inode.find(file_name).is_ok() {
-        return Err(Error::new(EEXIST, ""));
+        return errno!(EEXIST, "");
     }
     inode.create(file_name, FileType::Dir, mode as u32)?;
     Ok(())
@@ -324,7 +317,7 @@ pub fn do_rmdir(path: &str) -> Result<(), Error> {
     let dir_inode = current_process.lookup_inode(dir_path)?;
     let file_inode = dir_inode.find(file_name)?;
     if file_inode.metadata()?.type_ != FileType::Dir {
-        return Err(Error::new(ENOTDIR, "rmdir on not directory"));
+        return errno!(ENOTDIR, "rmdir on not directory");
     }
     dir_inode.unlink(file_name)?;
     Ok(())
@@ -351,17 +344,93 @@ pub fn do_unlink(path: &str) -> Result<(), Error> {
     let dir_inode = current_process.lookup_inode(dir_path)?;
     let file_inode = dir_inode.find(file_name)?;
     if file_inode.metadata()?.type_ == FileType::Dir {
-        return Err(Error::new(EISDIR, "unlink on directory"));
+        return errno!(EISDIR, "unlink on directory");
     }
     dir_inode.unlink(file_name)?;
     Ok(())
 }
 
+pub fn do_sendfile(
+    out_fd: FileDesc,
+    in_fd: FileDesc,
+    offset: Option<off_t>,
+    count: usize,
+) -> Result<(usize, usize), Error> {
+    // (len, offset)
+    info!(
+        "sendfile: out: {}, in: {}, offset: {:?}, count: {}",
+        out_fd, in_fd, offset, count
+    );
+    let current_ref = process::get_current();
+    let current_process = current_ref.lock().unwrap();
+    let file_table_ref = current_process.get_files();
+    let mut file_table = file_table_ref.lock().unwrap();
+
+    let in_file = file_table.get(in_fd)?;
+    let out_file = file_table.get(out_fd)?;
+    let mut buffer: [u8; 1024 * 11] = unsafe { uninitialized() };
+
+    let mut read_offset = match offset {
+        Some(offset) => offset,
+        None => in_file.seek(SeekFrom::Current(0))?,
+    } as usize;
+
+    // read from specified offset and write new offset back
+    let mut bytes_read = 0;
+    while bytes_read < count {
+        let len = min(buffer.len(), count - bytes_read);
+        let read_len = in_file.read_at(read_offset, &mut buffer[..len])?;
+        if read_len == 0 {
+            break;
+        }
+        bytes_read += read_len;
+        read_offset += read_len;
+        let mut bytes_written = 0;
+        while bytes_written < read_len {
+            let write_len = out_file.write(&buffer[bytes_written..])?;
+            if write_len == 0 {
+                return errno!(EBADF, "sendfile write return 0");
+            }
+            bytes_written += write_len;
+        }
+    }
+
+    if offset.is_none() {
+        in_file.seek(SeekFrom::Current(bytes_read as i64))?;
+    }
+    Ok((bytes_read, read_offset))
+}
+
 extern "C" {
     fn ocall_sync() -> sgx_status_t;
 }
 
 impl Process {
+    /// Open a file on the process. But DO NOT add it to file table.
+    pub fn open_file(&self, path: &str, flags: OpenFlags, mode: u32) -> Result<Box<File>, Error> {
+        if path == "/dev/null" {
+            return Ok(Box::new(NullFile));
+        }
+        let inode = if flags.contains(OpenFlags::CREATE) {
+            let (dir_path, file_name) = split_path(&path);
+            let dir_inode = self.lookup_inode(dir_path)?;
+            match dir_inode.find(file_name) {
+                Ok(file_inode) => {
+                    if flags.contains(OpenFlags::EXCLUSIVE) {
+                        return errno!(EEXIST, "file exists");
+                    }
+                    file_inode
+                }
+                Err(FsError::EntryNotFound) => dir_inode.create(file_name, FileType::File, mode)?,
+                Err(e) => return Err(Error::from(e)),
+            }
+        } else {
+            self.lookup_inode(&path)?
+        };
+        Ok(Box::new(INodeFile::open(inode, flags.to_options())?))
+    }
+
+    /// Lookup INode from the cwd of the process
     pub fn lookup_inode(&self, path: &str) -> Result<Arc<INode>, Error> {
         debug!("lookup_inode: cwd: {:?}, path: {:?}", self.get_cwd(), path);
         if path.len() > 0 && path.as_bytes()[0] == b'/' {
@@ -405,6 +474,8 @@ bitflags! {
         const TRUNCATE = 1 << 9;
         /// append on each write
         const APPEND = 1 << 10;
+        /// non block
+        const NONBLOCK = 1 << 11;
         /// close on exec
         const CLOEXEC = 1 << 19;
     }
@@ -428,7 +499,6 @@ impl OpenFlags {
     }
 }
 
-#[derive(Debug)]
 #[repr(packed)] // Don't use 'C'. Or its size will align up to 8 bytes.
 pub struct LinuxDirent64 {
     /// Inode number
@@ -615,7 +685,6 @@ pub unsafe fn write_cstr(ptr: *mut u8, s: &str) {
     ptr.add(s.len()).write(0);
 }
 
-
 #[derive(Debug)]
 pub enum FcntlCmd {
     /// Duplicate the file descriptor fd using the lowest-numbered available
@@ -631,35 +700,26 @@ pub enum FcntlCmd {
     /// Get the file status flags
     GetFl(),
     /// Set the file status flags
-    SetFl(OpenFlags),
+    SetFl(u32),
 }
 
-pub const F_DUPFD : u32             = 0;
-pub const F_GETFD : u32             = 1;
-pub const F_SETFD : u32             = 2;
-pub const F_GETFL : u32             = 3;
-pub const F_SETFL : u32             = 4;
-pub const F_DUPFD_CLOEXEC : u32     = 1030;
-
-pub const FD_CLOEXEC : u32          = 1;
-
 impl FcntlCmd {
     #[deny(unreachable_patterns)]
     pub fn from_raw(cmd: u32, arg: u64) -> Result<FcntlCmd, Error> {
-        Ok(match cmd {
-            F_DUPFD => FcntlCmd::DupFd(arg as FileDesc),
-            F_DUPFD_CLOEXEC => FcntlCmd::DupFdCloexec(arg as FileDesc),
-            F_GETFD => FcntlCmd::GetFd(),
-            F_SETFD => FcntlCmd::SetFd(arg as u32),
-            F_GETFL => FcntlCmd::GetFl(),
-            F_SETFL => FcntlCmd::SetFl(OpenFlags::from_bits_truncate(arg as u32)),
+        Ok(match cmd as c_int {
+            libc::F_DUPFD => FcntlCmd::DupFd(arg as FileDesc),
+            libc::F_DUPFD_CLOEXEC => FcntlCmd::DupFdCloexec(arg as FileDesc),
+            libc::F_GETFD => FcntlCmd::GetFd(),
+            libc::F_SETFD => FcntlCmd::SetFd(arg as u32),
+            libc::F_GETFL => FcntlCmd::GetFl(),
+            libc::F_SETFL => FcntlCmd::SetFl(arg as u32),
             _ => return errno!(EINVAL, "invalid command"),
         })
     }
 }
 
 pub fn do_fcntl(fd: FileDesc, cmd: &FcntlCmd) -> Result<isize, Error> {
-    info!("do_fcntl: {:?}, {:?}", &fd, cmd);
+    info!("fcntl: fd: {:?}, cmd: {:?}", &fd, cmd);
     let current_ref = process::get_current();
     let mut current = current_ref.lock().unwrap();
     let files_ref = current.get_files();
@@ -668,35 +728,67 @@ pub fn do_fcntl(fd: FileDesc, cmd: &FcntlCmd) -> Result<isize, Error> {
         FcntlCmd::DupFd(min_fd) => {
             let dup_fd = files.dup(fd, *min_fd, false)?;
             dup_fd as isize
-        },
+        }
         FcntlCmd::DupFdCloexec(min_fd) => {
             let dup_fd = files.dup(fd, *min_fd, true)?;
             dup_fd as isize
-        },
+        }
         FcntlCmd::GetFd() => {
             let entry = files.get_entry(fd)?;
             let fd_flags = if entry.is_close_on_spawn() {
-                FD_CLOEXEC
+                libc::FD_CLOEXEC
             } else {
                 0
             };
             fd_flags as isize
-        },
+        }
         FcntlCmd::SetFd(fd_flags) => {
             let entry = files.get_entry_mut(fd)?;
-            entry.set_close_on_spawn((fd_flags & FD_CLOEXEC) != 0);
+            entry.set_close_on_spawn((fd_flags & libc::FD_CLOEXEC as u32) != 0);
             0
-        },
+        }
         FcntlCmd::GetFl() => {
-            unimplemented!();
-        },
+            let file = files.get(fd)?;
+            if let Ok(socket) = file.as_socket() {
+                let ret = try_libc!(libc::ocall::fcntl_arg0(socket.fd(), libc::F_GETFL));
+                ret as isize
+            } else {
+                warn!("fcntl.getfl is unimplemented");
+                0
+            }
+        }
         FcntlCmd::SetFl(flags) => {
-            unimplemented!();
-        },
+            let file = files.get(fd)?;
+            if let Ok(socket) = file.as_socket() {
+                let ret = try_libc!(libc::ocall::fcntl_arg1(
+                    socket.fd(),
+                    libc::F_SETFL,
+                    *flags as c_int
+                ));
+                ret as isize
+            } else {
+                warn!("fcntl.setfl is unimplemented");
+                0
+            }
+        }
     })
 }
 
 pub fn do_readlink(path: &str, buf: &mut [u8]) -> Result<usize, Error> {
+    info!("readlink: path: {:?}", path);
+    match path {
+        "/proc/self/exe" => {
+            // get cwd
+            let current_ref = process::get_current();
+            let current = current_ref.lock().unwrap();
+            let cwd = current.get_cwd();
+            let len = cwd.len().min(buf.len());
+            buf[0..len].copy_from_slice(&cwd.as_bytes()[0..len]);
+            Ok(0)
+        }
+        _ => {
             // TODO: support symbolic links
             errno!(EINVAL, "not a symbolic link")
+        }
+    }
 }

src/libos/src/fs/null.rs

@@ -0,0 +1,58 @@
+use super::*;
+
+#[derive(Debug)]
+pub struct NullFile;
+
+impl File for NullFile {
+    fn read(&self, _buf: &mut [u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn write(&self, _buf: &[u8]) -> Result<usize, Error> {
+        Ok(0)
+    }
+
+    fn read_at(&self, _offset: usize, _buf: &mut [u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn write_at(&self, _offset: usize, _buf: &[u8]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
+        unimplemented!()
+    }
+
+    fn seek(&self, pos: SeekFrom) -> Result<off_t, Error> {
+        unimplemented!()
+    }
+
+    fn metadata(&self) -> Result<Metadata, Error> {
+        unimplemented!()
+    }
+
+    fn set_len(&self, len: u64) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_all(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_data(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn read_entry(&self) -> Result<String, Error> {
+        unimplemented!()
+    }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
+}

src/libos/src/fs/pipe.rs

@@ -37,10 +37,7 @@ impl File for PipeReader {
     }
 
     fn write(&self, buf: &[u8]) -> Result<usize, Error> {
-        Err(Error::new(
-            Errno::EBADF,
-            "PipeReader does not support write",
-        ))
+        errno!(EBADF, "PipeReader does not support write")
     }
 
     fn read_at(&self, offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
@@ -76,14 +73,11 @@ impl File for PipeReader {
     }
 
     fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
-        Err(Error::new(
-            Errno::EBADF,
-            "PipeReader does not support write",
-        ))
+        errno!(EBADF, "PipeReader does not support write")
     }
 
     fn seek(&self, pos: SeekFrom) -> Result<off_t, Error> {
-        Err(Error::new(Errno::ESPIPE, "Pipe does not support seek"))
+        errno!(ESPIPE, "Pipe does not support seek")
     }
 
     fn metadata(&self) -> Result<Metadata, Error> {
@@ -105,6 +99,10 @@ impl File for PipeReader {
     fn read_entry(&self) -> Result<String, Error> {
         unimplemented!()
     }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
 }
 
 unsafe impl Send for PipeReader {}
@@ -117,7 +115,7 @@ pub struct PipeWriter {
 
 impl File for PipeWriter {
     fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
-        Err(Error::new(Errno::EBADF, "PipeWriter does not support read"))
+        errno!(EBADF, "PipeWriter does not support read")
     }
 
     fn write(&self, buf: &[u8]) -> Result<usize, Error> {
@@ -133,7 +131,7 @@ impl File for PipeWriter {
     }
 
     fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
-        Err(Error::new(Errno::EBADF, "PipeWriter does not support read"))
+        errno!(EBADF, "PipeWriter does not support read")
     }
 
     fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
@@ -161,7 +159,7 @@ impl File for PipeWriter {
     }
 
     fn seek(&self, seek_pos: SeekFrom) -> Result<off_t, Error> {
-        Err(Error::new(Errno::ESPIPE, "Pipe does not support seek"))
+        errno!(ESPIPE, "Pipe does not support seek")
     }
 
     fn metadata(&self) -> Result<Metadata, Error> {
@@ -183,6 +181,10 @@ impl File for PipeWriter {
     fn read_entry(&self) -> Result<String, Error> {
         unimplemented!()
     }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
 }
 
 unsafe impl Send for PipeWriter {}

src/libos/src/fs/sgx_impl.rs

@@ -2,14 +2,16 @@ use rcore_fs::dev::TimeProvider;
 use rcore_fs::vfs::Timespec;
 use rcore_fs_sefs::dev::*;
 use std::boxed::Box;
+use std::collections::BTreeMap;
 use std::io::{Read, Seek, SeekFrom, Write};
 use std::path::{Path, PathBuf};
-use std::sgxfs::{OpenOptions, remove, SgxFile};
-use std::sync::SgxMutex as Mutex;
+use std::sgxfs::{remove, OpenOptions, SgxFile};
+use std::sync::{Arc, SgxMutex as Mutex};
 use std::time::{SystemTime, UNIX_EPOCH};
 
 pub struct SgxStorage {
     path: PathBuf,
+    file_cache: Mutex<BTreeMap<usize, LockedFile>>,
 }
 
 impl SgxStorage {
@@ -17,13 +19,37 @@ impl SgxStorage {
         //        assert!(path.as_ref().is_dir());
         SgxStorage {
             path: path.as_ref().to_path_buf(),
+            file_cache: Mutex::new(BTreeMap::new()),
         }
     }
+    /// Get file by `file_id`.
+    /// It lookups cache first, if miss, then call `open_fn` to open one,
+    /// and add it to cache before return.
+    #[cfg(feature = "sgx_file_cache")]
+    fn get(&self, file_id: usize, open_fn: impl FnOnce(&Self) -> LockedFile) -> LockedFile {
+        // query cache
+        let mut caches = self.file_cache.lock().unwrap();
+        if let Some(locked_file) = caches.get(&file_id) {
+            // hit, return
+            return locked_file.clone();
+        }
+        // miss, open one
+        let locked_file = open_fn(self);
+        // add to cache
+        caches.insert(file_id, locked_file.clone());
+        locked_file
+    }
+    /// Get file by `file_id` without cache.
+    #[cfg(not(feature = "sgx_file_cache"))]
+    fn get(&self, file_id: usize, open_fn: impl FnOnce(&Self) -> LockedFile) -> LockedFile {
+        open_fn(self)
+    }
 }
 
 impl Storage for SgxStorage {
     fn open(&self, file_id: usize) -> DevResult<Box<File>> {
-        let mut path = self.path.to_path_buf();
+        let locked_file = self.get(file_id, |this| {
+            let mut path = this.path.to_path_buf();
             path.push(format!("{}", file_id));
             // TODO: key
             let key = [0u8; 16];
@@ -32,11 +58,14 @@ impl Storage for SgxStorage {
                 .update(true)
                 .open_ex(path, &key)
                 .expect("failed to open SgxFile");
-        Ok(Box::new(LockedFile(Mutex::new(file))))
+            LockedFile(Arc::new(Mutex::new(file)))
+        });
+        Ok(Box::new(locked_file))
     }
 
     fn create(&self, file_id: usize) -> DevResult<Box<File>> {
-        let mut path = self.path.to_path_buf();
+        let locked_file = self.get(file_id, |this| {
+            let mut path = this.path.to_path_buf();
             path.push(format!("{}", file_id));
             // TODO: key
             let key = [0u8; 16];
@@ -45,18 +74,24 @@ impl Storage for SgxStorage {
                 .update(true)
                 .open_ex(path, &key)
                 .expect("failed to create SgxFile");
-        Ok(Box::new(LockedFile(Mutex::new(file))))
+            LockedFile(Arc::new(Mutex::new(file)))
+        });
+        Ok(Box::new(locked_file))
     }
 
     fn remove(&self, file_id: usize) -> DevResult<()> {
         let mut path = self.path.to_path_buf();
         path.push(format!("{}", file_id));
         remove(path).expect("failed to remove SgxFile");
+        // remove from cache
+        let mut caches = self.file_cache.lock().unwrap();
+        caches.remove(&file_id);
         Ok(())
     }
 }
 
-pub struct LockedFile(Mutex<SgxFile>);
+#[derive(Clone)]
+pub struct LockedFile(Arc<Mutex<SgxFile>>);
 
 // `sgx_tstd::sgxfs::SgxFile` not impl Send ...
 unsafe impl Send for LockedFile {}
@@ -64,6 +99,9 @@ unsafe impl Sync for LockedFile {}
 
 impl File for LockedFile {
     fn read_at(&self, buf: &mut [u8], offset: usize) -> DevResult<usize> {
+        if buf.len() == 0 {
+            return Ok(0);
+        }
         let mut file = self.0.lock().unwrap();
         let offset = offset as u64;
         file.seek(SeekFrom::Start(offset))
@@ -73,7 +111,24 @@ impl File for LockedFile {
     }
 
     fn write_at(&self, buf: &[u8], offset: usize) -> DevResult<usize> {
+        if buf.len() == 0 {
+            return Ok(0);
+        }
         let mut file = self.0.lock().unwrap();
+
+        // SgxFile do not support seek a position after the end.
+        // So check the size and padding zeros if necessary.
+        let file_size = file.seek(SeekFrom::End(0)).expect("failed to tell SgxFile") as usize;
+        if file_size < offset {
+            static ZEROS: [u8; 0x1000] = [0; 0x1000];
+            let mut rest_len = offset - file_size;
+            while rest_len != 0 {
+                let l = rest_len.min(0x1000);
+                let len = file.write(&ZEROS[..l]).expect("failed to write SgxFile");
+                rest_len -= len;
+            }
+        }
+
         let offset = offset as u64;
         file.seek(SeekFrom::Start(offset))
             .expect("failed to seek SgxFile");

src/libos/src/fs/socket_file.rs

@@ -0,0 +1,152 @@
+use super::*;
+use std::any::Any;
+
+/// Native Linux socket
+#[derive(Debug)]
+pub struct SocketFile {
+    fd: c_int,
+}
+
+impl SocketFile {
+    pub fn new(domain: c_int, socket_type: c_int, protocol: c_int) -> Result<Self, Error> {
+        let ret = try_libc!(libc::ocall::socket(domain, socket_type, protocol));
+        Ok(SocketFile { fd: ret })
+    }
+
+    pub fn accept(
+        &self,
+        addr: *mut libc::sockaddr,
+        addr_len: *mut libc::socklen_t,
+        flags: c_int,
+    ) -> Result<Self, Error> {
+        let ret = try_libc!(libc::ocall::accept4(self.fd, addr, addr_len, flags));
+        Ok(SocketFile { fd: ret })
+    }
+
+    pub fn fd(&self) -> c_int {
+        self.fd
+    }
+}
+
+impl Drop for SocketFile {
+    fn drop(&mut self) {
+        let ret = unsafe { libc::ocall::close(self.fd) };
+        if ret < 0 {
+            let errno = unsafe { libc::errno() };
+            warn!(
+                "socket (host fd: {}) close failed: errno = {}",
+                self.fd, errno
+            );
+        }
+    }
+}
+
+impl File for SocketFile {
+    fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let ret = try_libc!(libc::ocall::read(
+            self.fd,
+            buf.as_mut_ptr() as *mut c_void,
+            buf.len()
+        ));
+        Ok(ret as usize)
+    }
+
+    fn write(&self, buf: &[u8]) -> Result<usize, Error> {
+        let ret = try_libc!(libc::ocall::write(
+            self.fd,
+            buf.as_ptr() as *const c_void,
+            buf.len()
+        ));
+        Ok(ret as usize)
+    }
+
+    fn read_at(&self, _offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
+        self.read(buf)
+    }
+
+    fn write_at(&self, _offset: usize, buf: &[u8]) -> Result<usize, Error> {
+        self.write(buf)
+    }
+
+    fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
+        let mut total_len = 0;
+        for buf in bufs {
+            match self.read(buf) {
+                Ok(len) => {
+                    total_len += len;
+                }
+                Err(_) if total_len != 0 => break,
+                Err(e) => return Err(e.into()),
+            }
+        }
+        Ok(total_len)
+    }
+
+    fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
+        let mut total_len = 0;
+        for buf in bufs {
+            match self.write(buf) {
+                Ok(len) => {
+                    total_len += len;
+                }
+                Err(_) if total_len != 0 => break,
+                Err(e) => return Err(e.into()),
+            }
+        }
+        Ok(total_len)
+    }
+
+    fn seek(&self, pos: SeekFrom) -> Result<off_t, Error> {
+        errno!(ESPIPE, "Socket does not support seek")
+    }
+
+    fn metadata(&self) -> Result<Metadata, Error> {
+        Ok(Metadata {
+            dev: 0,
+            inode: 0,
+            size: 0,
+            blk_size: 0,
+            blocks: 0,
+            atime: Timespec { sec: 0, nsec: 0 },
+            mtime: Timespec { sec: 0, nsec: 0 },
+            ctime: Timespec { sec: 0, nsec: 0 },
+            type_: FileType::Socket,
+            mode: 0,
+            nlinks: 0,
+            uid: 0,
+            gid: 0,
+        })
+    }
+
+    fn set_len(&self, len: u64) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_all(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_data(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn read_entry(&self) -> Result<String, Error> {
+        unimplemented!()
+    }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
+}
+
+pub trait AsSocket {
+    fn as_socket(&self) -> Result<&SocketFile, Error>;
+}
+
+impl AsSocket for FileRef {
+    fn as_socket(&self) -> Result<&SocketFile, Error> {
+        self.as_any()
+            .downcast_ref::<SocketFile>()
+            .ok_or(Error::new(Errno::EBADF, "not a socket"))
+    }
+}

src/libos/src/fs/unix_socket.rs

@@ -0,0 +1,350 @@
+use super::*;
+use alloc::prelude::ToString;
+use std::collections::btree_map::BTreeMap;
+use std::fmt;
+use std::sync::atomic::spin_loop_hint;
+use std::sync::SgxMutex as Mutex;
+use util::ring_buf::{RingBuf, RingBufReader, RingBufWriter};
+
+pub struct UnixSocketFile {
+    inner: Mutex<UnixSocket>,
+}
+
+impl File for UnixSocketFile {
+    fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.read(buf)
+    }
+
+    fn write(&self, buf: &[u8]) -> Result<usize, Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.write(buf)
+    }
+
+    fn read_at(&self, _offset: usize, buf: &mut [u8]) -> Result<usize, Error> {
+        self.read(buf)
+    }
+
+    fn write_at(&self, _offset: usize, buf: &[u8]) -> Result<usize, Error> {
+        self.write(buf)
+    }
+
+    fn readv(&self, bufs: &mut [&mut [u8]]) -> Result<usize, Error> {
+        let mut inner = self.inner.lock().unwrap();
+        let mut total_len = 0;
+        for buf in bufs {
+            match inner.read(buf) {
+                Ok(len) => {
+                    total_len += len;
+                }
+                Err(_) if total_len != 0 => break,
+                Err(e) => return Err(e.into()),
+            }
+        }
+        Ok(total_len)
+    }
+
+    fn writev(&self, bufs: &[&[u8]]) -> Result<usize, Error> {
+        let mut inner = self.inner.lock().unwrap();
+        let mut total_len = 0;
+        for buf in bufs {
+            match inner.write(buf) {
+                Ok(len) => {
+                    total_len += len;
+                }
+                Err(_) if total_len != 0 => break,
+                Err(e) => return Err(e.into()),
+            }
+        }
+        Ok(total_len)
+    }
+
+    fn seek(&self, pos: SeekFrom) -> Result<off_t, Error> {
+        errno!(ESPIPE, "UnixSocket does not support seek")
+    }
+
+    fn metadata(&self) -> Result<Metadata, Error> {
+        Ok(Metadata {
+            dev: 0,
+            inode: 0,
+            size: 0,
+            blk_size: 0,
+            blocks: 0,
+            atime: Timespec { sec: 0, nsec: 0 },
+            mtime: Timespec { sec: 0, nsec: 0 },
+            ctime: Timespec { sec: 0, nsec: 0 },
+            type_: FileType::Socket,
+            mode: 0,
+            nlinks: 0,
+            uid: 0,
+            gid: 0,
+        })
+    }
+
+    fn set_len(&self, len: u64) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_all(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn sync_data(&self) -> Result<(), Error> {
+        unimplemented!()
+    }
+
+    fn read_entry(&self) -> Result<String, Error> {
+        unimplemented!()
+    }
+
+    fn as_any(&self) -> &Any {
+        self
+    }
+}
+
+impl UnixSocketFile {
+    pub fn new(socket_type: c_int, protocol: c_int) -> Result<Self, Error> {
+        let inner = UnixSocket::new(socket_type, protocol)?;
+        Ok(UnixSocketFile {
+            inner: Mutex::new(inner),
+        })
+    }
+
+    pub fn bind(&self, path: impl AsRef<str>) -> Result<(), Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.bind(path)
+    }
+
+    pub fn listen(&self) -> Result<(), Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.listen()
+    }
+
+    pub fn accept(&self) -> Result<UnixSocketFile, Error> {
+        let mut inner = self.inner.lock().unwrap();
+        let new_socket = inner.accept()?;
+        Ok(UnixSocketFile {
+            inner: Mutex::new(new_socket),
+        })
+    }
+
+    pub fn connect(&self, path: impl AsRef<str>) -> Result<(), Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.connect(path)
+    }
+
+    pub fn poll(&self) -> Result<(bool, bool, bool), Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.poll()
+    }
+
+    pub fn ioctl(&self, cmd: c_int, argp: *mut c_int) -> Result<(), Error> {
+        let mut inner = self.inner.lock().unwrap();
+        inner.ioctl(cmd, argp)
+    }
+}
+
+impl Debug for UnixSocketFile {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "UnixSocketFile {{ ... }}")
+    }
+}
+
+pub trait AsUnixSocket {
+    fn as_unix_socket(&self) -> Result<&UnixSocketFile, Error>;
+}
+
+impl AsUnixSocket for FileRef {
+    fn as_unix_socket(&self) -> Result<&UnixSocketFile, Error> {
+        self.as_any()
+            .downcast_ref::<UnixSocketFile>()
+            .ok_or(Error::new(Errno::EBADF, "not a unix socket"))
+    }
+}
+
+pub struct UnixSocket {
+    obj: Option<Arc<UnixSocketObject>>,
+    status: Status,
+}
+
+enum Status {
+    None,
+    Listening,
+    Connected(Channel),
+}
+
+impl UnixSocket {
+    /// C/S 1: Create a new unix socket
+    pub fn new(socket_type: c_int, protocol: c_int) -> Result<Self, Error> {
+        if socket_type == libc::SOCK_STREAM && protocol == 0 {
+            Ok(UnixSocket {
+                obj: None,
+                status: Status::None,
+            })
+        } else {
+            errno!(ENOSYS, "unimplemented unix socket type")
+        }
+    }
+
+    /// Server 2: Bind the socket to a file system path
+    pub fn bind(&mut self, path: impl AsRef<str>) -> Result<(), Error> {
+        // TODO: check permission
+        if self.obj.is_some() {
+            return errno!(EINVAL, "The socket is already bound to an address.");
+        }
+        self.obj = Some(UnixSocketObject::create(path)?);
+        Ok(())
+    }
+
+    /// Server 3: Listen to a socket
+    pub fn listen(&mut self) -> Result<(), Error> {
+        self.status = Status::Listening;
+        Ok(())
+    }
+
+    /// Server 4: Accept a connection on listening.
+    pub fn accept(&mut self) -> Result<UnixSocket, Error> {
+        match self.status {
+            Status::Listening => {}
+            _ => return errno!(EINVAL, "unix socket is not listening"),
+        };
+        // FIXME: Block. Now spin loop.
+        let socket = loop {
+            if let Some(socket) = self.obj.as_mut().unwrap().pop() {
+                break socket;
+            }
+            spin_loop_hint();
+        };
+        Ok(socket)
+    }
+
+    /// Client 2: Connect to a path
+    pub fn connect(&mut self, path: impl AsRef<str>) -> Result<(), Error> {
+        if let Status::Listening = self.status {
+            return errno!(EINVAL, "unix socket is listening?");
+        }
+        let obj =
+            UnixSocketObject::get(path).ok_or(Error::new(EINVAL, "unix socket path not found"))?;
+        let (channel1, channel2) = Channel::new_pair();
+        self.status = Status::Connected(channel1);
+        obj.push(UnixSocket {
+            obj: Some(obj.clone()),
+            status: Status::Connected(channel2),
+        });
+        Ok(())
+    }
+
+    pub fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
+        self.channel()?.reader.read(buf)
+    }
+
+    pub fn write(&self, buf: &[u8]) -> Result<usize, Error> {
+        self.channel()?.writer.write(buf)
+    }
+
+    pub fn poll(&self) -> Result<(bool, bool, bool), Error> {
+        // (read, write, error)
+        let channel = self.channel()?;
+        let r = channel.reader.can_read();
+        let w = channel.writer.can_write();
+        Ok((r, w, false))
+    }
+
+    pub fn ioctl(&self, cmd: c_int, argp: *mut c_int) -> Result<(), Error> {
+        const FIONREAD: c_int = 0x541B; // Get the number of bytes to read
+        if cmd == FIONREAD {
+            let bytes_to_read = self.channel()?.reader.bytes_to_read();
+            unsafe {
+                argp.write(bytes_to_read as c_int);
+            }
+            Ok(())
+        } else {
+            warn!("ioctl for unix socket is unimplemented");
+            errno!(ENOSYS, "ioctl for unix socket is unimplemented")
+        }
+    }
+
+    fn channel(&self) -> Result<&Channel, Error> {
+        if let Status::Connected(channel) = &self.status {
+            Ok(channel)
+        } else {
+            errno!(EBADF, "UnixSocket is not connected")
+        }
+    }
+}
+
+impl Drop for UnixSocket {
+    fn drop(&mut self) {
+        if let Status::Listening = self.status {
+            let path = &self.obj.as_ref().unwrap().path;
+            UnixSocketObject::remove(path);
+        }
+    }
+}
+
+pub struct UnixSocketObject {
+    path: String,
+    accepted_sockets: Mutex<VecDeque<UnixSocket>>,
+}
+
+impl UnixSocketObject {
+    fn push(&self, unix_socket: UnixSocket) {
+        let mut queue = self.accepted_sockets.lock().unwrap();
+        queue.push_back(unix_socket);
+    }
+    fn pop(&self) -> Option<UnixSocket> {
+        let mut queue = self.accepted_sockets.lock().unwrap();
+        queue.pop_front()
+    }
+    fn get(path: impl AsRef<str>) -> Option<Arc<Self>> {
+        let mut paths = UNIX_SOCKET_OBJS.lock().unwrap();
+        paths.get(path.as_ref()).map(|obj| obj.clone())
+    }
+    fn create(path: impl AsRef<str>) -> Result<Arc<Self>, Error> {
+        let mut paths = UNIX_SOCKET_OBJS.lock().unwrap();
+        if paths.contains_key(path.as_ref()) {
+            return errno!(EADDRINUSE, "unix socket path already exists");
+        }
+        let obj = Arc::new(UnixSocketObject {
+            path: path.as_ref().to_string(),
+            accepted_sockets: Mutex::new(VecDeque::new()),
+        });
+        paths.insert(path.as_ref().to_string(), obj.clone());
+        Ok(obj)
+    }
+    fn remove(path: impl AsRef<str>) {
+        let mut paths = UNIX_SOCKET_OBJS.lock().unwrap();
+        paths.remove(path.as_ref());
+    }
+}
+
+struct Channel {
+    reader: RingBufReader,
+    writer: RingBufWriter,
+}
+
+unsafe impl Send for Channel {}
+unsafe impl Sync for Channel {}
+
+impl Channel {
+    fn new_pair() -> (Channel, Channel) {
+        let buf1 = RingBuf::new(DEFAULT_BUF_SIZE);
+        let buf2 = RingBuf::new(DEFAULT_BUF_SIZE);
+        let channel1 = Channel {
+            reader: buf1.reader,
+            writer: buf2.writer,
+        };
+        let channel2 = Channel {
+            reader: buf2.reader,
+            writer: buf1.writer,
+        };
+        (channel1, channel2)
+    }
+}
+
+pub const DEFAULT_BUF_SIZE: usize = 1 * 1024 * 1024;
+
+lazy_static! {
+    static ref UNIX_SOCKET_OBJS: Mutex<BTreeMap<String, Arc<UnixSocketObject>>> =
+        Mutex::new(BTreeMap::new());
+}

src/libos/src/lib.rs

@@ -36,12 +36,12 @@ mod prelude;
 mod entry;
 mod errno;
 mod fs;
+mod misc;
 mod process;
 mod syscall;
 mod time;
 mod util;
 mod vm;
-mod misc;
 
 use prelude::*;
 

src/libos/src/misc/mod.rs

@@ -1,7 +1,7 @@
 use super::*;
 
-mod uname;
 mod rlimit;
+mod uname;
 
-pub use self::uname::{utsname_t, do_uname};
-pub use self::rlimit::{rlimit_t, resource_t, ResourceLimits, ResourceLimitsRef, do_prlimit};
+pub use self::rlimit::{do_prlimit, resource_t, rlimit_t, ResourceLimits, ResourceLimitsRef};
+pub use self::uname::{do_uname, utsname_t};

src/libos/src/misc/rlimit.rs

@@ -1,5 +1,5 @@
 use super::*;
-use process::{pid_t};
+use process::pid_t;
 
 #[derive(Debug, Copy, Clone)]
 pub struct ResourceLimits {
@@ -21,13 +21,12 @@ impl Default for ResourceLimits {
     fn default() -> ResourceLimits {
         // TODO: set appropriate limits for resources
         let mut rlimits = ResourceLimits {
-            rlimits: [ Default::default(); RLIMIT_COUNT ],
+            rlimits: [Default::default(); RLIMIT_COUNT],
         };
         rlimits
     }
 }
 
-
 #[derive(Debug, Copy, Clone)]
 #[allow(non_camel_case_types)]
 pub struct rlimit_t {
@@ -44,7 +43,6 @@ impl Default for rlimit_t {
     }
 }
 
-
 #[derive(Debug, Copy, Clone)]
 #[allow(non_camel_case_types)]
 pub enum resource_t {
@@ -89,7 +87,6 @@ impl resource_t {
     }
 }
 
-
 pub fn do_prlimit(
     pid: pid_t,
     resource: resource_t,
@@ -98,8 +95,7 @@ pub fn do_prlimit(
 ) -> Result<(), Error> {
     let process_ref = if pid == 0 {
         process::get_current()
-    }
-    else {
+    } else {
         process::get(pid)?
     };
     let mut process = process_ref.lock().unwrap();

src/libos/src/misc/uname.rs

@@ -34,7 +34,7 @@ pub fn do_uname(name: &mut utsname_t) -> Result<(), Error> {
 }
 
 lazy_static! {
-    static ref SYSNAME : CString = CString::new("Occlum").unwrap();
+    static ref SYSNAME: CString = CString::new("Occlum").unwrap();
     static ref NODENAME: CString = CString::new("occlum-node").unwrap();
     static ref RELEASE: CString = CString::new("0.1").unwrap();
     static ref VERSION: CString = CString::new("0.1").unwrap();
@@ -43,9 +43,8 @@ lazy_static! {
 }
 
 fn copy_from_cstr_to_u8_array(src: &CStr, dst: &mut [u8]) {
-    let src : &[u8] = src.to_bytes_with_nul();
+    let src: &[u8] = src.to_bytes_with_nul();
     let len = min(dst.len() - 1, src.len());
     dst[..len].copy_from_slice(&src[..len]);
     dst[len] = 0;
 }
-

src/libos/src/prelude.rs

@@ -1,4 +1,5 @@
 pub use sgx_trts::libc;
+pub use sgx_trts::libc::off_t;
 pub use sgx_types::*;
 use std;
 
@@ -13,6 +14,7 @@ pub use std::sync::{
 //pub use std::borrow::BorrowMut;
 pub use std::borrow::ToOwned;
 pub use std::boxed::Box;
+pub use std::cmp::{max, min};
 pub use std::cmp::{Ordering, PartialOrd};
 pub use std::collections::{HashMap, VecDeque};
 pub use std::fmt::{Debug, Display};
@@ -21,14 +23,11 @@ pub use std::iter::Iterator;
 pub use std::rc::Rc;
 pub use std::string::String;
 pub use std::vec::Vec;
-pub use std::cmp::{min, max};
 
 pub use errno::Errno;
 pub use errno::Errno::*;
 pub use errno::Error;
 
-pub use fs::off_t;
-
 macro_rules! debug_trace {
     () => {
         println!("> Line = {}, File = {}", line!(), file!())
@@ -37,7 +36,7 @@ macro_rules! debug_trace {
 
 macro_rules! errno {
     ($errno: ident, $msg: expr) => {{
-        println!(
+        error!(
             "ERROR: {} ({}, line {} in file {})",
             $errno,
             $msg,
@@ -48,6 +47,25 @@ macro_rules! errno {
     }};
 }
 
+// return Err(errno) if libc return -1
+macro_rules! try_libc {
+    ($ret: expr) => {{
+        let ret = unsafe { $ret };
+        if ret == -1 {
+            let errno = unsafe { libc::errno() };
+            // println will cause libc ocall and overwrite errno
+            error!(
+                "ERROR from libc: {} (line {} in file {})",
+                errno,
+                line!(),
+                file!()
+            );
+            return Err(Error::new(Errno::from_errno(errno), "libc error"));
+        }
+        ret
+    }};
+}
+
 pub fn align_up(addr: usize, align: usize) -> usize {
     (addr + (align - 1)) / align * align
 }

src/libos/src/process/arch_prctl.rs

@@ -22,22 +22,28 @@ impl ArchPrctlCode {
 }
 
 pub fn do_arch_prctl(code: ArchPrctlCode, addr: *mut usize) -> Result<(), Error> {
-    info!("do_arch_prctl: code: {:?}, addr: {:#o}", code, addr as usize);
+    info!(
+        "do_arch_prctl: code: {:?}, addr: {:#o}",
+        code, addr as usize
+    );
     match code {
         ArchPrctlCode::ARCH_SET_FS => {
             let current_ref = get_current();
             let mut current = current_ref.lock().unwrap();
             let task = &mut current.task;
             task.user_fsbase_addr = addr as usize;
-       },
+        }
         ArchPrctlCode::ARCH_GET_FS => {
             let current_ref = get_current();
             let current = current_ref.lock().unwrap();
             let task = &current.task;
-            unsafe { *addr = task.user_fsbase_addr; }
-       },
-       ArchPrctlCode::ARCH_SET_GS | ArchPrctlCode::ARCH_GET_GS
-           => return errno!(EINVAL, "GS cannot be accessed from the user space"),
+            unsafe {
+                *addr = task.user_fsbase_addr;
+            }
+        }
+        ArchPrctlCode::ARCH_SET_GS | ArchPrctlCode::ARCH_GET_GS => {
+            return errno!(EINVAL, "GS cannot be accessed from the user space");
+        }
     }
     Ok(())
 }

src/libos/src/process/exit.rs

@@ -20,8 +20,7 @@ pub fn do_exit(exit_status: i32) {
     current.status = Status::ZOMBIE;
 
     // Update children
-    for child_weak in &current.children {
-        let child_ref = child_weak.upgrade().unwrap();
+    for child_ref in current.get_children_iter() {
         let mut child = child_ref.lock().unwrap();
         child.parent = Some(IDLE_PROCESS.clone());
     }
@@ -29,7 +28,9 @@ pub fn do_exit(exit_status: i32) {
 
     // Notify another process, if any, that waits on ctid (see set_tid_address)
     if let Some(ctid) = current.clear_child_tid {
-        unsafe { atomic_store(ctid, 0); }
+        unsafe {
+            atomic_store(ctid, 0);
+        }
         futex_wake(ctid as *const i32, 1);
     }
 
@@ -69,8 +70,7 @@ pub fn do_wait4(child_filter: &ChildProcessFilter, exit_status: &mut i32) -> Res
         let mut current = current_ref.lock().unwrap();
 
         let mut any_child_to_wait_for = false;
-        for child_weak in current.get_children() {
-            let child_ref = child_weak.upgrade().unwrap();
+        for child_ref in current.get_children_iter() {
             let child = child_ref.lock().unwrap();
 
             let may_wait_for = match child_filter {
@@ -82,8 +82,9 @@ pub fn do_wait4(child_filter: &ChildProcessFilter, exit_status: &mut i32) -> Res
                 continue;
             }
 
-            // Return immediately as a child that we wait for has alreay exited
+            // Return immediately as a child that we wait for has already exited
             if child.status == Status::ZOMBIE {
+                process_table::remove(child.pid);
                 return Ok(child.pid);
             }
 
@@ -102,13 +103,12 @@ pub fn do_wait4(child_filter: &ChildProcessFilter, exit_status: &mut i32) -> Res
         waiter
     };
 
-    let child_pid = Waiter::sleep_until_woken_with_result(waiter);
+    let child_pid = waiter.sleep_until_woken_with_result();
 
     let mut current = current_ref.lock().unwrap();
     let child_i = {
         let mut child_i_opt = None;
-        for (child_i, child_weak) in current.children.iter().enumerate() {
-            let child_ref = child_weak.upgrade().unwrap();
+        for (child_i, child_ref) in current.get_children_iter().enumerate() {
             let child = child_ref.lock().unwrap();
             if child.get_pid() != child_pid {
                 continue;

src/libos/src/process/futex.rs

@@ -18,7 +18,7 @@ pub enum FutexOp {
     FUTEX_TRYLOCK_PI = 8,
     FUTEX_WAIT_BITSET = 9,
 }
-const FUTEX_OP_MASK : u32 = 0x0000_000F;
+const FUTEX_OP_MASK: u32 = 0x0000_000F;
 
 impl FutexOp {
     pub fn from_u32(bits: u32) -> Result<FutexOp, Error> {
@@ -44,12 +44,11 @@ bitflags! {
         const FUTEX_CLOCK_REALTIME  = 256;
     }
 }
-const FUTEX_FLAGS_MASK : u32 = 0xFFFF_FFF0;
+const FUTEX_FLAGS_MASK: u32 = 0xFFFF_FFF0;
 
 impl FutexFlags {
     pub fn from_u32(bits: u32) -> Result<FutexFlags, Error> {
-        FutexFlags::from_bits(bits).ok_or_else(||
-            Error::new(Errno::EINVAL, "Unknown futex flags"))
+        FutexFlags::from_bits(bits).ok_or_else(|| Error::new(Errno::EINVAL, "Unknown futex flags"))
     }
 }
 
@@ -65,12 +64,10 @@ pub fn futex_op_and_flags_from_u32(bits: u32) -> Result<(FutexOp, FutexFlags), E
     Ok((op, flags))
 }
 
-
 /// Do futex wait
 pub fn futex_wait(futex_addr: *const i32, futex_val: i32) -> Result<(), Error> {
     let futex_key = FutexKey::new(futex_addr);
-    let futex_item = FUTEX_TABLE.lock().unwrap()
-        .get_or_new_item(futex_key);
+    let futex_item = FUTEX_TABLE.lock().unwrap().get_or_new_item(futex_key);
 
     futex_item.wait(futex_val);
 
@@ -87,11 +84,8 @@ pub fn futex_wake(futex_addr: *const i32, max_count: usize) -> Result<usize, Err
     Ok(count)
 }
 
-
 lazy_static! {
-    static ref FUTEX_TABLE : SgxMutex<FutexTable> = {
-        SgxMutex::new(FutexTable::new())
-    };
+    static ref FUTEX_TABLE: SgxMutex<FutexTable> = { SgxMutex::new(FutexTable::new()) };
 }
 
 #[derive(PartialEq, Eq, Hash, Copy, Clone)]
@@ -126,7 +120,9 @@ impl FutexItem {
         while count < max_count {
             let waiter = {
                 let waiter_option = queue.pop_front();
-                if waiter_option.is_none() { break; }
+                if waiter_option.is_none() {
+                    break;
+                }
                 waiter_option.unwrap()
             };
             waiter.wake();
@@ -165,15 +161,17 @@ impl FutexTable {
 
     pub fn get_or_new_item(&mut self, key: FutexKey) -> FutexItemRef {
         let table = &mut self.table;
-        let item = table.entry(key).or_insert_with(|| {
-            Arc::new(FutexItem::new(key))
-        });
+        let item = table
+            .entry(key)
+            .or_insert_with(|| Arc::new(FutexItem::new(key)));
         item.clone()
     }
 
     pub fn get_item(&mut self, key: FutexKey) -> Result<FutexItemRef, Error> {
         let table = &mut self.table;
-        table.get_mut(&key).map(|item| item.clone())
+        table
+            .get_mut(&key)
+            .map(|item| item.clone())
             .ok_or_else(|| Error::new(Errno::ENOENT, "futex key cannot be found"))
     }
 
@@ -193,7 +191,6 @@ impl FutexTable {
     }
 }
 
-
 #[derive(Debug)]
 struct Waiter {
     thread: *const c_void,

src/libos/src/process/mod.rs

@@ -1,12 +1,12 @@
-pub use self::process::{Status, IDLE_PROCESS};
-pub use self::task::{get_current, run_task};
-pub use self::process_table::{get};
+pub use self::arch_prctl::{do_arch_prctl, ArchPrctlCode};
 pub use self::exit::{do_exit, do_wait4, ChildProcessFilter};
+pub use self::futex::{futex_op_and_flags_from_u32, futex_wait, futex_wake, FutexFlags, FutexOp};
+pub use self::process::{Status, IDLE_PROCESS};
+pub use self::process_table::get;
 pub use self::spawn::{do_spawn, FileAction};
+pub use self::task::{current_pid, get_current, run_task};
+pub use self::thread::{do_clone, do_set_tid_address, CloneFlags, ThreadGroup};
 pub use self::wait::{WaitQueue, Waiter};
-pub use self::thread::{do_clone, CloneFlags, ThreadGroup, do_set_tid_address};
-pub use self::futex::{FutexOp, FutexFlags, futex_op_and_flags_from_u32, futex_wake, futex_wait};
-pub use self::arch_prctl::{ArchPrctlCode, do_arch_prctl};
 
 #[allow(non_camel_case_types)]
 pub type pid_t = u32;
@@ -64,18 +64,18 @@ pub fn do_getppid() -> pid_t {
     parent.get_pid()
 }
 
+mod arch_prctl;
 mod exit;
+mod futex;
 mod process;
 mod process_table;
 mod spawn;
 mod task;
-mod wait;
 mod thread;
-mod futex;
-mod arch_prctl;
+mod wait;
 
 use self::task::Task;
 use super::*;
 use fs::{File, FileRef, FileTable};
+use misc::ResourceLimitsRef;
 use vm::{ProcessVM, VMRangeTrait};
-use misc::{ResourceLimitsRef};

src/libos/src/process/process.rs

@@ -88,8 +88,10 @@ impl Process {
     pub fn get_parent(&self) -> &ProcessRef {
         self.parent.as_ref().unwrap()
     }
-    pub fn get_children(&self) -> &[ProcessWeakRef] {
-        &self.children
+    pub fn get_children_iter(&self) -> impl Iterator<Item = ProcessRef> + '_ {
+        self.children
+            .iter()
+            .filter_map(|child_weak| child_weak.upgrade())
     }
     pub fn change_cwd(&mut self, path: &str) {
         if path.len() > 0 && path.as_bytes()[0] == b'/' {

src/libos/src/process/process_table.rs

@@ -15,7 +15,10 @@ pub fn remove(pid: pid_t) {
 }
 
 pub fn get(pid: pid_t) -> Result<ProcessRef, Error> {
-    PROCESS_TABLE.lock().unwrap().get(&pid)
+    PROCESS_TABLE
+        .lock()
+        .unwrap()
+        .get(&pid)
         .map(|pr| pr.clone())
         .ok_or_else(|| Error::new(Errno::ENOENT, "process not found"))
 }

src/libos/src/process/spawn/init_stack.rs

@@ -139,7 +139,7 @@ impl StackBuf {
             let old_pos = self.stack_pos.get();
             let new_pos = align_down(old_pos - size, align);
             if new_pos < self.stack_bottom {
-                return Err(Error::new(Errno::ENOMEM, "No enough space in buffer"));
+                return errno!(ENOMEM, "No enough space in buffer");
             }
             new_pos
         };
@@ -274,7 +274,7 @@ impl AuxTable {
 
     pub fn set_val(&mut self, key: AuxKey, val: u64) -> Result<(), Error> {
         if key == AuxKey::AT_NULL || key == AuxKey::AT_IGNORE {
-            return Err(Error::new(Errno::EINVAL, "Illegal key"));
+            return errno!(EINVAL, "Illegal key");
         }
         self.values[key as usize] = Some(val);
         Ok(())

src/libos/src/process/spawn/init_vm.rs

@@ -4,8 +4,8 @@ use std::ptr;
 use xmas_elf::{header, program, sections, ElfFile};
 
 pub const DEFAULT_STACK_SIZE: usize = 1 * 1024 * 1024;
-pub const DEFAULT_HEAP_SIZE: usize = 2 * 1024 * 1024;
-pub const DEFAULT_MMAP_SIZE: usize = 2 * 1024 * 1024;
+pub const DEFAULT_HEAP_SIZE: usize = 10 * 1024 * 1024;
+pub const DEFAULT_MMAP_SIZE: usize = 40 * 1024 * 1024;
 
 pub fn do_init(elf_file: &ElfFile, elf_buf: &[u8]) -> Result<ProcessVM, Error> {
     let mut code_seg = get_code_segment(elf_file)?;

src/libos/src/process/spawn/mod.rs

@@ -1,15 +1,15 @@
-use xmas_elf::{ElfFile, header, program, sections};
 use xmas_elf::symbol_table::Entry;
+use xmas_elf::{header, program, sections, ElfFile};
 
-use fs::{File, FileDesc, FileTable, INodeExt, ROOT_INODE, StdinFile, StdoutFile};
+use fs::{File, FileDesc, FileTable, INodeExt, OpenFlags, StdinFile, StdoutFile, ROOT_INODE};
+use misc::ResourceLimitsRef;
 use std::ffi::{CStr, CString};
 use std::path::Path;
 use std::sgxfs::SgxFile;
 use vm::{ProcessVM, VMRangeTrait};
-use misc::{ResourceLimitsRef};
 
-use super::*;
 use super::task::Task;
+use super::*;
 
 use self::init_stack::{AuxKey, AuxTable};
 
@@ -20,8 +20,14 @@ mod segment;
 
 #[derive(Debug)]
 pub enum FileAction {
-    // TODO: Add open action
-    // Open(...)
+    /// open(path, oflag, mode) had been called, and the returned file
+    /// descriptor, if not `fd`, had been changed to `fd`.
+    Open {
+        path: String,
+        mode: u32,
+        oflag: u32,
+        fd: FileDesc,
+    },
     Dup2(FileDesc, FileDesc),
     Close(FileDesc),
 }
@@ -59,7 +65,7 @@ pub fn do_spawn<P: AsRef<Path>>(
         let program_entry = {
             let program_entry = base_addr + elf_helper::get_start_address(&elf_file)?;
             if !vm.get_code_vma().contains_obj(program_entry, 16) {
-                return Err(Error::new(Errno::EINVAL, "Invalid program entry"));
+                return errno!(EINVAL, "Invalid program entry");
             }
             program_entry
         };
@@ -87,21 +93,38 @@ fn init_files(parent_ref: &ProcessRef, file_actions: &[FileAction]) -> Result<Fi
     let parent = parent_ref.lock().unwrap();
     let should_inherit_file_table = parent.get_pid() > 0;
     if should_inherit_file_table {
+        // Fork: clone file table
         let mut cloned_file_table = parent.get_files().lock().unwrap().clone();
         // Perform file actions to modify the cloned file table
         for file_action in file_actions {
             match file_action {
-                FileAction::Dup2(old_fd, new_fd) => {
-                    let file = cloned_file_table.get(*old_fd)?;
+                &FileAction::Open {
+                    ref path,
+                    mode,
+                    oflag,
+                    fd,
+                } => {
+                    let flags = OpenFlags::from_bits_truncate(oflag);
+                    let file = parent.open_file(path.as_str(), flags, mode)?;
+                    let file_ref: Arc<Box<File>> = Arc::new(file);
+
+                    let close_on_spawn = flags.contains(OpenFlags::CLOEXEC);
+                    cloned_file_table.put_at(fd, file_ref, close_on_spawn);
+                }
+                &FileAction::Dup2(old_fd, new_fd) => {
+                    let file = cloned_file_table.get(old_fd)?;
                     if old_fd != new_fd {
-                        cloned_file_table.put_at(*new_fd, file, false);
+                        cloned_file_table.put_at(new_fd, file, false);
                     }
                 }
-                FileAction::Close(fd) => {
-                    cloned_file_table.del(*fd)?;
+                &FileAction::Close(fd) => {
+                    // ignore error
+                    cloned_file_table.del(fd);
                 }
             }
         }
+        // Exec: close fd with close_on_spawn
+        cloned_file_table.close_on_spawn();
         return Ok(cloned_file_table);
     }
     drop(parent);
@@ -133,7 +156,11 @@ fn init_task(
     })
 }
 
-fn init_auxtbl(base_addr: usize, program_entry: usize, elf_file: &ElfFile) -> Result<AuxTable, Error> {
+fn init_auxtbl(
+    base_addr: usize,
+    program_entry: usize,
+    elf_file: &ElfFile,
+) -> Result<AuxTable, Error> {
     let mut auxtbl = AuxTable::new();
     auxtbl.set_val(AuxKey::AT_PAGESZ, 4096)?;
     auxtbl.set_val(AuxKey::AT_UID, 0)?;

src/libos/src/process/spawn/segment.rs

@@ -66,10 +66,15 @@ impl Segment {
         let mut target_buf = unsafe {
             slice::from_raw_parts_mut(
                 (self.process_base_addr + self.mem_addr) as *mut u8,
-                self.file_size,
+                self.mem_size,
             )
         };
-        target_buf.copy_from_slice(&elf_buf[self.file_offset..(self.file_offset + self.file_size)]);
+        target_buf[0..self.file_size]
+            .copy_from_slice(&elf_buf[self.file_offset..(self.file_offset + self.file_size)]);
+        #[cfg(feature = "integrity_only_opt")]
+        for i in &mut target_buf[self.file_size..self.mem_size] {
+            *i = 0;
+        }
     }
 
     pub fn set_runtime_info(

src/libos/src/process/task.rs

@@ -68,6 +68,12 @@ thread_local! {
     static _CURRENT_PROCESS_PTR: Cell<*const SgxMutex<Process>> = {
         Cell::new(0 as *const SgxMutex<Process>)
     };
+    // for log getting pid without locking process
+    static _PID: Cell<pid_t> = Cell::new(0);
+}
+
+pub fn current_pid() -> pid_t {
+    _PID.with(|p| p.get())
 }
 
 pub fn get_current() -> ProcessRef {
@@ -81,6 +87,9 @@ pub fn get_current() -> ProcessRef {
 }
 
 fn set_current(process: &ProcessRef) {
+    let pid = process.lock().unwrap().get_pid();
+    _PID.with(|p| p.set(pid));
+
     let process_ref_clone = process.clone();
     let process_ptr = Arc::into_raw(process_ref_clone);
 
@@ -90,6 +99,7 @@ fn set_current(process: &ProcessRef) {
 }
 
 fn reset_current() {
+    _PID.with(|p| p.set(0));
     let mut process_ptr = _CURRENT_PROCESS_PTR.with(|cp| cp.replace(0 as *const SgxMutex<Process>));
 
     // Prevent memory leakage

src/libos/src/process/thread.rs

@@ -4,7 +4,6 @@ pub struct ThreadGroup {
     threads: Vec<ProcessRef>,
 }
 
-
 bitflags! {
     pub struct CloneFlags : u32 {
         const CLONE_VM              = 0x00000100;
@@ -40,8 +39,10 @@ pub fn do_clone(
     ctid: Option<*mut pid_t>,
     new_tls: Option<usize>,
 ) -> Result<pid_t, Error> {
-    info!("clone: flags: {:?}, stack_addr: {:?}, ptid: {:?}, ctid: {:?}, new_tls: {:?}",
-          flags, stack_addr, ptid, ctid, new_tls);
+    info!(
+        "clone: flags: {:?}, stack_addr: {:?}, ptid: {:?}, ctid: {:?}, new_tls: {:?}",
+        flags, stack_addr, ptid, ctid, new_tls
+    );
     // TODO: return error for unsupported flags
 
     let current_ref = get_current();
@@ -75,7 +76,9 @@ pub fn do_clone(
     process_table::put(new_thread_pid, new_thread_ref.clone());
 
     if let Some(ptid) = ptid {
-        unsafe { *ptid = new_thread_pid; }
+        unsafe {
+            *ptid = new_thread_pid;
+        }
     }
 
     task::enqueue_task(new_thread_ref);

src/libos/src/process/wait.rs

@@ -48,14 +48,14 @@ where
         self.inner.lock().unwrap().data
     }
 
-    pub fn sleep_until_woken_with_result(waiter: Waiter<D, R>) -> R {
-        while !waiter.inner.lock().unwrap().is_woken {
+    pub fn sleep_until_woken_with_result(self) -> R {
+        while !self.inner.lock().unwrap().is_woken {
             unsafe {
-                wait_event(waiter.thread);
+                wait_event(self.thread);
             }
         }
 
-        waiter.inner.lock().unwrap().result.unwrap()
+        self.inner.lock().unwrap().result.unwrap()
     }
 }
 

src/libos/src/syscall/mod.rs

@@ -1,17 +1,27 @@
-use {fs, process, std, vm};
-use fs::{File, FileDesc, off_t, AccessModes, AccessFlags, AT_FDCWD, FcntlCmd};
+//! System call handler
+//!
+//! # Syscall processing flow
+//!
+//! 1. User call `__occlum_syscall` (at `syscall_entry_x86_64.S`)
+//! 2. Do some bound checks then call `dispatch_syscall` (at this file)
+//! 3. Dispatch the syscall to `do_*` (at this file)
+//! 4. Do some memory checks then call `mod::do_*` (at each module)
+
+use fs::*;
+use misc::{resource_t, rlimit_t, utsname_t};
 use prelude::*;
-use process::{ChildProcessFilter, FileAction, pid_t, CloneFlags, FutexFlags, FutexOp};
+use process::{pid_t, ChildProcessFilter, CloneFlags, FileAction, FutexFlags, FutexOp};
 use std::ffi::{CStr, CString};
 use std::ptr;
 use time::timeval_t;
 use util::mem_util::from_user::*;
 use vm::{VMAreaFlags, VMResizeOptions};
-use misc::{utsname_t, resource_t, rlimit_t};
+use {fs, process, std, vm};
 
 use super::*;
 
 use self::consts::*;
+use std::any::Any;
 
 // Use the internal syscall wrappers from sgx_tstd
 //use std::libc_fs as fs;
@@ -19,6 +29,8 @@ use self::consts::*;
 
 mod consts;
 
+static mut SYSCALL_TIMING: [usize; 361] = [0; 361];
+
 #[no_mangle]
 #[deny(unreachable_patterns)]
 pub extern "C" fn dispatch_syscall(
@@ -34,7 +46,21 @@ pub extern "C" fn dispatch_syscall(
         "syscall {}: {:#x}, {:#x}, {:#x}, {:#x}, {:#x}, {:#x}",
         num, arg0, arg1, arg2, arg3, arg4, arg5
     );
+    #[cfg(feature = "syscall_timing")]
+    let time_start = {
+        static mut LAST_PRINT: usize = 0;
+        let time = crate::time::do_gettimeofday().as_usec();
+        unsafe {
+            if time / 1000000 / 5 > LAST_PRINT {
+                LAST_PRINT = time / 1000000 / 5;
+                print_syscall_timing();
+            }
+        }
+        time
+    };
+
     let ret = match num {
+        // file
         SYS_OPEN => do_open(arg0 as *const i8, arg1 as u32, arg2 as u32),
         SYS_CLOSE => do_close(arg0 as FileDesc),
         SYS_READ => do_read(arg0 as FileDesc, arg1 as *mut u8, arg2 as usize),
@@ -73,8 +99,44 @@ pub extern "C" fn dispatch_syscall(
         SYS_LINK => do_link(arg0 as *const i8, arg1 as *const i8),
         SYS_UNLINK => do_unlink(arg0 as *const i8),
         SYS_READLINK => do_readlink(arg0 as *const i8, arg1 as *mut u8, arg2 as usize),
+        SYS_SENDFILE => do_sendfile(
+            arg0 as FileDesc,
+            arg1 as FileDesc,
+            arg2 as *mut off_t,
+            arg3 as usize,
+        ),
         SYS_FCNTL => do_fcntl(arg0 as FileDesc, arg1 as u32, arg2 as u64),
+        SYS_IOCTL => do_ioctl(arg0 as FileDesc, arg1 as c_int, arg2 as *mut c_int),
 
+        // IO multiplexing
+        SYS_SELECT => do_select(
+            arg0 as c_int,
+            arg1 as *mut libc::fd_set,
+            arg2 as *mut libc::fd_set,
+            arg3 as *mut libc::fd_set,
+            arg4 as *const libc::timeval,
+        ),
+        SYS_POLL => do_poll(
+            arg0 as *mut libc::pollfd,
+            arg1 as libc::nfds_t,
+            arg2 as c_int,
+        ),
+        SYS_EPOLL_CREATE => do_epoll_create(arg0 as c_int),
+        SYS_EPOLL_CREATE1 => do_epoll_create1(arg0 as c_int),
+        SYS_EPOLL_CTL => do_epoll_ctl(
+            arg0 as c_int,
+            arg1 as c_int,
+            arg2 as c_int,
+            arg3 as *const libc::epoll_event,
+        ),
+        SYS_EPOLL_WAIT => do_epoll_wait(
+            arg0 as c_int,
+            arg1 as *mut libc::epoll_event,
+            arg2 as c_int,
+            arg3 as c_int,
+        ),
+
+        // process
         SYS_EXIT => do_exit(arg0 as i32),
         SYS_SPAWN => do_spawn(
             arg0 as *mut u32,
@@ -114,6 +176,7 @@ pub extern "C" fn dispatch_syscall(
         SYS_ARCH_PRCTL => do_arch_prctl(arg0 as u32, arg1 as *mut usize),
         SYS_SET_TID_ADDRESS => do_set_tid_address(arg0 as *mut pid_t),
 
+        // memory
         SYS_MMAP => do_mmap(
             arg0 as usize,
             arg1 as usize,
@@ -130,11 +193,7 @@ pub extern "C" fn dispatch_syscall(
             arg3 as i32,
             arg4 as usize,
         ),
-        SYS_MPROTECT => do_mprotect(
-            arg0 as usize,
-            arg1 as usize,
-            arg2 as u32,
-        ),
+        SYS_MPROTECT => do_mprotect(arg0 as usize, arg1 as usize, arg2 as u32),
         SYS_BRK => do_brk(arg0 as usize),
 
         SYS_PIPE => do_pipe2(arg0 as *mut i32, 0),
@@ -147,18 +206,115 @@ pub extern "C" fn dispatch_syscall(
 
         SYS_UNAME => do_uname(arg0 as *mut utsname_t),
 
-        SYS_PRLIMIT64 => do_prlimit(arg0 as pid_t, arg1 as u32, arg2 as *const rlimit_t, arg3 as *mut rlimit_t),
+        SYS_PRLIMIT64 => do_prlimit(
+            arg0 as pid_t,
+            arg1 as u32,
+            arg2 as *const rlimit_t,
+            arg3 as *mut rlimit_t,
+        ),
+
+        // socket
+        SYS_SOCKET => do_socket(arg0 as c_int, arg1 as c_int, arg2 as c_int),
+        SYS_CONNECT => do_connect(
+            arg0 as c_int,
+            arg1 as *const libc::sockaddr,
+            arg2 as libc::socklen_t,
+        ),
+        SYS_ACCEPT => do_accept4(
+            arg0 as c_int,
+            arg1 as *mut libc::sockaddr,
+            arg2 as *mut libc::socklen_t,
+            0,
+        ),
+        SYS_ACCEPT4 => do_accept4(
+            arg0 as c_int,
+            arg1 as *mut libc::sockaddr,
+            arg2 as *mut libc::socklen_t,
+            arg3 as c_int,
+        ),
+        SYS_SHUTDOWN => do_shutdown(arg0 as c_int, arg1 as c_int),
+        SYS_BIND => do_bind(
+            arg0 as c_int,
+            arg1 as *const libc::sockaddr,
+            arg2 as libc::socklen_t,
+        ),
+        SYS_LISTEN => do_listen(arg0 as c_int, arg1 as c_int),
+        SYS_SETSOCKOPT => do_setsockopt(
+            arg0 as c_int,
+            arg1 as c_int,
+            arg2 as c_int,
+            arg3 as *const c_void,
+            arg4 as libc::socklen_t,
+        ),
+        SYS_GETSOCKOPT => do_getsockopt(
+            arg0 as c_int,
+            arg1 as c_int,
+            arg2 as c_int,
+            arg3 as *mut c_void,
+            arg4 as *mut libc::socklen_t,
+        ),
+        SYS_GETPEERNAME => do_getpeername(
+            arg0 as c_int,
+            arg1 as *mut libc::sockaddr,
+            arg2 as *mut libc::socklen_t,
+        ),
+        SYS_GETSOCKNAME => do_getsockname(
+            arg0 as c_int,
+            arg1 as *mut libc::sockaddr,
+            arg2 as *mut libc::socklen_t,
+        ),
+        SYS_SENDTO => do_sendto(
+            arg0 as c_int,
+            arg1 as *const c_void,
+            arg2 as size_t,
+            arg3 as c_int,
+            arg4 as *const libc::sockaddr,
+            arg5 as libc::socklen_t,
+        ),
+        SYS_RECVFROM => do_recvfrom(
+            arg0 as c_int,
+            arg1 as *mut c_void,
+            arg2 as size_t,
+            arg3 as c_int,
+            arg4 as *mut libc::sockaddr,
+            arg5 as *mut libc::socklen_t,
+        ),
 
         _ => do_unknown(num, arg0, arg1, arg2, arg3, arg4, arg5),
     };
-    debug!("syscall return: {:?}", ret);
+
+    #[cfg(feature = "syscall_timing")]
+    {
+        let time_end = crate::time::do_gettimeofday().as_usec();
+        let time = time_end - time_start;
+        unsafe {
+            SYSCALL_TIMING[num as usize] += time as usize;
+        }
+    }
+
+    info!("=> {:?}", ret);
 
     match ret {
         Ok(code) => code as isize,
+        Err(e) if e.errno.as_retval() == 0 => panic!("undefined errno"),
         Err(e) => e.errno.as_retval() as isize,
     }
 }
 
+#[cfg(feature = "syscall_timing")]
+fn print_syscall_timing() {
+    println!("syscall timing:");
+    for (i, &time) in unsafe { SYSCALL_TIMING }.iter().enumerate() {
+        if time == 0 {
+            continue;
+        }
+        println!("{:>3}: {:>6} us", i, time);
+    }
+    for x in unsafe { SYSCALL_TIMING.iter_mut() } {
+        *x = 0;
+    }
+}
+
 #[allow(non_camel_case_types)]
 pub struct iovec_t {
     base: *const c_void,
@@ -184,7 +340,7 @@ pub struct FdOp {
     srcfd: u32,
     oflag: u32,
     mode: u32,
-    path: *const u8,
+    path: *const i8,
 }
 
 fn clone_file_actions_safely(fdop_ptr: *const FdOp) -> Result<Vec<FileAction>, Error> {
@@ -198,9 +354,14 @@ fn clone_file_actions_safely(fdop_ptr: *const FdOp) -> Result<Vec<FileAction>, E
         let file_action = match fdop.cmd {
             FDOP_CLOSE => FileAction::Close(fdop.fd),
             FDOP_DUP2 => FileAction::Dup2(fdop.srcfd, fdop.fd),
-            FDOP_OPEN => {
-                return errno!(EINVAL, "Not implemented");
-            }
+            FDOP_OPEN => FileAction::Open {
+                path: clone_cstring_safely(fdop.path)?
+                    .to_string_lossy()
+                    .into_owned(),
+                mode: fdop.mode,
+                oflag: fdop.oflag,
+                fd: fdop.fd,
+            },
             _ => {
                 return errno!(EINVAL, "Unknown file action command");
             }
@@ -250,8 +411,7 @@ pub fn do_clone(
         if flags.contains(CloneFlags::CLONE_PARENT_SETTID) {
             check_mut_ptr(ptid)?;
             Some(ptid)
-        }
-        else {
+        } else {
             None
         }
     };
@@ -259,8 +419,7 @@ pub fn do_clone(
         if flags.contains(CloneFlags::CLONE_CHILD_CLEARTID) {
             check_mut_ptr(ctid)?;
             Some(ctid)
-        }
-        else {
+        } else {
             None
         }
     };
@@ -268,8 +427,7 @@ pub fn do_clone(
         if flags.contains(CloneFlags::CLONE_SETTLS) {
             check_mut_ptr(new_tls as *mut usize)?;
             Some(new_tls)
-        }
-        else {
+        } else {
             None
         }
     };
@@ -279,17 +437,11 @@ pub fn do_clone(
     Ok(child_pid as isize)
 }
 
-pub fn do_futex(
-    futex_addr: *const i32,
-    futex_op: u32,
-    futex_val: i32,
-) -> Result<isize, Error> {
+pub fn do_futex(futex_addr: *const i32, futex_op: u32, futex_val: i32) -> Result<isize, Error> {
     check_ptr(futex_addr)?;
     let (futex_op, futex_flags) = process::futex_op_and_flags_from_u32(futex_op)?;
     match futex_op {
-        FutexOp::FUTEX_WAIT => {
-            process::futex_wait(futex_addr, futex_val).map(|_| 0)
-        }
+        FutexOp::FUTEX_WAIT => process::futex_wait(futex_addr, futex_val).map(|_| 0),
         FutexOp::FUTEX_WAKE => {
             let max_count = {
                 if futex_val < 0 {
@@ -297,9 +449,8 @@ pub fn do_futex(
                 }
                 futex_val as usize
             };
-            process::futex_wake(futex_addr, max_count)
-                .map(|count| count as isize)
-        },
+            process::futex_wake(futex_addr, max_count).map(|count| count as isize)
+        }
         _ => errno!(ENOSYS, "the futex operation is not supported"),
     }
 }
@@ -336,7 +487,7 @@ fn do_write(fd: FileDesc, buf: *const u8, size: usize) -> Result<isize, Error> {
 fn do_writev(fd: FileDesc, iov: *const iovec_t, count: i32) -> Result<isize, Error> {
     let count = {
         if count < 0 {
-            return Err(Error::new(Errno::EINVAL, "Invalid count of iovec"));
+            return errno!(EINVAL, "Invalid count of iovec");
         }
         count as usize
     };
@@ -361,7 +512,7 @@ fn do_writev(fd: FileDesc, iov: *const iovec_t, count: i32) -> Result<isize, Err
 fn do_readv(fd: FileDesc, iov: *mut iovec_t, count: i32) -> Result<isize, Error> {
     let count = {
         if count < 0 {
-            return Err(Error::new(Errno::EINVAL, "Invalid count of iovec"));
+            return errno!(EINVAL, "Invalid count of iovec");
         }
         count as usize
     };
@@ -438,7 +589,7 @@ fn do_lseek(fd: FileDesc, offset: off_t, whence: i32) -> Result<isize, Error> {
         0 => {
             // SEEK_SET
             if offset < 0 {
-                return Err(Error::new(Errno::EINVAL, "Invalid offset"));
+                return errno!(EINVAL, "Invalid offset");
             }
             SeekFrom::Start(offset as u64)
         }
@@ -451,7 +602,7 @@ fn do_lseek(fd: FileDesc, offset: off_t, whence: i32) -> Result<isize, Error> {
             SeekFrom::End(offset)
         }
         _ => {
-            return Err(Error::new(Errno::EINVAL, "Invalid whence"));
+            return errno!(EINVAL, "Invalid whence");
         }
     };
 
@@ -525,11 +676,7 @@ fn do_mremap(
     Ok(ret_addr as isize)
 }
 
-fn do_mprotect(
-    addr: usize,
-    len: usize,
-    prot: u32,
-) -> Result<isize, Error> {
+fn do_mprotect(addr: usize, len: usize, prot: u32) -> Result<isize, Error> {
     // TODO: implement it
     Ok(0)
 }
@@ -608,7 +755,6 @@ fn do_getegid() -> Result<isize, Error> {
     Ok(0)
 }
 
-
 fn do_pipe2(fds_u: *mut i32, flags: u32) -> Result<isize, Error> {
     check_mut_array(fds_u, 2)?;
     // TODO: how to deal with open flags???
@@ -649,6 +795,7 @@ fn do_gettimeofday(tv_u: *mut timeval_t) -> Result<isize, Error> {
 const MAP_FAILED: *const c_void = ((-1) as i64) as *const c_void;
 
 fn do_exit(status: i32) -> ! {
+    info!("exit: {}", status);
     extern "C" {
         fn do_exit_task() -> !;
     }
@@ -658,12 +805,20 @@ fn do_exit(status: i32) -> ! {
     }
 }
 
-fn do_unknown(num: u32, arg0: isize, arg1: isize, arg2: isize, arg3: isize, arg4: isize, arg5: isize) -> Result<isize, Error> {
+fn do_unknown(
+    num: u32,
+    arg0: isize,
+    arg1: isize,
+    arg2: isize,
+    arg3: isize,
+    arg4: isize,
+    arg5: isize,
+) -> Result<isize, Error> {
     warn!(
         "unknown or unsupported syscall (# = {}): {:#x}, {:#x}, {:#x}, {:#x}, {:#x}, {:#x}",
         num, arg0, arg1, arg2, arg3, arg4, arg5
     );
-    Err(Error::new(ENOSYS, "Unknown syscall"))
+    errno!(ENOSYS, "Unknown syscall")
 }
 
 fn do_getcwd(buf: *mut u8, size: usize) -> Result<isize, Error> {
@@ -675,7 +830,7 @@ fn do_getcwd(buf: *mut u8, size: usize) -> Result<isize, Error> {
     let mut proc = proc_ref.lock().unwrap();
     let cwd = proc.get_cwd();
     if cwd.len() + 1 > safe_buf.len() {
-        return Err(Error::new(ERANGE, "buf is not long enough"));
+        return errno!(ERANGE, "buf is not long enough");
     }
     safe_buf[..cwd.len()].copy_from_slice(cwd.as_bytes());
     safe_buf[cwd.len()] = 0;
@@ -738,11 +893,51 @@ fn do_readlink(path: *const i8, buf: *mut u8, size: usize) -> Result<isize, Erro
     Ok(len as isize)
 }
 
+fn do_sendfile(
+    out_fd: FileDesc,
+    in_fd: FileDesc,
+    offset_ptr: *mut off_t,
+    count: usize,
+) -> Result<isize, Error> {
+    let offset = if offset_ptr.is_null() {
+        None
+    } else {
+        check_mut_ptr(offset_ptr)?;
+        Some(unsafe { offset_ptr.read() })
+    };
+
+    let (len, offset) = fs::do_sendfile(out_fd, in_fd, offset, count)?;
+    if !offset_ptr.is_null() {
+        unsafe {
+            offset_ptr.write(offset as off_t);
+        }
+    }
+    Ok(len as isize)
+}
+
 fn do_fcntl(fd: FileDesc, cmd: u32, arg: u64) -> Result<isize, Error> {
     let cmd = FcntlCmd::from_raw(cmd, arg)?;
     fs::do_fcntl(fd, &cmd)
 }
 
+fn do_ioctl(fd: FileDesc, cmd: c_int, argp: *mut c_int) -> Result<isize, Error> {
+    info!("ioctl: fd: {}, cmd: {}, argp: {:?}", fd, cmd, argp);
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::ioctl_arg1(socket.fd(), cmd, argp));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        // TODO: check argp
+        unix_socket.ioctl(cmd, argp)?;
+        Ok(0)
+    } else {
+        warn!("ioctl is unimplemented");
+        errno!(ENOSYS, "ioctl is unimplemented")
+    }
+}
+
 fn do_arch_prctl(code: u32, addr: *mut usize) -> Result<isize, Error> {
     let code = process::ArchPrctlCode::from_u32(code)?;
     check_mut_ptr(addr)?;
@@ -754,20 +949,427 @@ fn do_set_tid_address(tidptr: *mut pid_t) -> Result<isize, Error> {
     process::do_set_tid_address(tidptr).map(|tid| tid as isize)
 }
 
+fn do_socket(domain: c_int, socket_type: c_int, protocol: c_int) -> Result<isize, Error> {
+    info!(
+        "socket: domain: {}, socket_type: {}, protocol: {}",
+        domain, socket_type, protocol
+    );
+
+    let file_ref: Arc<Box<File>> = match domain {
+        libc::AF_LOCAL => {
+            let unix_socket = UnixSocketFile::new(socket_type, protocol)?;
+            Arc::new(Box::new(unix_socket))
+        }
+        _ => {
+            let socket = SocketFile::new(domain, socket_type, protocol)?;
+            Arc::new(Box::new(socket))
+        }
+    };
+
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+
+    let fd = proc.get_files().lock().unwrap().put(file_ref, false);
+    Ok(fd as isize)
+}
+
+fn do_connect(
+    fd: c_int,
+    addr: *const libc::sockaddr,
+    addr_len: libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "connect: fd: {}, addr: {:?}, addr_len: {}",
+        fd, addr, addr_len
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::connect(socket.fd(), addr, addr_len));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        let addr = addr as *const libc::sockaddr_un;
+        check_ptr(addr)?; // TODO: check addr_len
+        let path = clone_cstring_safely(unsafe { (&*addr).sun_path.as_ptr() })?
+            .to_string_lossy()
+            .into_owned();
+        unix_socket.connect(path)?;
+        Ok(0)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_accept4(
+    fd: c_int,
+    addr: *mut libc::sockaddr,
+    addr_len: *mut libc::socklen_t,
+    flags: c_int,
+) -> Result<isize, Error> {
+    info!(
+        "accept4: fd: {}, addr: {:?}, addr_len: {:?}, flags: {:#x}",
+        fd, addr, addr_len, flags
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let socket = file_ref.as_socket()?;
+
+        let new_socket = socket.accept(addr, addr_len, flags)?;
+        let new_file_ref: Arc<Box<File>> = Arc::new(Box::new(new_socket));
+        let new_fd = proc.get_files().lock().unwrap().put(new_file_ref, false);
+
+        Ok(new_fd as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        let addr = addr as *mut libc::sockaddr_un;
+        check_mut_ptr(addr)?; // TODO: check addr_len
+
+        let new_socket = unix_socket.accept()?;
+        let new_file_ref: Arc<Box<File>> = Arc::new(Box::new(new_socket));
+        let new_fd = proc.get_files().lock().unwrap().put(new_file_ref, false);
+
+        Ok(new_fd as isize)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_shutdown(fd: c_int, how: c_int) -> Result<isize, Error> {
+    info!("shutdown: fd: {}, how: {}", fd, how);
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::shutdown(socket.fd(), how));
+        Ok(ret as isize)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_bind(
+    fd: c_int,
+    addr: *const libc::sockaddr,
+    addr_len: libc::socklen_t,
+) -> Result<isize, Error> {
+    info!("bind: fd: {}, addr: {:?}, addr_len: {}", fd, addr, addr_len);
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        check_ptr(addr)?; // TODO: check addr_len
+        let ret = try_libc!(libc::ocall::bind(socket.fd(), addr, addr_len));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        let addr = addr as *const libc::sockaddr_un;
+        check_ptr(addr)?; // TODO: check addr_len
+        let path = clone_cstring_safely(unsafe { (&*addr).sun_path.as_ptr() })?
+            .to_string_lossy()
+            .into_owned();
+        unix_socket.bind(path)?;
+        Ok(0)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_listen(fd: c_int, backlog: c_int) -> Result<isize, Error> {
+    info!("listen: fd: {}, backlog: {}", fd, backlog);
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::listen(socket.fd(), backlog));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        unix_socket.listen()?;
+        Ok(0)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_setsockopt(
+    fd: c_int,
+    level: c_int,
+    optname: c_int,
+    optval: *const c_void,
+    optlen: libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "setsockopt: fd: {}, level: {}, optname: {}, optval: {:?}, optlen: {:?}",
+        fd, level, optname, optval, optlen
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::setsockopt(
+            socket.fd(),
+            level,
+            optname,
+            optval,
+            optlen
+        ));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        warn!("setsockopt for unix socket is unimplemented");
+        Ok(0)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_getsockopt(
+    fd: c_int,
+    level: c_int,
+    optname: c_int,
+    optval: *mut c_void,
+    optlen: *mut libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "getsockopt: fd: {}, level: {}, optname: {}, optval: {:?}, optlen: {:?}",
+        fd, level, optname, optval, optlen
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    let socket = file_ref.as_socket()?;
+
+    let ret = try_libc!(libc::ocall::getsockopt(
+        socket.fd(),
+        level,
+        optname,
+        optval,
+        optlen
+    ));
+    Ok(ret as isize)
+}
+
+fn do_getpeername(
+    fd: c_int,
+    addr: *mut libc::sockaddr,
+    addr_len: *mut libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "getpeername: fd: {}, addr: {:?}, addr_len: {:?}",
+        fd, addr, addr_len
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::getpeername(socket.fd(), addr, addr_len));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        warn!("getpeername for unix socket is unimplemented");
+        errno!(
+            ENOTCONN,
+            "hack for php: Transport endpoint is not connected"
+        )
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_getsockname(
+    fd: c_int,
+    addr: *mut libc::sockaddr,
+    addr_len: *mut libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "getsockname: fd: {}, addr: {:?}, addr_len: {:?}",
+        fd, addr, addr_len
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    if let Ok(socket) = file_ref.as_socket() {
+        let ret = try_libc!(libc::ocall::getsockname(socket.fd(), addr, addr_len));
+        Ok(ret as isize)
+    } else if let Ok(unix_socket) = file_ref.as_unix_socket() {
+        warn!("getsockname for unix socket is unimplemented");
+        Ok(0)
+    } else {
+        errno!(EBADF, "not a socket")
+    }
+}
+
+fn do_sendto(
+    fd: c_int,
+    base: *const c_void,
+    len: size_t,
+    flags: c_int,
+    addr: *const libc::sockaddr,
+    addr_len: libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "sendto: fd: {}, base: {:?}, len: {}, addr: {:?}, addr_len: {}",
+        fd, base, len, addr, addr_len
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    let socket = file_ref.as_socket()?;
+
+    let ret = try_libc!(libc::ocall::sendto(
+        socket.fd(),
+        base,
+        len,
+        flags,
+        addr,
+        addr_len
+    ));
+    Ok(ret as isize)
+}
+
+fn do_recvfrom(
+    fd: c_int,
+    base: *mut c_void,
+    len: size_t,
+    flags: c_int,
+    addr: *mut libc::sockaddr,
+    addr_len: *mut libc::socklen_t,
+) -> Result<isize, Error> {
+    info!(
+        "recvfrom: fd: {}, base: {:?}, len: {}, flags: {}, addr: {:?}, addr_len: {:?}",
+        fd, base, len, flags, addr, addr_len
+    );
+    let current_ref = process::get_current();
+    let mut proc = current_ref.lock().unwrap();
+    let file_ref = proc.get_files().lock().unwrap().get(fd as FileDesc)?;
+    let socket = file_ref.as_socket()?;
+
+    let ret = try_libc!(libc::ocall::recvfrom(
+        socket.fd(),
+        base,
+        len,
+        flags,
+        addr,
+        addr_len
+    ));
+    Ok(ret as isize)
+}
+
+fn do_select(
+    nfds: c_int,
+    readfds: *mut libc::fd_set,
+    writefds: *mut libc::fd_set,
+    exceptfds: *mut libc::fd_set,
+    timeout: *const libc::timeval,
+) -> Result<isize, Error> {
+    // check arguments
+    if nfds < 0 || nfds >= libc::FD_SETSIZE as c_int {
+        return errno!(EINVAL, "nfds is negative or exceeds the resource limit");
+    }
+    let nfds = nfds as usize;
+
+    let mut zero_fds0: libc::fd_set = unsafe { core::mem::zeroed() };
+    let mut zero_fds1: libc::fd_set = unsafe { core::mem::zeroed() };
+    let mut zero_fds2: libc::fd_set = unsafe { core::mem::zeroed() };
+
+    let readfds = if !readfds.is_null() {
+        check_mut_ptr(readfds)?;
+        unsafe { &mut *readfds }
+    } else {
+        &mut zero_fds0
+    };
+    let writefds = if !writefds.is_null() {
+        check_mut_ptr(writefds)?;
+        unsafe { &mut *writefds }
+    } else {
+        &mut zero_fds1
+    };
+    let exceptfds = if !exceptfds.is_null() {
+        check_mut_ptr(exceptfds)?;
+        unsafe { &mut *exceptfds }
+    } else {
+        &mut zero_fds2
+    };
+    let timeout = if !timeout.is_null() {
+        check_ptr(timeout)?;
+        Some(unsafe { timeout.read() })
+    } else {
+        None
+    };
+
+    let n = fs::do_select(nfds, readfds, writefds, exceptfds, timeout)?;
+    Ok(n as isize)
+}
+
+fn do_poll(fds: *mut libc::pollfd, nfds: libc::nfds_t, timeout: c_int) -> Result<isize, Error> {
+    check_mut_array(fds, nfds as usize)?;
+    let polls = unsafe { std::slice::from_raw_parts_mut(fds, nfds as usize) };
+
+    let n = fs::do_poll(polls, timeout)?;
+    Ok(n as isize)
+}
+
+fn do_epoll_create(size: c_int) -> Result<isize, Error> {
+    if size <= 0 {
+        return errno!(EINVAL, "size is not positive");
+    }
+    do_epoll_create1(0)
+}
+
+fn do_epoll_create1(flags: c_int) -> Result<isize, Error> {
+    let fd = fs::do_epoll_create1(flags)?;
+    Ok(fd as isize)
+}
+
+fn do_epoll_ctl(
+    epfd: c_int,
+    op: c_int,
+    fd: c_int,
+    event: *const libc::epoll_event,
+) -> Result<isize, Error> {
+    if !event.is_null() {
+        check_ptr(event)?;
+    }
+    fs::do_epoll_ctl(epfd as FileDesc, op, fd as FileDesc, event)?;
+    Ok(0)
+}
+
+fn do_epoll_wait(
+    epfd: c_int,
+    events: *mut libc::epoll_event,
+    maxevents: c_int,
+    timeout: c_int,
+) -> Result<isize, Error> {
+    let maxevents = {
+        if maxevents <= 0 {
+            return errno!(EINVAL, "maxevents <= 0");
+        }
+        maxevents as usize
+    };
+    let events = {
+        check_mut_array(events, maxevents)?;
+        unsafe { std::slice::from_raw_parts_mut(events, maxevents) }
+    };
+    let count = fs::do_epoll_wait(epfd as FileDesc, events, timeout)?;
+    Ok(count as isize)
+}
+
 fn do_uname(name: *mut utsname_t) -> Result<isize, Error> {
     check_mut_ptr(name)?;
     let name = unsafe { &mut *name };
     misc::do_uname(name).map(|_| 0)
 }
 
-fn do_prlimit(pid: pid_t, resource: u32, new_limit: *const rlimit_t, old_limit: *mut rlimit_t) -> Result<isize, Error> {
+fn do_prlimit(
+    pid: pid_t,
+    resource: u32,
+    new_limit: *const rlimit_t,
+    old_limit: *mut rlimit_t,
+) -> Result<isize, Error> {
     let resource = resource_t::from_u32(resource)?;
     let new_limit = {
         if new_limit != ptr::null() {
             check_ptr(new_limit)?;
             Some(unsafe { &*new_limit })
-        }
-        else {
+        } else {
             None
         }
     };
@@ -775,8 +1377,7 @@ fn do_prlimit(pid: pid_t, resource: u32, new_limit: *const rlimit_t, old_limit:
         if old_limit != ptr::null_mut() {
             check_mut_ptr(old_limit)?;
             Some(unsafe { &mut *old_limit })
-        }
-        else {
+        } else {
             None
         }
     };

src/libos/src/time/mod.rs

@@ -16,6 +16,12 @@ pub struct timeval_t {
     usec: suseconds_t,
 }
 
+impl timeval_t {
+    pub fn as_usec(&self) -> usize {
+        (self.sec * 1000000 + self.usec) as usize
+    }
+}
+
 pub fn do_gettimeofday() -> timeval_t {
     let mut tv: timeval_t = Default::default();
     unsafe {

src/libos/src/util/log.rs

@@ -22,12 +22,11 @@ impl Log for SimpleLogger {
     fn log(&self, record: &Record) {
         if self.enabled(record.metadata()) {
             let color = Color::from(record.level());
-            let (show, code) = color.to_console_code();
             println!(
-                "\u{1B}[{};{}m[{:>5}] {}\u{1B}[0m",
-                show,
-                code + 30,
+                "\u{1B}[{}m[{:>5}][{}] {}\u{1B}[0m",
+                color as u8,
                 record.level(),
+                crate::process::current_pid(),
                 record.args()
             );
         }
@@ -42,7 +41,7 @@ impl From<Level> for Color {
             Level::Warn => Color::Yellow,
             Level::Info => Color::Blue,
             Level::Debug => Color::Green,
-            Level::Trace => Color::DarkGray,
+            Level::Trace => Color::BrightBlack,
         }
     }
 }
@@ -51,43 +50,20 @@ impl From<Level> for Color {
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 #[repr(u8)]
 pub enum Color {
-    Black = 0,
-    Blue = 1,
-    Green = 2,
-    Cyan = 3,
-    Red = 4,
-    Magenta = 5,
-    Brown = 6,
-    LightGray = 7,
-    DarkGray = 8,
-    LightBlue = 9,
-    LightGreen = 10,
-    LightCyan = 11,
-    LightRed = 12,
-    Pink = 13,
-    Yellow = 14,
-    White = 15,
-}
-
-impl Color {
-    fn to_console_code(&self) -> (u8, u8) {
-        match self {
-            Color::Black => (0, 0),
-            Color::Blue => (0, 4),
-            Color::Green => (0, 2),
-            Color::Cyan => (0, 6),
-            Color::Red => (0, 1),
-            Color::Magenta => (0, 5),
-            Color::Brown => (0, 3),
-            Color::LightGray => (1, 7),
-            Color::DarkGray => (0, 7),
-            Color::LightBlue => (1, 4),
-            Color::LightGreen => (1, 2),
-            Color::LightCyan => (1, 6),
-            Color::LightRed => (1, 1),
-            Color::Pink => (1, 5),
-            Color::Yellow => (1, 3),
-            Color::White => (1, 0),
-        }
-    }
+    Black = 30,
+    Red = 31,
+    Green = 32,
+    Yellow = 33,
+    Blue = 34,
+    Magenta = 35,
+    Cyan = 36,
+    White = 37,
+    BrightBlack = 90,
+    BrightRed = 91,
+    BrightGreen = 92,
+    BrightYellow = 93,
+    BrightBlue = 94,
+    BrightMagenta = 95,
+    BrightCyan = 96,
+    BrightWhite = 97,
 }

src/libos/src/util/ring_buf.rs

@@ -2,8 +2,8 @@ use alloc::alloc::{alloc, dealloc, Layout};
 
 use std::cmp::{max, min};
 use std::ptr;
-use std::sync::Arc;
 use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+use std::sync::Arc;
 
 use super::*;
 
@@ -155,6 +155,20 @@ impl RingBufReader {
         }
         Ok(buf_pos)
     }
+
+    pub fn can_read(&self) -> bool {
+        self.bytes_to_read() != 0
+    }
+
+    pub fn bytes_to_read(&self) -> usize {
+        let tail = self.inner.get_tail();
+        let head = self.inner.get_head();
+        if tail <= head {
+            head - tail
+        } else {
+            self.inner.capacity - tail + head
+        }
+    }
 }
 
 impl Drop for RingBufReader {
@@ -202,4 +216,15 @@ impl RingBufWriter {
         }
         Ok(buf_pos)
     }
+
+    pub fn can_write(&self) -> bool {
+        let tail = self.inner.get_tail();
+        let head = self.inner.get_head();
+        let may_write_nbytes = if tail <= head {
+            self.inner.capacity - head
+        } else {
+            tail - head - 1
+        };
+        may_write_nbytes != 0
+    }
 }

src/libos/src/vm/mod.rs

@@ -1,21 +1,26 @@
-use fs::{off_t, FileDesc};
+use fs::FileDesc;
 use prelude::*;
 use process::{get_current, Process, ProcessRef};
 use std::fmt;
 
 #[macro_use]
 mod vm_range;
-mod vm_area;
 mod process_vm;
+mod vm_area;
 
-pub use self::vm_range::{VMRange, VMRangeTrait};
-pub use self::vm_area::{VMSpace, VMDomain, VMArea, VMAreaFlags, VM_AREA_FLAG_R, VM_AREA_FLAG_W, VM_AREA_FLAG_X};
 pub use self::process_vm::ProcessVM;
-
+pub use self::vm_area::{
+    VMArea, VMAreaFlags, VMDomain, VMSpace, VM_AREA_FLAG_R, VM_AREA_FLAG_W, VM_AREA_FLAG_X,
+};
+pub use self::vm_range::{VMRange, VMRangeTrait};
 
 // TODO: separate proc and flags
 // TODO: accept fd and offset
 pub fn do_mmap(addr: usize, size: usize, flags: VMAreaFlags) -> Result<usize, Error> {
+    info!(
+        "mmap: addr: {:#x}, size: {:#x}, flags: {:?}",
+        addr, size, flags
+    );
     let current_ref = get_current();
     let current_process = current_ref.lock().unwrap();
     let current_vm_ref = current_process.get_vm();
@@ -24,6 +29,7 @@ pub fn do_mmap(addr: usize, size: usize, flags: VMAreaFlags) -> Result<usize, Er
 }
 
 pub fn do_munmap(addr: usize, size: usize) -> Result<(), Error> {
+    info!("munmap: addr: {:#x}, size: {:#x}", addr, size);
     let current_ref = get_current();
     let current_process = current_ref.lock().unwrap();
     let current_vm_ref = current_process.get_vm();
@@ -37,6 +43,10 @@ pub fn do_mremap(
     old_size: usize,
     options: &VMResizeOptions,
 ) -> Result<usize, Error> {
+    info!(
+        "mremap: oldaddr: {:#x}, oldsize: {:#x}, options: {:?}",
+        old_addr, old_size, options
+    );
     let current_ref = get_current();
     let current_process = current_ref.lock().unwrap();
     let current_vm_ref = current_process.get_vm();
@@ -45,6 +55,7 @@ pub fn do_mremap(
 }
 
 pub fn do_brk(addr: usize) -> Result<usize, Error> {
+    info!("brk: addr: {:#x}", addr);
     let current_ref = get_current();
     let current_process = current_ref.lock().unwrap();
     let current_vm_ref = current_process.get_vm();
@@ -61,7 +72,6 @@ pub enum VMGuardAreaType {
     Dynamic { size: usize },
 }
 
-
 #[derive(Clone, PartialEq, Default)]
 pub struct VMAllocOptions {
     size: usize,
@@ -74,7 +84,7 @@ pub struct VMAllocOptions {
 impl VMAllocOptions {
     pub fn new(size: usize) -> Result<VMAllocOptions, Error> {
         if size % PAGE_SIZE != 0 {
-            return Err(Error::new(Errno::EINVAL, "Size is not page-aligned"));
+            return errno!(EINVAL, "Size is not page-aligned");
         }
         Ok(VMAllocOptions {
             size,
@@ -84,7 +94,7 @@ impl VMAllocOptions {
 
     pub fn addr(&mut self, addr: VMAddrOption) -> Result<&mut Self, Error> {
         if addr.is_addr_given() && addr.get_addr() % PAGE_SIZE != 0 {
-            return Err(Error::new(Errno::EINVAL, "Invalid address"));
+            return errno!(EINVAL, "Invalid address");
         }
         self.addr = addr;
         Ok(self)
@@ -116,7 +126,6 @@ impl fmt::Debug for VMAllocOptions {
     }
 }
 
-
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub enum VMAddrOption {
     Any,           // Free to choose any address
@@ -149,7 +158,6 @@ impl VMAddrOption {
     }
 }
 
-
 /// How VMRange may grow:
 #[derive(Clone, Copy, Debug, PartialEq)]
 pub enum VMGrowthType {
@@ -164,7 +172,6 @@ impl Default for VMGrowthType {
     }
 }
 
-
 #[derive(Clone, Debug, Default)]
 pub struct VMResizeOptions {
     new_size: usize,
@@ -175,7 +182,7 @@ pub struct VMResizeOptions {
 impl VMResizeOptions {
     pub fn new(new_size: usize) -> Result<VMResizeOptions, Error> {
         if new_size % PAGE_SIZE != 0 {
-            return Err(Error::new(Errno::EINVAL, "Size is not page-aligned"));
+            return errno!(EINVAL, "Size is not page-aligned");
         }
         Ok(VMResizeOptions {
             new_size,

src/libos/src/vm/process_vm.rs

@@ -46,22 +46,27 @@ impl ProcessVM {
     ) -> Result<ProcessVM, Error> {
         // Allocate the data domain from the global data space
         let mut data_domain = {
-            let data_domain_size = code_size + data_size + heap_size
-                                 + stack_size + mmap_size;
-            let data_domain = DATA_SPACE.lock().unwrap().alloc_domain(
-                                    data_domain_size, "data_domain")?;
+            let data_domain_size = code_size + data_size + heap_size + stack_size + mmap_size;
+            let data_domain = DATA_SPACE
+                .lock()
+                .unwrap()
+                .alloc_domain(data_domain_size, "data_domain")?;
             data_domain
         };
         // Allocate vmas from the data domain
-        let (code_vma, data_vma, heap_vma, stack_vma) =
-            match ProcessVM::alloc_vmas(&mut data_domain, code_size,
-                                        data_size, heap_size, stack_size) {
+        let (code_vma, data_vma, heap_vma, stack_vma) = match ProcessVM::alloc_vmas(
+            &mut data_domain,
+            code_size,
+            data_size,
+            heap_size,
+            stack_size,
+        ) {
             Err(e) => {
                 // Note: we need to handle error here so that we can
                 // deallocate the data domain explictly.
                 DATA_SPACE.lock().unwrap().dealloc_domain(data_domain);
                 return Err(e);
-                },
+            }
             Ok(vmas) => vmas,
         };
         // Initial value of the program break
@@ -92,7 +97,8 @@ impl ProcessVM {
         let mut alloc_vma_continuously =
             |addr: &mut usize, desc, size, flags, growth, fill_zeros| -> Result<_, Error> {
                 let mut options = VMAllocOptions::new(size)?;
-                options.addr(VMAddrOption::Fixed(*addr))?
+                options
+                    .addr(VMAddrOption::Fixed(*addr))?
                     .growth(growth)?
                     .description(desc)?
                     .fill_zeros(fill_zeros)?;
@@ -104,17 +110,41 @@ impl ProcessVM {
         let rx_flags = VMAreaFlags(VM_AREA_FLAG_R | VM_AREA_FLAG_X);
         let rw_flags = VMAreaFlags(VM_AREA_FLAG_R | VM_AREA_FLAG_W);
 
-        let code_vma = alloc_vma_continuously(&mut addr, "code_vma", code_size,
-                                              rx_flags, VMGrowthType::Fixed, true)?;
-        let data_vma = alloc_vma_continuously(&mut addr, "data_vma", data_size,
-                                              rw_flags, VMGrowthType::Fixed, true)?;
-        let heap_vma = alloc_vma_continuously(&mut addr, "heap_vma", 0,
-                                              rw_flags, VMGrowthType::Upward, true)?;
+        let code_vma = alloc_vma_continuously(
+            &mut addr,
+            "code_vma",
+            code_size,
+            rx_flags,
+            VMGrowthType::Fixed,
+            !cfg!(feature = "integrity_only_opt"),
+        )?;
+        let data_vma = alloc_vma_continuously(
+            &mut addr,
+            "data_vma",
+            data_size,
+            rw_flags,
+            VMGrowthType::Fixed,
+            !cfg!(feature = "integrity_only_opt"),
+        )?;
+        let heap_vma = alloc_vma_continuously(
+            &mut addr,
+            "heap_vma",
+            0,
+            rw_flags,
+            VMGrowthType::Upward,
+            true,
+        )?;
         // Preserve the space for heap
         addr += heap_size;
         // After the heap is the stack
-        let stack_vma = alloc_vma_continuously(&mut addr, "stack_vma", stack_size,
-                                               rw_flags, VMGrowthType::Downward, false)?;
+        let stack_vma = alloc_vma_continuously(
+            &mut addr,
+            "stack_vma",
+            stack_size,
+            rw_flags,
+            VMGrowthType::Downward,
+            false,
+        )?;
         Ok((code_vma, data_vma, heap_vma, stack_vma))
     }
 
@@ -169,7 +199,7 @@ impl ProcessVM {
                     VMAddrOption::Beyond(mmap_start_addr)
                 } else {
                     if addr < mmap_start_addr {
-                        return Err(Error::new(Errno::EINVAL, "Beyond valid memory range"));
+                        return errno!(EINVAL, "Beyond valid memory range");
                     }
                     // TODO: Fixed or Hint? Should hanle mmap flags
                     VMAddrOption::Hint(addr)
@@ -178,7 +208,8 @@ impl ProcessVM {
             alloc_options
         };
         // TODO: when failed, try to resize data_domain
-        let new_mmap_vma = self.get_data_domain_mut()
+        let new_mmap_vma = self
+            .get_data_domain_mut()
             .alloc_area(&alloc_options, flags)?;
         let addr = new_mmap_vma.get_start();
         self.mmap_vmas.push(Box::new(new_mmap_vma));
@@ -201,7 +232,8 @@ impl ProcessVM {
         };
 
         let removed_mmap_vma = self.mmap_vmas.swap_remove(mmap_vma_i);
-        self.get_data_domain_mut().dealloc_area(unbox(removed_mmap_vma));
+        self.get_data_domain_mut()
+            .dealloc_area(unbox(removed_mmap_vma));
         Ok(())
     }
 
@@ -212,7 +244,7 @@ impl ProcessVM {
         options: &VMResizeOptions,
     ) -> Result<usize, Error> {
         // TODO: Implement this!
-        Err(Error::new(Errno::EINVAL, "Not implemented"))
+        errno!(EINVAL, "Not implemented")
     }
 
     pub fn brk(&mut self, new_brk: usize) -> Result<usize, Error> {
@@ -229,7 +261,8 @@ impl ProcessVM {
                 let new_heap_end = align_up(new_brk, PAGE_SIZE);
                 let new_heap_size = new_heap_end - heap_start;
                 let mut options = VMResizeOptions::new(new_heap_size)?;
-                options.addr(VMAddrOption::Fixed(heap_start))
+                options
+                    .addr(VMAddrOption::Fixed(heap_start))
                     .fill_zeros(true);
                 options
             };
@@ -261,7 +294,9 @@ impl Drop for ProcessVM {
         }
 
         // Remove the domain from its parent space
-        DATA_SPACE.lock().unwrap().dealloc_domain(
-            unbox(self.data_domain.take().unwrap()));
+        DATA_SPACE
+            .lock()
+            .unwrap()
+            .dealloc_domain(unbox(self.data_domain.take().unwrap()));
     }
 }

src/libos/src/vm/vm_area.rs

@@ -27,8 +27,7 @@ impl VMSpace {
 
     pub fn alloc_domain(&mut self, size: usize, desc: &str) -> Result<VMDomain, Error> {
         let mut options = VMAllocOptions::new(size)?;
-        options.growth(VMGrowthType::Upward)?
-            .description(desc)?;
+        options.growth(VMGrowthType::Upward)?.description(desc)?;
 
         let new_range = self.range.alloc_subrange(&options)?;
         Ok(VMDomain { range: new_range })
@@ -44,7 +43,6 @@ impl VMSpace {
     }
 }
 
-
 #[derive(Debug)]
 pub struct VMDomain {
     range: VMRange,
@@ -78,7 +76,6 @@ impl VMDomain {
     }
 }
 
-
 #[derive(Debug)]
 pub struct VMArea {
     range: VMRange,
@@ -97,7 +94,6 @@ impl VMArea {
     }
 }
 
-
 #[derive(Copy, Clone, Debug, Default, PartialEq)]
 pub struct VMAreaFlags(pub u32);
 

src/libos/src/vm/vm_range.rs

@@ -46,7 +46,12 @@ pub struct VMRange {
 impl_vmrange_trait_for!(VMRange, inner);
 
 impl VMRange {
-    pub unsafe fn new(start: usize, end: usize, growth: VMGrowthType, description: &str) -> Result<VMRange, Error> {
+    pub unsafe fn new(
+        start: usize,
+        end: usize,
+        growth: VMGrowthType,
+        description: &str,
+    ) -> Result<VMRange, Error> {
         if start % PAGE_SIZE != 0 || end % PAGE_SIZE != 0 {
             return errno!(EINVAL, "Invalid start and/or end");
         }
@@ -75,8 +80,10 @@ impl VMRange {
             debug_assert!(free_space.contains(new_subrange_start));
             debug_assert!(free_space.contains(new_subrange_end));
 
-            (free_space.index_in_subranges, VMRangeInner::new(
-                    new_subrange_start, new_subrange_end, options.growth))
+            (
+                free_space.index_in_subranges,
+                VMRangeInner::new(new_subrange_start, new_subrange_end, options.growth),
+            )
         };
         self.get_subranges_mut()
             .insert(new_subrange_idx, new_subrange_inner);
@@ -186,7 +193,7 @@ impl VMRange {
             let pre_range = &range_pair[0];
             let next_range = &range_pair[1];
 
-            let (free_range_start, free_range_end)= {
+            let (free_range_start, free_range_end) = {
                 let free_range_start = pre_range.get_end();
                 let free_range_end = next_range.get_start();
 
@@ -209,7 +216,7 @@ impl VMRange {
 
             match addr {
                 // Want a minimal free_space
-                VMAddrOption::Any => { },
+                VMAddrOption::Any => {}
                 // Prefer to have free_space.start == addr
                 VMAddrOption::Hint(addr) => {
                     if free_space.contains(addr) {
@@ -218,7 +225,7 @@ impl VMRange {
                             return Ok(free_space);
                         }
                     }
-                },
+                }
                 // Must have free_space.start == addr
                 VMAddrOption::Fixed(addr) => {
                     if !free_space.contains(addr) {
@@ -241,20 +248,24 @@ impl VMRange {
                             continue;
                         }
                     }
-                },
+                }
             }
 
-            if min_big_enough_free_space == None ||
-                free_space < *min_big_enough_free_space.as_ref().unwrap() {
+            if min_big_enough_free_space == None
+                || free_space < *min_big_enough_free_space.as_ref().unwrap()
+            {
                 min_big_enough_free_space = Some(free_space);
             }
         }
 
-        min_big_enough_free_space
-            .ok_or_else(|| Error::new(Errno::ENOMEM, "not enough space"))
+        min_big_enough_free_space.ok_or_else(|| Error::new(Errno::ENOMEM, "not enough space"))
     }
 
-    fn alloc_from_free_space(&self, free_space: &FreeSpace, options: &VMAllocOptions) -> (usize, usize) {
+    fn alloc_from_free_space(
+        &self,
+        free_space: &FreeSpace,
+        options: &VMAllocOptions,
+    ) -> (usize, usize) {
         // Get valid parameters from options
         let size = options.size;
         let addr_option = options.addr;
@@ -262,8 +273,7 @@ impl VMRange {
 
         if let VMAddrOption::Fixed(addr) = addr_option {
             return (addr, addr + size);
-        }
-        else if let VMAddrOption::Hint(addr) = addr_option {
+        } else if let VMAddrOption::Hint(addr) = addr_option {
             if free_space.start == addr {
                 return (addr, addr + size);
             }
@@ -349,7 +359,12 @@ impl VMRange {
         Ok(())
     }
 
-    fn grow_subrange_to(&mut self, subrange: &mut VMRange, new_size: usize, fill_zeros: bool) -> Result<(), Error> {
+    fn grow_subrange_to(
+        &mut self,
+        subrange: &mut VMRange,
+        new_size: usize,
+        fill_zeros: bool,
+    ) -> Result<(), Error> {
         let subrange_i = self.position_subrange(subrange);
         let subranges = self.get_subranges_mut();
 
@@ -379,7 +394,8 @@ impl VMRange {
                     memset(mem_ptr, 0 as c_int, mem_size);
                 }
             }
-        } else { // self.growth == VMGrowthType::Downward
+        } else {
+            // self.growth == VMGrowthType::Downward
             // Can we grow downard?
             let max_new_size = {
                 let pre_subrange = &subranges[subrange_i - 1];
@@ -429,7 +445,6 @@ impl Drop for VMRange {
 unsafe impl Send for VMRange {}
 unsafe impl Sync for VMRange {}
 
-
 #[derive(Clone, Copy)]
 pub struct VMRangeInner {
     start: usize,

src/libos/src/vm/vm_space_prealloced.c

@@ -1,6 +1,6 @@
 #include <stddef.h>
 
-#define DATA_SPACE_SIZE (32*1024*1024)
+#define DATA_SPACE_SIZE (128*1024*1024)
 
 static char __prealloced_data_space[DATA_SPACE_SIZE]
     __attribute__ ((

src/pal/pal.c

@@ -213,6 +213,9 @@ void ocall_sync(void) {
 /* Application entry */
 int SGX_CDECL main(int argc, const char *argv[])
 {
+    struct timeval startup, libosready, appdie;
+
+    gettimeofday(&startup, NULL);
     sgx_status_t sgx_ret = SGX_SUCCESS;
     int status = 0;
     uint32_t sealed_log_size = 1024;
@@ -238,9 +241,22 @@ int SGX_CDECL main(int argc, const char *argv[])
         print_error_message(sgx_ret);
         return status;
     }
+    // First ecall do a lot initializations.
+    // Count it as startup time.
+    dummy_ecall(global_eid, &status);
+
+    gettimeofday(&libosready, NULL);
 
     status = wait_all_tasks();
 
+    gettimeofday(&appdie, NULL);
+
+    uint64_t libos_startup_time, app_runtime;
+    libos_startup_time = (libosready.tv_sec - startup.tv_sec) * 1000000 + (libosready.tv_usec - startup.tv_usec);
+    app_runtime = (appdie.tv_sec - libosready.tv_sec) * 1000000 + (appdie.tv_usec - libosready.tv_usec);
+    printf("LibOS startup time: %d microseconds\n", libos_startup_time);
+    printf("Apps running time: %d microseconds\n", app_runtime);
+
     /* Destroy the enclave */
     sgx_destroy_enclave(global_eid);
 

src/sgxenv.mk

@@ -38,7 +38,7 @@ else
 endif
 
 RUST_SGX_SDK_DIR := $(PROJECT_DIR)/deps/rust-sgx-sdk
-SGX_COMMON_CFLAGS += -I$(RUST_SGX_SDK_DIR)/common/ -I$(RUST_SGX_SDK_DIR)/edl/
+SGX_COMMON_CFLAGS += -I$(RUST_SGX_SDK_DIR)/common/ -I$(RUST_SGX_SDK_DIR)/common/inc/ -I$(RUST_SGX_SDK_DIR)/edl/
 
 ifneq ($(SGX_MODE), HW)
 	Urts_Library_Name := sgx_urts_sim

test/Makefile

@@ -4,9 +4,9 @@ PROJECT_DIR := $(realpath $(CUR_DIR)/../)
 # Dependencies: need to be compiled but not to run by any Makefile target
 TEST_DEPS := dev_null
 # Tests: need to be compiled and run by test-% target
-TESTS := empty argv hello_world malloc file getpid spawn pipe time truncate readdir mkdir link tls pthread uname rlimit
+TESTS := empty argv hello_world malloc file getpid spawn pipe time truncate readdir mkdir link tls pthread uname rlimit client server server_epoll unix_socket
 # Benchmarks: need to be compiled and run by bench-% target
-BENCHES := spawn_and_exit_latency pipe_throughput
+BENCHES := spawn_and_exit_latency pipe_throughput unix_socket_throughput
 
 # Top-level Makefile targets
 BUILD_TARGETS := $(TEST_DEPS) $(TESTS) $(BENCHES)

test/client/Makefile

@@ -0,0 +1,5 @@
+include ../test_common.mk
+
+EXTRA_C_FLAGS :=
+EXTRA_LINK_FLAGS :=
+BIN_ARGS :=

test/client/main.c

@@ -0,0 +1,52 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <spawn.h>
+
+int main(int argc, const char *argv[]) {
+	const int BUF_SIZE = 0x1000;
+	const char* message = "Hello world!";
+	int ret;
+
+	if (argc != 2) {
+		printf("usage: ./client <ipaddress>\n");
+		return 0;
+	}
+
+	int sockfd = socket(AF_INET, SOCK_STREAM, 0);
+	if (sockfd < 0) {
+		printf("create socket error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	struct sockaddr_in servaddr;
+	memset(&servaddr, 0, sizeof(servaddr));
+	servaddr.sin_family = AF_INET;
+	servaddr.sin_port = htons(6666);
+
+	ret = inet_pton(AF_INET, argv[1], &servaddr.sin_addr);
+	if (ret <= 0) {
+		printf("inet_pton error for %s\n", argv[1]);
+		return -1;
+	}
+
+	ret = connect(sockfd, (struct sockaddr *) &servaddr, sizeof(servaddr));
+	if (ret < 0) {
+		printf("connect error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	printf("send msg to server: %s\n", message);
+	ret = send(sockfd, message, strlen(message), 0);
+	if (ret < 0) {
+		printf("send msg error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	close(sockfd);
+	return 0;
+}

test/server/Makefile

@@ -0,0 +1,5 @@
+include ../test_common.mk
+
+EXTRA_C_FLAGS :=
+EXTRA_LINK_FLAGS :=
+BIN_ARGS :=

test/server/main.c

@@ -0,0 +1,59 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <spawn.h>
+
+int main(int argc, const char *argv[]) {
+	const int BUF_SIZE = 0x1000;
+	int ret;
+
+	int listenfd = socket(AF_INET, SOCK_STREAM, 0);
+	if (listenfd < 0) {
+		printf("create socket error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	struct sockaddr_in servaddr;
+	memset(&servaddr, 0, sizeof(servaddr));
+	servaddr.sin_family = AF_INET;
+	servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
+	servaddr.sin_port = htons(6666);
+
+	ret = bind(listenfd, (struct sockaddr *) &servaddr, sizeof(servaddr));
+	if (ret < 0) {
+		printf("bind socket error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	ret = listen(listenfd, 10);
+	if (ret < 0) {
+		printf("listen socket error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	int client_pid;
+	char* client_argv[] = {"client", "127.0.0.1"};
+	ret = posix_spawn(&client_pid, "client", NULL, NULL, client_argv, NULL);
+	if (ret < 0) {
+		printf("spawn client process error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	printf("====== waiting for client's request ======\n");
+	int connect_fd = accept(listenfd, (struct sockaddr *) NULL, NULL);
+	if (connect_fd < 0) {
+		printf("accept socket error: %s(errno: %d)", strerror(errno), errno);
+		return -1;
+	}
+	char buff[BUF_SIZE];
+	int n = recv(connect_fd, buff, BUF_SIZE, 0);
+	buff[n] = '\0';
+	printf("recv msg from client: %s\n", buff);
+	close(connect_fd);
+
+	close(listenfd);
+}

test/server_epoll/Makefile

@@ -0,0 +1,5 @@
+include ../test_common.mk
+
+EXTRA_C_FLAGS :=
+EXTRA_LINK_FLAGS :=
+BIN_ARGS :=

test/server_epoll/main.c

@@ -0,0 +1,186 @@
+// Modified from https://banu.com/blog/2/how-to-use-epoll-a-complete-example-in-c/epoll-example.c
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <netdb.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/epoll.h>
+#include <errno.h>
+
+#define MAXEVENTS 64
+
+static int
+create_and_bind() {
+	int listenfd = socket(AF_INET, SOCK_STREAM | SOCK_NONBLOCK, 0);
+	if (listenfd < 0) {
+		printf("create socket error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+
+	struct sockaddr_in servaddr;
+	memset(&servaddr, 0, sizeof(servaddr));
+	servaddr.sin_family = AF_INET;
+	servaddr.sin_addr.s_addr = htonl(INADDR_ANY);
+	servaddr.sin_port = htons(6666);
+
+	int ret = bind(listenfd, (struct sockaddr *) &servaddr, sizeof(servaddr));
+	if (ret < 0) {
+		printf("bind socket error: %s(errno: %d)\n", strerror(errno), errno);
+		return -1;
+	}
+	return listenfd;
+}
+
+int
+main(int argc, char *argv[]) {
+	int sfd = create_and_bind();
+
+	int s = listen(sfd, SOMAXCONN);
+	if (s == -1) {
+		perror("listen");
+		return -1;
+	}
+
+	int efd = epoll_create1(0);
+	if (efd == -1) {
+		perror("epoll_create");
+		return -1;
+	}
+
+	struct epoll_event event;
+	event.data.fd = sfd;
+	event.events = EPOLLIN | EPOLLET;
+	s = epoll_ctl(efd, EPOLL_CTL_ADD, sfd, &event);
+	if (s == -1) {
+		perror("epoll_ctl");
+		return -1;
+	}
+
+	/* Buffer where events are returned */
+	struct epoll_event *events = calloc(MAXEVENTS, sizeof event);
+
+	// spawn clients
+	int client_pid;
+	char* client_argv[] = {"client", "127.0.0.1"};
+	for(int i=0; i<3; ++i) {
+		int ret = posix_spawn(&client_pid, "client", NULL, NULL, client_argv, NULL);
+		if (ret < 0) {
+			printf("spawn client process error: %s(errno: %d)\n", strerror(errno), errno);
+			return -1;
+		}
+	}
+
+	/* The event loop */
+	int done_count = 0;
+	while (done_count < 3) {
+		int n = epoll_wait(efd, events, MAXEVENTS, -1);
+		for (int i = 0; i < n; i++) {
+			if ((events[i].events & EPOLLERR) ||
+				(events[i].events & EPOLLHUP) ||
+				(!(events[i].events & EPOLLIN))) {
+				/* An error has occured on this fd, or the socket is not
+				   ready for reading (why were we notified then?) */
+				fprintf(stderr, "epoll error\n");
+				close(events[i].data.fd);
+				continue;
+			} else if (sfd == events[i].data.fd) {
+				/* We have a notification on the listening socket, which
+				   means one or more incoming connections. */
+				while (1) {
+					struct sockaddr in_addr;
+					socklen_t in_len;
+					int infd;
+					char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
+
+					in_len = sizeof in_addr;
+					infd = accept4(sfd, &in_addr, &in_len, SOCK_NONBLOCK);
+					if (infd == -1) {
+						if ((errno == EAGAIN) ||
+							(errno == EWOULDBLOCK)) {
+							/* We have processed all incoming
+							   connections. */
+							break;
+						} else {
+							perror("accept");
+							break;
+						}
+					}
+
+					s = getnameinfo(&in_addr, in_len,
+									hbuf, sizeof hbuf,
+									sbuf, sizeof sbuf,
+									NI_NUMERICHOST | NI_NUMERICSERV);
+					if (s == 0) {
+						printf("Accepted connection on descriptor %d "
+							   "(host=%s, port=%s)\n", infd, hbuf, sbuf);
+					}
+
+					// add it to the list of fds to monitor
+					event.data.fd = infd;
+					event.events = EPOLLIN | EPOLLET;
+					s = epoll_ctl(efd, EPOLL_CTL_ADD, infd, &event);
+					if (s == -1) {
+						perror("epoll_ctl");
+						return -1;
+					}
+				}
+				continue;
+			} else {
+				/* We have data on the fd waiting to be read. Read and
+				   display it. We must read whatever data is available
+				   completely, as we are running in edge-triggered mode
+				   and won't get a notification again for the same
+				   data. */
+				int done = 0;
+
+				while (1) {
+					ssize_t count;
+					char buf[512];
+
+					count = read(events[i].data.fd, buf, sizeof buf);
+					if (count == -1) {
+						/* If errno == EAGAIN, that means we have read all
+						   data. So go back to the main loop. */
+						if (errno != EAGAIN) {
+							perror("read");
+							done = 1;
+						}
+						break;
+					} else if (count == 0) {
+						/* End of file. The remote has closed the
+						   connection. */
+						done = 1;
+						break;
+					}
+
+					/* Write the buffer to standard output */
+					s = write(1, buf, count);
+					if (s == -1) {
+						perror("write");
+						return -1;
+					}
+				}
+
+				if (done) {
+					printf("Closed connection on descriptor %d\n",
+						   events[i].data.fd);
+
+					/* Closing the descriptor will make epoll remove it
+					   from the set of descriptors which are monitored. */
+					close(events[i].data.fd);
+
+					done_count ++;
+				}
+			}
+		}
+	}
+
+	free(events);
+
+	close(sfd);
+
+	return EXIT_SUCCESS;
+}

test/unix_socket/Makefile

@@ -0,0 +1,5 @@
+include ../test_common.mk
+
+EXTRA_C_FLAGS := -Wno-incompatible-pointer-types-discards-qualifiers
+EXTRA_LINK_FLAGS :=
+BIN_ARGS :=

test/unix_socket/main.c

@@ -0,0 +1,113 @@
+#include <sys/syscall.h>
+#include <sys/wait.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <spawn.h>
+#include <string.h>
+
+const char SOCK_PATH[] = "echo_socket";
+
+int create_server_socket() {
+	int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd == -1) {
+		printf("ERROR: failed to create a unix socket\n");
+		return -1;
+	}
+
+	struct sockaddr_un local;
+	local.sun_family = AF_UNIX;
+	strcpy(local.sun_path, SOCK_PATH);
+	socklen_t len = strlen(local.sun_path) + sizeof(local.sun_family);
+
+	if (bind(fd, (struct sockaddr *)&local, len) == -1) {
+		printf("ERROR: failed to bind\n");
+		return -1;
+	}
+
+	if (listen(fd, 5) == -1) {
+		printf("ERROR: failed to listen\n");
+		return -1;
+	}
+	return fd;
+}
+
+int create_client_socket() {
+	int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd == -1) {
+		printf("ERROR: failed to create a unix socket\n");
+		return -1;
+	}
+
+	struct sockaddr_un remote;
+	remote.sun_family = AF_UNIX;
+	strcpy(remote.sun_path, SOCK_PATH);
+	socklen_t len = strlen(remote.sun_path) + sizeof(remote.sun_family);
+
+	if (connect(fd, (struct sockaddr *)&remote, len) == -1) {
+		printf("ERROR: failed to connect\n");
+		return -1;
+	}
+	return fd;
+}
+
+int main(int argc, const char* argv[]) {
+	int listen_fd = create_server_socket();
+	if (listen_fd == -1) {
+		printf("ERROR: failed to create server socket\n");
+		return -1;
+	}
+
+	int socket_rd_fd = create_client_socket();
+	if (socket_rd_fd == -1) {
+		printf("ERROR: failed to create client socket\n");
+		return -1;
+	}
+
+	struct sockaddr_un remote;
+	socklen_t len = sizeof(remote);
+	int socket_wr_fd = accept(listen_fd, (struct sockaddr *)&remote, &len);
+	if (socket_wr_fd == -1) {
+		printf("ERROR: failed to accept socket\n");
+		return -1;
+	}
+
+	// The following is same as 'pipe'
+
+    posix_spawn_file_actions_t file_actions;
+    posix_spawn_file_actions_init(&file_actions);
+    posix_spawn_file_actions_adddup2(&file_actions, socket_wr_fd, STDOUT_FILENO);
+    posix_spawn_file_actions_addclose(&file_actions, socket_rd_fd);
+
+    const char* msg = "Echo!\n";
+    const char* child_prog = "hello_world";
+    const char* child_argv[3] = { child_prog, msg, NULL };
+    int child_pid;
+    if (posix_spawn(&child_pid, child_prog, &file_actions,
+            NULL, child_argv, NULL) < 0) {
+        printf("ERROR: failed to spawn a child process\n");
+        return -1;
+    }
+    close(socket_wr_fd);
+
+    const char* expected_str = msg;
+    size_t expected_len = strlen(expected_str);
+    char actual_str[32] = {0};
+    ssize_t actual_len;
+    do {
+        actual_len = read(socket_rd_fd, actual_str, sizeof(actual_str) - 1);
+    } while (actual_len == 0);
+    if (strncmp(expected_str, actual_str, expected_len) != 0) {
+        printf("ERROR: received string is not as expected\n");
+        return -1;
+    }
+
+    int status = 0;
+    if (wait4(child_pid, &status, 0, NULL) < 0) {
+        printf("ERROR: failed to wait4 the child process\n");
+        return -1;
+    }
+    return 0;
+}

test/unix_socket_throughput/Makefile

@@ -0,0 +1,5 @@
+include ../test_common.mk
+
+EXTRA_C_FLAGS := -Wno-incompatible-pointer-types-discards-qualifiers
+EXTRA_LINK_FLAGS :=
+BIN_ARGS :=

test/unix_socket_throughput/main.c

@@ -0,0 +1,170 @@
+#include <sys/syscall.h>
+#include <sys/time.h>
+#include <sys/wait.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <spawn.h>
+#include <string.h>
+
+#define KB              (1024UL)
+#define MB              (1024UL * 1024UL)
+#define GB              (1024UL * 1024UL * 1024UL)
+
+#define TOTAL_BYTES     (2 * GB)
+#define BUF_SIZE        (128 * KB)
+
+#define MIN(x, y)       ((x) <= (y) ? (x) : (y))
+
+const char SOCK_PATH[] = "echo_socket";
+
+int create_server_socket() {
+	int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd == -1) {
+		printf("ERROR: failed to create a unix socket\n");
+		return -1;
+	}
+
+	struct sockaddr_un local;
+	local.sun_family = AF_UNIX;
+	strcpy(local.sun_path, SOCK_PATH);
+	unlink(local.sun_path);
+	socklen_t len = strlen(local.sun_path) + sizeof(local.sun_family);
+
+	if (bind(fd, (struct sockaddr *)&local, len) == -1) {
+		printf("ERROR: failed to bind\n");
+		return -1;
+	}
+
+	if (listen(fd, 5) == -1) {
+		printf("ERROR: failed to listen\n");
+		return -1;
+	}
+	return fd;
+}
+
+int create_client_socket() {
+	int fd = socket(AF_UNIX, SOCK_STREAM, 0);
+	if (fd == -1) {
+		printf("ERROR: failed to create a unix socket\n");
+		return -1;
+	}
+
+	struct sockaddr_un remote;
+	remote.sun_family = AF_UNIX;
+	strcpy(remote.sun_path, SOCK_PATH);
+	socklen_t len = strlen(remote.sun_path) + sizeof(remote.sun_family);
+
+	if (connect(fd, (struct sockaddr *)&remote, len) == -1) {
+		printf("ERROR: failed to connect\n");
+		return -1;
+	}
+	return fd;
+}
+
+int main(int argc, const char* argv[]) {
+	size_t buf_size, total_bytes;
+	if (argc >= 2) {
+		buf_size = atol(argv[1]);
+	} else {
+		buf_size = BUF_SIZE;
+	}
+	if (argc >= 3) {
+		total_bytes = atol(argv[2]);
+	} else {
+		// BUG: throughput fall down when buf_size > 65536
+		total_bytes = buf_size > 65536? buf_size << 15: buf_size << 21;
+	}
+	printf("buf_size = 0x%zx\n", buf_size);
+	printf("total_bytes = 0x%zx\n", total_bytes);
+
+	int listen_fd = create_server_socket();
+	if (listen_fd == -1) {
+		printf("ERROR: failed to create server socket\n");
+		return -1;
+	}
+
+	int socket_rd_fd = create_client_socket();
+	if (socket_rd_fd == -1) {
+		printf("ERROR: failed to create client socket\n");
+		return -1;
+	}
+
+	struct sockaddr_un remote;
+	socklen_t len = sizeof(remote);
+	int socket_wr_fd = accept(listen_fd, (struct sockaddr *)&remote, &len);
+	if (socket_wr_fd == -1) {
+		printf("ERROR: failed to accept socket\n");
+		return -1;
+	}
+
+	// The following is same as 'pipe_throughput'
+
+	// Spawn a child process that reads from the pipe
+	posix_spawn_file_actions_t file_actions;
+	posix_spawn_file_actions_init(&file_actions);
+	posix_spawn_file_actions_adddup2(&file_actions, socket_rd_fd, STDIN_FILENO);
+	posix_spawn_file_actions_addclose(&file_actions, socket_wr_fd);
+
+	int child_pid;
+	extern char ** environ;
+	const char* new_argv[] = {"./dev_null", NULL};
+	if (posix_spawn(&child_pid, "dev_null", &file_actions,
+					NULL, new_argv, environ) < 0) {
+		printf("ERROR: failed to spawn a child process\n");
+		return -1;
+	}
+	close(socket_rd_fd);
+
+	// Start the timer
+	struct timeval tv_start, tv_end;
+	gettimeofday(&tv_start, NULL);
+
+	// Tell the reader how many data are to be transfered
+	size_t remain_bytes = total_bytes;
+	if (write(socket_wr_fd, &remain_bytes, sizeof(remain_bytes)) != sizeof(remain_bytes)) {
+		printf("ERROR: failed to write to pipe\n");
+		return -1;
+	}
+
+	// Tell the reader the buffer size that it should use
+	if (write(socket_wr_fd, &buf_size, sizeof(buf_size)) != sizeof(buf_size)) {
+		printf("ERROR: failed to write to pipe\n");
+		return -1;
+	}
+
+	// Write a specified amount of data in a buffer of specified size
+	char buf[BUF_SIZE] = {0};
+	while (remain_bytes > 0) {
+		size_t len = MIN(buf_size, remain_bytes);
+		if ((len = write(socket_wr_fd, &buf, len)) < 0) {
+			printf("ERROR: failed to write to pipe\n");
+			return -1;
+		}
+		remain_bytes -= len;
+	}
+
+	// Wait for the child process to read all data and exit
+	int status = 0;
+	if (wait4(child_pid, &status, 0, NULL) < 0) {
+		printf("ERROR: failed to wait4 the child process\n");
+		return -1;
+	}
+
+	// Stop the timer
+	gettimeofday(&tv_end, NULL);
+
+	// Calculate the throughput
+	double total_s = (tv_end.tv_sec - tv_start.tv_sec)
+					 + (double)(tv_end.tv_usec - tv_start.tv_usec) / 1000000;
+	if (total_s < 1.0) {
+		printf("WARNING: run long enough to get meaningful results\n");
+		if (total_s == 0) { return 0; }
+	}
+	double total_mb = (double)total_bytes / MB;
+	double throughput = total_mb / total_s;
+	printf("Throughput of unix socket is %.2f MB/s\n", throughput);
+	return 0;
+}