From e1648fc870e3a4f6300ed42b89a5d5e049c4ef6d Mon Sep 17 00:00:00 2001 From: LI Qing Date: Thu, 5 Mar 2020 04:50:39 +0000 Subject: [PATCH] Add the redirection of standard I/O for process --- demos/embedded_mode/bench_driver/main.c | 2 +- src/Enclave.edl | 3 +- src/libos/include/edl/occlum_edl_types.h | 6 + src/libos/src/entry.rs | 42 +++-- src/libos/src/fs/mod.rs | 2 +- src/libos/src/fs/stdio.rs | 190 +++++++++++++++++++++-- src/libos/src/process/spawn/mod.rs | 39 +++-- src/libos/src/syscall/mod.rs | 3 +- src/pal/include/edl/occlum_edl_types.h | 1 + src/pal/include/occlum_pal_api.h | 17 +- src/pal/src/pal_api.c | 7 +- src/run/main.c | 8 +- 12 files changed, 274 insertions(+), 46 deletions(-) diff --git a/demos/embedded_mode/bench_driver/main.c b/demos/embedded_mode/bench_driver/main.c index 615c2bdc..f9d2e336 100644 --- a/demos/embedded_mode/bench_driver/main.c +++ b/demos/embedded_mode/bench_driver/main.c @@ -65,7 +65,7 @@ int main(int argc, char* argv[]) { // Use Occlum PAL to execute the cmd int exit_status = 0; - if (occlum_pal_exec(cmd_path, cmd_args, &exit_status) < 0) { + if (occlum_pal_exec(cmd_path, cmd_args, NULL, &exit_status) < 0) { return EXIT_FAILURE; } diff --git a/src/Enclave.edl b/src/Enclave.edl index 3f32a7ac..3bf97fc4 100644 --- a/src/Enclave.edl +++ b/src/Enclave.edl @@ -27,7 +27,8 @@ enclave { */ public int occlum_ecall_new_process( [in, string] const char* executable_path, - [user_check] const char** argv); + [user_check] const char** argv, + [in] const struct occlum_stdio_fds* io_fds); /* * Execute the LibOS thread specified by the TID. diff --git a/src/libos/include/edl/occlum_edl_types.h b/src/libos/include/edl/occlum_edl_types.h index ec7bdcb2..98160112 100644 --- a/src/libos/include/edl/occlum_edl_types.h +++ b/src/libos/include/edl/occlum_edl_types.h @@ -10,4 +10,10 @@ struct timeval { suseconds_t tv_usec; /* microseconds */ }; +struct occlum_stdio_fds { + int stdin_fd; + int stdout_fd; + int stderr_fd; +}; + #endif /* __OCCLUM_EDL_TYPES_H__ */ diff --git a/src/libos/src/entry.rs b/src/libos/src/entry.rs index 074d38f2..eb3c674a 100644 --- a/src/libos/src/entry.rs +++ b/src/libos/src/entry.rs @@ -1,5 +1,6 @@ use super::*; use exception::*; +use fs::HostStdioFds; use process::pid_t; use std::ffi::{CStr, CString, OsString}; use std::path::{Path, PathBuf}; @@ -56,13 +57,14 @@ pub extern "C" fn occlum_ecall_init(log_level: *const c_char) -> i32 { pub extern "C" fn occlum_ecall_new_process( path_buf: *const c_char, argv: *const *const c_char, + host_stdio_fds: *const HostStdioFds, ) -> i32 { if HAS_INIT.load(Ordering::SeqCst) == false { return EXIT_STATUS_INTERNAL_ERROR; } - let (path, args) = match parse_arguments(path_buf, argv) { - Ok(path_and_args) => path_and_args, + let (path, args, host_stdio_fds) = match parse_arguments(path_buf, argv, host_stdio_fds) { + Ok(path_and_args_and_host_stdio_fds) => path_and_args_and_host_stdio_fds, Err(e) => { eprintln!("invalid arguments for LibOS: {}", e.backtrace()); return EXIT_STATUS_INTERNAL_ERROR; @@ -70,11 +72,13 @@ pub extern "C" fn occlum_ecall_new_process( }; let _ = backtrace::enable_backtrace(ENCLAVE_PATH, PrintFormat::Short); panic::catch_unwind(|| { - backtrace::__rust_begin_short_backtrace(|| match do_new_process(&path, &args) { - Ok(pid_t) => pid_t as i32, - Err(e) => { - eprintln!("failed to boot up LibOS: {}", e.backtrace()); - EXIT_STATUS_INTERNAL_ERROR + backtrace::__rust_begin_short_backtrace(|| { + match do_new_process(&path, &args, &host_stdio_fds) { + Ok(pid_t) => pid_t as i32, + Err(e) => { + eprintln!("failed to boot up LibOS: {}", e.backtrace()); + EXIT_STATUS_INTERNAL_ERROR + } } }) }) @@ -135,7 +139,8 @@ fn parse_log_level(level_chars: *const c_char) -> Result { fn parse_arguments( path_ptr: *const c_char, argv: *const *const c_char, -) -> Result<(PathBuf, Vec)> { + host_stdio_fds: *const HostStdioFds, +) -> Result<(PathBuf, Vec, HostStdioFds)> { let path_buf = { let path_cstring = clone_cstring_safely(path_ptr)?; let path_string = path_cstring @@ -155,18 +160,31 @@ fn parse_arguments( let mut args = clone_cstrings_safely(argv)?; args.insert(0, program_cstring); - Ok((path_buf, args)) + + let host_stdio_fds = HostStdioFds::from_user(host_stdio_fds)?; + + Ok((path_buf, args, host_stdio_fds)) } -fn do_new_process(program_path: &PathBuf, argv: &Vec) -> Result { +fn do_new_process( + program_path: &PathBuf, + argv: &Vec, + host_stdio_fds: &HostStdioFds, +) -> Result { validate_program_path(program_path)?; let envp = &config::LIBOS_CONFIG.env; let file_actions = Vec::new(); let parent = &process::IDLE_PROCESS; let program_path_str = program_path.to_str().unwrap(); - let new_tid = - process::do_spawn_without_exec(&program_path_str, argv, envp, &file_actions, parent)?; + let new_tid = process::do_spawn_without_exec( + &program_path_str, + argv, + envp, + &file_actions, + host_stdio_fds, + parent, + )?; Ok(new_tid) } diff --git a/src/libos/src/fs/mod.rs b/src/libos/src/fs/mod.rs index 7cdadd1e..85f5798e 100644 --- a/src/libos/src/fs/mod.rs +++ b/src/libos/src/fs/mod.rs @@ -19,7 +19,7 @@ pub use self::file_table::{FileDesc, FileTable}; pub use self::inode_file::{AsINodeFile, INodeExt, INodeFile}; pub use self::pipe::Pipe; pub use self::rootfs::ROOT_INODE; -pub use self::stdio::{StdinFile, StdoutFile}; +pub use self::stdio::{HostStdioFds, StdinFile, StdoutFile}; pub use self::syscalls::*; mod dev_fs; diff --git a/src/libos/src/fs/stdio.rs b/src/libos/src/fs/stdio.rs index f76086e3..8836e485 100644 --- a/src/libos/src/fs/stdio.rs +++ b/src/libos/src/fs/stdio.rs @@ -1,20 +1,114 @@ use super::*; +use core::cell::RefCell; +use core::cmp; +use std::io::{BufReader, LineWriter}; +use std::sync::SgxMutex; + +macro_rules! try_libc_stdio { + ($ret: expr) => {{ + let ret = unsafe { $ret }; + if ret < 0 { + let errno_c = unsafe { libc::errno() }; + Err(errno!(Errno::from(errno_c as u32))) + } else { + Ok(ret) + } + }}; +} + +// Struct for the occlum_stdio_fds +#[repr(C)] +pub struct HostStdioFds { + pub stdin_fd: i32, + pub stdout_fd: i32, + pub stderr_fd: i32, +} + +impl HostStdioFds { + pub fn from_user(ptr: *const HostStdioFds) -> Result { + if ptr.is_null() { + return Ok(Self { + stdin_fd: libc::STDIN_FILENO, + stdout_fd: libc::STDOUT_FILENO, + stderr_fd: libc::STDERR_FILENO, + }); + } + let host_stdio_fds_c = unsafe { &*ptr }; + if host_stdio_fds_c.stdin_fd < 0 + || host_stdio_fds_c.stdout_fd < 0 + || host_stdio_fds_c.stderr_fd < 0 + { + return_errno!(EBADF, "invalid file descriptor"); + } + Ok(Self { + stdin_fd: host_stdio_fds_c.stdin_fd, + stdout_fd: host_stdio_fds_c.stdout_fd, + stderr_fd: host_stdio_fds_c.stderr_fd, + }) + } +} + +struct StdoutRaw { + host_fd: i32, +} + +impl StdoutRaw { + pub fn new(host_fd: FileDesc) -> Self { + Self { + host_fd: host_fd as i32, + } + } +} + +impl std::io::Write for StdoutRaw { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let writting_len = cmp::min(buf.len(), size_t::max_value() as usize); + let ret = try_libc_stdio!(libc::ocall::write( + self.host_fd, + buf.as_ptr() as *const c_void, + writting_len, + )) + .unwrap_or_else(|err| { + warn!("tolerate the write error: {:?}", err.errno()); + writting_len as isize + }); + // sanity check + assert!(ret <= writting_len as isize); + Ok(ret as usize) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} pub struct StdoutFile { - inner: std::io::Stdout, + inner: SgxMutex>, + host_fd: FileDesc, } impl StdoutFile { - pub fn new() -> StdoutFile { + pub fn new(host_fd: FileDesc) -> Self { StdoutFile { - inner: std::io::stdout(), + inner: SgxMutex::new(LineWriter::new(StdoutRaw::new(host_fd))), + host_fd, } } + + fn get_host_fd(&self) -> FileDesc { + self.host_fd + } } impl File for StdoutFile { fn write(&self, buf: &[u8]) -> Result { - let write_len = { self.inner.lock().write(buf).map_err(|e| errno!(e))? }; + let write_len = { + self.inner + .lock() + .unwrap() + .write(buf) + .map_err(|e| errno!(e))? + }; Ok(write_len) } @@ -23,7 +117,7 @@ impl File for StdoutFile { } fn writev(&self, bufs: &[&[u8]]) -> Result { - let mut guard = self.inner.lock(); + let mut guard = self.inner.lock().unwrap(); let mut total_bytes = 0; for buf in bufs { match guard.write(buf) { @@ -70,7 +164,7 @@ impl File for StdoutFile { } fn sync_data(&self) -> Result<()> { - self.inner.lock().flush()?; + self.inner.lock().unwrap().flush()?; Ok(()) } @@ -86,10 +180,7 @@ impl File for StdoutFile { let cmd_bits = cmd.cmd_num() as c_int; let cmd_arg_ptr = cmd.arg_ptr() as *const c_int; - let host_stdout_fd = { - use std::os::unix::io::AsRawFd; - self.inner.as_raw_fd() as i32 - }; + let host_stdout_fd = self.get_host_fd() as i32; try_libc!(libc::ocall::ioctl_arg1( host_stdout_fd, cmd_bits, @@ -107,33 +198,75 @@ impl File for StdoutFile { impl Debug for StdoutFile { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "StdoutFile") + write!(f, "StdoutFile with host_fd: {}", self.host_fd) } } unsafe impl Send for StdoutFile {} unsafe impl Sync for StdoutFile {} +struct StdinRaw { + host_fd: i32, +} + +impl StdinRaw { + pub fn new(host_fd: FileDesc) -> Self { + Self { + host_fd: host_fd as i32, + } + } +} + +impl std::io::Read for StdinRaw { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let reading_len = cmp::min(buf.len(), size_t::max_value() as usize); + let ret = try_libc_stdio!(libc::ocall::read( + self.host_fd, + buf.as_mut_ptr() as *mut c_void, + reading_len, + )) + .unwrap_or_else(|err| { + warn!("tolerate the read error: {:?}", err.errno()); + 0 + }); + // sanity check + assert!(ret <= reading_len as isize); + Ok(ret as usize) + } +} + pub struct StdinFile { - inner: std::io::Stdin, + inner: SgxMutex>, + host_fd: FileDesc, } impl StdinFile { - pub fn new() -> StdinFile { + pub fn new(host_fd: FileDesc) -> Self { StdinFile { - inner: std::io::stdin(), + inner: SgxMutex::new(BufReader::new(StdinRaw::new(host_fd))), + host_fd, } } + + fn get_host_fd(&self) -> FileDesc { + self.host_fd + } } impl File for StdinFile { fn read(&self, buf: &mut [u8]) -> Result { - let read_len = { self.inner.lock().read(buf).map_err(|e| errno!(e))? }; + let read_len = { + self.inner + .lock() + .unwrap() + .read(buf) + .map_err(|e| errno!(e))? + }; Ok(read_len) } fn readv(&self, bufs: &mut [&mut [u8]]) -> Result { - let mut guard = self.inner.lock(); + let mut guard = self.inner.lock().unwrap(); let mut total_bytes = 0; for buf in bufs { match guard.read(buf) { @@ -175,6 +308,29 @@ impl File for StdinFile { }) } + fn ioctl(&self, cmd: &mut IoctlCmd) -> Result<()> { + let can_delegate_to_host = match cmd { + IoctlCmd::TIOCGWINSZ(_) => true, + IoctlCmd::TIOCSWINSZ(_) => true, + _ => false, + }; + if !can_delegate_to_host { + return_errno!(EINVAL, "unknown ioctl cmd for stdin"); + } + + let cmd_bits = cmd.cmd_num() as c_int; + let cmd_arg_ptr = cmd.arg_ptr() as *const c_int; + let host_stdin_fd = self.get_host_fd() as i32; + try_libc!(libc::ocall::ioctl_arg1( + host_stdin_fd, + cmd_bits, + cmd_arg_ptr + )); + cmd.validate_arg_val()?; + + Ok(()) + } + fn as_any(&self) -> &dyn Any { self } @@ -182,7 +338,7 @@ impl File for StdinFile { impl Debug for StdinFile { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "StdinFile") + write!(f, "StdinFile with host_fd: {}", self.host_fd) } } diff --git a/src/libos/src/process/spawn/mod.rs b/src/libos/src/process/spawn/mod.rs index a24309c4..d22f2f71 100644 --- a/src/libos/src/process/spawn/mod.rs +++ b/src/libos/src/process/spawn/mod.rs @@ -5,7 +5,8 @@ use std::path::Path; use std::sgxfs::SgxFile; use super::fs::{ - CreationFlags, File, FileDesc, FileTable, INodeExt, StdinFile, StdoutFile, ROOT_INODE, + CreationFlags, File, FileDesc, FileTable, HostStdioFds, INodeExt, StdinFile, StdoutFile, + ROOT_INODE, }; use super::misc::ResourceLimitsRef; use super::vm::{ProcessVM, ProcessVMBuilder}; @@ -24,7 +25,8 @@ pub fn do_spawn( file_actions: &[FileAction], parent_ref: &ProcessRef, ) -> Result { - let (new_tid, new_process_ref) = new_process(elf_path, argv, envp, file_actions, parent_ref)?; + let (new_tid, new_process_ref) = + new_process(elf_path, argv, envp, file_actions, None, parent_ref)?; task::enqueue_and_exec_task(new_tid, new_process_ref); Ok(new_tid) } @@ -34,9 +36,17 @@ pub fn do_spawn_without_exec( argv: &[CString], envp: &[CString], file_actions: &[FileAction], + host_stdio_fds: &HostStdioFds, parent_ref: &ProcessRef, ) -> Result { - let (new_tid, new_process_ref) = new_process(elf_path, argv, envp, file_actions, parent_ref)?; + let (new_tid, new_process_ref) = new_process( + elf_path, + argv, + envp, + file_actions, + Some(host_stdio_fds), + parent_ref, + )?; task::enqueue_task(new_tid, new_process_ref); Ok(new_tid) } @@ -46,6 +56,7 @@ fn new_process( argv: &[CString], envp: &[CString], file_actions: &[FileAction], + host_stdio_fds: Option<&HostStdioFds>, parent_ref: &ProcessRef, ) -> Result<(pid_t, ProcessRef)> { let elf_buf = load_elf_to_vec(elf_path, parent_ref) @@ -107,7 +118,7 @@ fn new_process( }; let vm_ref = Arc::new(SgxMutex::new(vm)); let files_ref = { - let files = init_files(parent_ref, file_actions)?; + let files = init_files(parent_ref, file_actions, host_stdio_fds)?; Arc::new(SgxMutex::new(files)) }; let rlimits_ref = Default::default(); @@ -145,7 +156,11 @@ fn load_elf_to_vec(elf_path: &str, parent_ref: &ProcessRef) -> Result> { .map_err(|e| errno!(e.errno(), "failed to read the executable ELF")) } -fn init_files(parent_ref: &ProcessRef, file_actions: &[FileAction]) -> Result { +fn init_files( + parent_ref: &ProcessRef, + file_actions: &[FileAction], + host_stdio_fds: Option<&HostStdioFds>, +) -> Result { // Usually, we just inherit the file table from the parent let parent = parent_ref.lock().unwrap(); let should_inherit_file_table = parent.get_pid() > 0; @@ -186,10 +201,16 @@ fn init_files(parent_ref: &ProcessRef, file_actions: &[FileAction]) -> Result> = Arc::new(Box::new(StdinFile::new())); - let stdout: Arc> = Arc::new(Box::new(StdoutFile::new())); - // TODO: implement and use a real stderr - let stderr = stdout.clone(); + let stdin: Arc> = Arc::new(Box::new(StdinFile::new( + host_stdio_fds.unwrap().stdin_fd as FileDesc, + ))); + let stdout: Arc> = Arc::new(Box::new(StdoutFile::new( + host_stdio_fds.unwrap().stdout_fd as FileDesc, + ))); + let stderr: Arc> = Arc::new(Box::new(StdoutFile::new( + host_stdio_fds.unwrap().stderr_fd as FileDesc, + ))); + file_table.put(stdin, false); file_table.put(stdout, false); file_table.put(stderr, false); diff --git a/src/libos/src/syscall/mod.rs b/src/libos/src/syscall/mod.rs index 780da236..55306eab 100644 --- a/src/libos/src/syscall/mod.rs +++ b/src/libos/src/syscall/mod.rs @@ -12,7 +12,8 @@ use fs::{ do_fcntl, do_fdatasync, do_fstat, do_fstatat, do_fsync, do_ftruncate, do_getdents64, do_ioctl, do_link, do_lseek, do_lstat, do_mkdir, do_open, do_openat, do_pipe, do_pipe2, do_pread, do_pwrite, do_read, do_readlink, do_readv, do_rename, do_rmdir, do_sendfile, do_stat, do_sync, - do_truncate, do_unlink, do_write, do_writev, iovec_t, File, FileDesc, FileRef, Stat, + do_truncate, do_unlink, do_write, do_writev, iovec_t, File, FileDesc, FileRef, HostStdioFds, + Stat, }; use misc::{resource_t, rlimit_t, utsname_t}; use net::{ diff --git a/src/pal/include/edl/occlum_edl_types.h b/src/pal/include/edl/occlum_edl_types.h index c1d04be3..06c794a8 100644 --- a/src/pal/include/edl/occlum_edl_types.h +++ b/src/pal/include/edl/occlum_edl_types.h @@ -4,5 +4,6 @@ #include // import struct timespec #include // import struct timeval #include // import struct iovec +#include // import occlum_stdio_fds #endif /* __OCCLUM_EDL_TYPES__ */ diff --git a/src/pal/include/occlum_pal_api.h b/src/pal/include/occlum_pal_api.h index a52bc2b3..b9a3dcf1 100644 --- a/src/pal/include/occlum_pal_api.h +++ b/src/pal/include/occlum_pal_api.h @@ -33,6 +33,15 @@ typedef struct { .log_level = NULL \ } +/* + * The struct which consists of file descriptors of standard I/O + */ +struct occlum_stdio_fds { + int stdin_fd; + int stdout_fd; + int stderr_fd; +}; + /* * @brief Initialize an Occlum enclave * @@ -48,13 +57,19 @@ int occlum_pal_init(occlum_pal_attr_t* attr); * @param cmd_path The path of the command to be executed * @param cmd_args The arguments to the command. The array must be NULL * terminated. + * @param io_fds The file descriptors of the redirected standard I/O + * (i.e., stdin, stdout, stderr), If set to NULL, will + * use the original standard I/O file descriptors. * @param exit_status Output. The exit status of the command. Note that the * exit status is returned if and only if the function * succeeds. * * @retval If 0, then success; otherwise, check errno for the exact error type. */ -int occlum_pal_exec(const char* cmd_path, const char** cmd_args, int* exit_status); +int occlum_pal_exec(const char* cmd_path, + const char** cmd_args, + const struct occlum_stdio_fds* io_fds, + int* exit_status); /* * @brief Destroy teh Occlum enclave diff --git a/src/pal/src/pal_api.c b/src/pal/src/pal_api.c index 337254f3..26bffb11 100644 --- a/src/pal/src/pal_api.c +++ b/src/pal/src/pal_api.c @@ -47,7 +47,10 @@ int occlum_pal_init(occlum_pal_attr_t* attr) { return 0; } -int occlum_pal_exec(const char* cmd_path, const char** cmd_args, int* exit_status) { +int occlum_pal_exec(const char* cmd_path, + const char** cmd_args, + const struct occlum_stdio_fds* io_fds, + int* exit_status) { errno = 0; if (cmd_path == NULL || cmd_args == NULL || exit_status == NULL) { @@ -63,7 +66,7 @@ int occlum_pal_exec(const char* cmd_path, const char** cmd_args, int* exit_statu } int libos_tid = -1; - sgx_status_t ecall_status = occlum_ecall_new_process(eid, &libos_tid, cmd_path, cmd_args); + sgx_status_t ecall_status = occlum_ecall_new_process(eid, &libos_tid, cmd_path, cmd_args, io_fds); if (ecall_status != SGX_SUCCESS) { const char* sgx_err = pal_get_sgx_error_msg(ecall_status); PAL_ERROR("Failed to do ECall: %s", sgx_err); diff --git a/src/run/main.c b/src/run/main.c index d5d3eecd..e6add33a 100644 --- a/src/run/main.c +++ b/src/run/main.c @@ -1,6 +1,7 @@ #include #include #include +#include #include static const char* get_instance_dir(void) { @@ -32,8 +33,13 @@ int main(int argc, char* argv[]) { } // Use Occlum PAL to execute the cmd + struct occlum_stdio_fds io_fds = { + .stdin_fd = STDIN_FILENO, + .stdout_fd = STDOUT_FILENO, + .stderr_fd = STDERR_FILENO, + }; int exit_status = 0; - if (occlum_pal_exec(cmd_path, cmd_args, &exit_status) < 0) { + if (occlum_pal_exec(cmd_path, cmd_args, &io_fds, &exit_status) < 0) { return EXIT_FAILURE; }