Fix two bugs related to open directories

1. Support O_DIRECTORY flag for open syscall
2. Disallow to open a directory in write mode

src/libos/src/fs/file_ops/file_flags.rs

@@ -55,6 +55,7 @@ bitflags! {
         /// close on exec
         const O_CLOEXEC = 1 << 19;
         /// create an unnamed temporary regular file
+        /// O_TMPFILE is (_O_TMPFILE | O_DIRECTORY)
         const _O_TMPFILE = 1 << 22;
     }
 }
@@ -75,6 +76,14 @@ impl CreationFlags {
     pub fn no_follow_symlink(&self) -> bool {
         self.contains(CreationFlags::O_NOFOLLOW)
     }
+
+    pub fn must_be_directory(&self) -> bool {
+        if self.contains(CreationFlags::_O_TMPFILE) {
+            warn!("O_TMPFILE is not supported, handle it as O_DIRECTORY");
+            return true;
+        }
+        self.contains(CreationFlags::O_DIRECTORY)
+    }
 }
 
 bitflags! {

src/libos/src/fs/fs_view.rs

@@ -64,6 +64,14 @@ impl FsView {
                     if creation_flags.can_create() && creation_flags.is_exclusive() {
                         return_errno!(EEXIST, "file exists");
                     }
+                    if creation_flags.must_be_directory()
+                        && inode.metadata()?.type_ != FileType::Dir
+                    {
+                        return_errno!(
+                            ENOTDIR,
+                            "O_DIRECTORY is specified but file is not a directory"
+                        );
+                    }
                     inode
                 }
                 Err(e) if e.errno() == ENOENT && creation_flags.can_create() => {
@@ -82,6 +90,14 @@ impl FsView {
                     if creation_flags.can_create() && creation_flags.is_exclusive() {
                         return_errno!(EEXIST, "file exists");
                     }
+                    if creation_flags.must_be_directory()
+                        && inode.metadata()?.type_ != FileType::Dir
+                    {
+                        return_errno!(
+                            ENOTDIR,
+                            "O_DIRECTORY is specified but file is not a directory"
+                        );
+                    }
                     inode
                 }
                 Err(e) if e.errno() == ENOENT && creation_flags.can_create() => {

src/libos/src/fs/inode_file.rs

@@ -210,6 +210,9 @@ impl INodeFile {
         if (access_mode.writable() && !inode.allow_write()?) {
             return_errno!(EACCES, "File not writable");
         }
+        if access_mode.writable() && inode.metadata()?.type_ == FileType::Dir {
+            return_errno!(EISDIR, "Directory cannot be open to write");
+        }
         let status_flags = StatusFlags::from_bits_truncate(flags);
         Ok(INodeFile {
             inode,

test/open/main.c

@@ -1,4 +1,5 @@
 #include <fcntl.h>
+#include <errno.h>
 #include "test_fs.h"
 
 // ============================================================================
@@ -28,6 +29,41 @@ static int __test_open(const char *file_path, int flags, int mode) {
     return 0;
 }
 
+static int __test_open_file_with_dir_flags(const char *file_path, int flags, int mode) {
+    flags = O_DIRECTORY | O_RDWR | O_CREAT;
+    int fd = open(file_path, flags, mode);
+    if (fd < 0) {
+        THROW_ERROR("failed to check creating file with O_DIRECTORY");
+    }
+    close(fd);
+
+    fd = open(file_path, flags, mode);
+    if (!(fd < 0 && errno == ENOTDIR)) {
+        THROW_ERROR("open file with O_DIRECTORY should return ENOTDIR");
+    }
+    return 0;
+}
+
+static int __test_open_dir_with_write_flags(const char *file_path, int flags, int mode) {
+    char dir_buf[PATH_MAX] = { 0 };
+    char *dir_name;
+    int fd;
+
+    if (__test_open(file_path, flags, mode) < 0) {
+        THROW_ERROR("failed to create file");
+    }
+    if (fs_split_path(file_path, dir_buf, &dir_name, NULL, NULL) < 0) {
+        THROW_ERROR("failed to split path");
+    }
+
+    flags = O_WRONLY;
+    fd = open(dir_name, flags, mode);
+    if (!(fd < 0 && errno == EISDIR)) {
+        THROW_ERROR("open dir with write flags should return EISDIR");
+    }
+    return 0;
+}
+
 static int __test_openat_with_abs_path(const char *file_path, int flags, int mode) {
     int fd = openat(AT_FDCWD, file_path, flags, mode);
     if (fd < 0) {
@@ -86,6 +122,14 @@ static int test_open() {
     return test_open_framework(__test_open);
 }
 
+static int test_open_file_with_dir_flags() {
+    return test_open_framework(__test_open_file_with_dir_flags);
+}
+
+static int test_open_dir_with_write_flags() {
+    return test_open_framework(__test_open_dir_with_write_flags);
+}
+
 static int test_openat_with_abs_path() {
     return test_open_framework(__test_openat_with_abs_path);
 }
@@ -100,6 +144,8 @@ static int test_openat_with_dirfd() {
 
 static test_case_t test_cases[] = {
     TEST_CASE(test_open),
+    TEST_CASE(test_open_file_with_dir_flags),
+    TEST_CASE(test_open_dir_with_write_flags),
     TEST_CASE(test_openat_with_abs_path),
     TEST_CASE(test_openat_with_dirfd),
 };