Add mremap system call

src/libos/src/syscall/mod.rs

@@ -41,7 +41,7 @@ use crate::signal::{
     do_kill, do_rt_sigaction, do_rt_sigpending, do_rt_sigprocmask, do_rt_sigreturn, do_sigaltstack,
     do_tgkill, do_tkill, sigaction_t, sigset_t, stack_t,
 };
-use crate::vm::{MMapFlags, VMPerms};
+use crate::vm::{MMapFlags, MRemapFlags, VMPerms};
 use crate::{fs, process, std, vm};
 
 use super::*;
@@ -738,8 +738,9 @@ fn do_mremap(
     flags: i32,
     new_addr: usize,
 ) -> Result<isize> {
-    warn!("mremap: not implemented!");
+    let flags = MRemapFlags::from_u32(flags as u32)?;
-    return_errno!(ENOSYS, "not supported yet")
+    let addr = vm::do_mremap(old_addr, old_size, new_size, flags, new_addr)?;
+    Ok(addr as isize)
 }
 
 fn do_mprotect(addr: usize, len: usize, prot: u32) -> Result<isize> {

src/libos/src/vm/mod.rs

@@ -12,7 +12,7 @@ mod vm_range;
 use self::vm_layout::VMLayout;
 use self::vm_manager::{VMManager, VMMapOptionsBuilder};
 
-pub use self::process_vm::{MMapFlags, ProcessVM, ProcessVMBuilder, VMPerms};
+pub use self::process_vm::{MMapFlags, MRemapFlags, ProcessVM, ProcessVMBuilder, VMPerms};
 pub use self::vm_range::VMRange;
 
 pub fn do_mmap(
@@ -47,6 +47,22 @@ pub fn do_munmap(addr: usize, size: usize) -> Result<()> {
     current_vm.munmap(addr, size)
 }
 
+pub fn do_mremap(
+    old_addr: usize,
+    old_size: usize,
+    new_size: usize,
+    flags: MRemapFlags,
+    new_addr: usize,
+) -> Result<usize> {
+    debug!(
+        "mremap: old_addr: {:#x}, old_size: {:#x}, new_size: {:#x}, flags: {:?}, new_addr: {:#x}",
+        old_addr, old_size, new_size, flags, new_addr
+    );
+    let current = current!();
+    let mut current_vm = current.vm().lock().unwrap();
+    current_vm.mremap(old_addr, old_size, new_size, flags, new_addr)
+}
+
 pub fn do_brk(addr: usize) -> Result<usize> {
     debug!("brk: addr: {:#x}", addr);
     let current = current!();

src/libos/src/vm/process_vm.rs

@@ -3,7 +3,9 @@ use super::*;
 use super::config;
 use super::process::elf_file::{ElfFile, ProgramHeaderExt};
 use super::user_space_vm::{UserSpaceVMManager, UserSpaceVMRange, USER_SPACE_VM_MANAGER};
-use super::vm_manager::{VMInitializer, VMManager, VMMapAddr, VMMapOptions, VMMapOptionsBuilder};
+use super::vm_manager::{
+    VMInitializer, VMManager, VMMapAddr, VMMapOptions, VMMapOptionsBuilder, VMRemapOptions,
+};
 
 #[derive(Debug)]
 pub struct ProcessVMBuilder<'a, 'b> {
@@ -317,6 +319,27 @@ impl ProcessVM {
         Ok(mmap_addr)
     }
 
+    pub fn mremap(
+        &mut self,
+        old_addr: usize,
+        old_size: usize,
+        new_size: usize,
+        flags: MRemapFlags,
+        new_addr: usize,
+    ) -> Result<usize> {
+        let new_addr_option = if flags.contains(MRemapFlags::MREMAP_FIXED) {
+            if !self.process_range.range().contains(new_addr) {
+                return_errno!(EINVAL, "new_addr is beyond valid memory range");
+            }
+            Some(new_addr)
+        } else {
+            None
+        };
+        let mremap_option =
+            VMRemapOptions::new(old_addr, old_size, new_addr_option, new_size, flags)?;
+        self.mmap_manager.mremap(&mremap_option)
+    }
+
     pub fn munmap(&mut self, addr: usize, size: usize) -> Result<()> {
         self.mmap_manager.munmap(addr, size)
     }
@@ -356,6 +379,25 @@ impl MMapFlags {
     }
 }
 
+bitflags! {
+    pub struct MRemapFlags : u32 {
+        const MREMAP_MAYMOVE      = 1;
+        const MREMAP_FIXED        = 2;
+    }
+}
+
+impl MRemapFlags {
+    pub fn from_u32(bits: u32) -> Result<MRemapFlags> {
+        let flags =
+            MRemapFlags::from_bits(bits).ok_or_else(|| errno!(EINVAL, "unknown mremap flags"))?;
+        if flags.contains(MRemapFlags::MREMAP_FIXED) && !flags.contains(MRemapFlags::MREMAP_MAYMOVE)
+        {
+            return_errno!(EINVAL, "MREMAP_FIXED was specified without MREMAP_MAYMOVE");
+        }
+        Ok(flags)
+    }
+}
+
 bitflags! {
     pub struct VMPerms : u32 {
         const READ        = 0x1;

src/libos/src/vm/vm_manager.rs

@@ -123,6 +123,78 @@ impl VMMapOptions {
     }
 }
 
+#[derive(Debug)]
+pub struct VMRemapOptions {
+    old_addr: usize,
+    old_size: usize,
+    new_addr: Option<usize>,
+    new_size: usize,
+    flags: MRemapFlags,
+}
+
+impl VMRemapOptions {
+    pub fn new(
+        old_addr: usize,
+        old_size: usize,
+        new_addr: Option<usize>,
+        new_size: usize,
+        flags: MRemapFlags,
+    ) -> Result<Self> {
+        let old_addr = if old_addr % PAGE_SIZE != 0 {
+            return_errno!(EINVAL, "unaligned old address");
+        } else {
+            old_addr
+        };
+        let old_size = if old_size == 0 {
+            // TODO: support old_size is zero for shareable mapping
+            warn!("do not support old_size is zero");
+            return_errno!(EINVAL, "invalid old size");
+        } else {
+            align_up(old_size, PAGE_SIZE)
+        };
+        let new_addr = {
+            if let Some(addr) = new_addr {
+                if addr % PAGE_SIZE != 0 {
+                    return_errno!(EINVAL, "unaligned new address");
+                }
+            }
+            new_addr
+        };
+        let new_size = if new_size == 0 {
+            return_errno!(EINVAL, "invalid new size");
+        } else {
+            align_up(new_size, PAGE_SIZE)
+        };
+        Ok(Self {
+            old_addr,
+            old_size,
+            new_addr,
+            new_size,
+            flags,
+        })
+    }
+
+    pub fn old_addr(&self) -> usize {
+        self.old_addr
+    }
+
+    pub fn old_size(&self) -> usize {
+        self.old_size
+    }
+
+    pub fn new_size(&self) -> usize {
+        self.new_size
+    }
+
+    pub fn new_addr(&self) -> Option<usize> {
+        self.new_addr
+    }
+
+    pub fn flags(&self) -> MRemapFlags {
+        self.flags
+    }
+}
+
 #[derive(Debug, Default)]
 pub struct VMManager {
     range: VMRange,
@@ -172,6 +244,76 @@ impl VMManager {
         Ok(new_subrange_addr)
     }
 
+    pub fn mremap(&mut self, options: &VMRemapOptions) -> Result<usize> {
+        let old_addr = options.old_addr();
+        let old_size = options.old_size();
+        let new_size = options.new_size();
+        let (vm_subrange, idx) = {
+            let idx = self.find_mmap_region_idx(old_addr)?;
+            let vm_subrange = self.sub_ranges[idx];
+            if (vm_subrange.end() - old_addr < old_size) {
+                // Across the vm range
+                return_errno!(EFAULT, "can not remap across vm range");
+            } else if (vm_subrange.end() - old_addr == old_size) {
+                // Exactly the vm range
+                (vm_subrange, idx)
+            } else {
+                // Part of the vm range
+                let old_subrange = VMRange::new(old_addr, old_addr + old_size)?;
+                let (subranges, offset) = {
+                    let mut subranges = vm_subrange.subtract(&old_subrange);
+                    let idx = subranges
+                        .iter()
+                        .position(|subrange| old_subrange.start() < subrange.start())
+                        .unwrap_or_else(|| subranges.len());
+                    subranges.insert(idx, old_subrange);
+                    (subranges, idx)
+                };
+                self.sub_ranges.splice(idx..=idx, subranges.iter().cloned());
+                (old_subrange, idx + offset)
+            }
+        };
+        // Remap with a fixed new_addr, move it to new_addr
+        if let Some(new_addr) = options.new_addr() {
+            let new_subrange = VMRange::new(new_addr, new_addr + new_size)?;
+            if vm_subrange.overlap_with(&new_subrange) {
+                return_errno!(EINVAL, "old/new vm range overlap");
+            }
+            let new_addr = VMMapAddr::Fixed(new_addr);
+            let (insert_idx, free_subrange) = self.find_free_subrange(new_size, new_addr)?;
+            let new_subrange = self.alloc_subrange_from(new_size, new_addr, &free_subrange);
+            return self.move_mmap_region(&vm_subrange, (insert_idx, &new_subrange));
+        }
+        // Remap without a fixed new_addr
+        if old_size > new_size {
+            // Shrink the mmap range, just unmap the useless range
+            self.munmap(old_addr + new_size, old_size - new_size)?;
+            Ok(old_addr)
+        } else if old_size == new_size {
+            // Same size, do nothing
+            Ok(old_addr)
+        } else {
+            // Need to expand the mmap range, check if we can expand it
+            if let Some(next_subrange) = self.sub_ranges.get(idx + 1) {
+                let expand_size = new_size - old_size;
+                if next_subrange.start() - vm_subrange.end() >= expand_size {
+                    // Memory between subranges is enough, resize it
+                    let vm_subrange = self.sub_ranges.get_mut(idx).unwrap();
+                    vm_subrange.resize(new_size);
+                    return Ok(vm_subrange.start());
+                }
+            }
+            // Not enough memory to expand, must move it to a new place
+            if !options.flags().contains(MRemapFlags::MREMAP_MAYMOVE) {
+                return_errno!(ENOMEM, "not enough memory to expand");
+            }
+            let new_addr = VMMapAddr::Any;
+            let (insert_idx, free_subrange) = self.find_free_subrange(new_size, new_addr)?;
+            let new_subrange = self.alloc_subrange_from(new_size, new_addr, &free_subrange);
+            self.move_mmap_region(&vm_subrange, (insert_idx, &new_subrange))
+        }
+    }
+
     pub fn munmap(&mut self, addr: usize, size: usize) -> Result<()> {
         let size = {
             if size == 0 {
@@ -217,6 +359,32 @@ impl VMManager {
             .ok_or_else(|| errno!(ESRCH, "no mmap regions that contains the address"))
     }
 
+    fn find_mmap_region_idx(&self, addr: usize) -> Result<usize> {
+        self.sub_ranges
+            .iter()
+            .position(|subrange| subrange.contains(addr))
+            .ok_or_else(|| errno!(ESRCH, "no mmap regions that contains the address"))
+    }
+
+    fn move_mmap_region(
+        &mut self,
+        src_subrange: &VMRange,
+        dst_idx_and_subrange: (usize, &VMRange),
+    ) -> Result<usize> {
+        let dst_idx = dst_idx_and_subrange.0;
+        let dst_subrange = dst_idx_and_subrange.1;
+        unsafe {
+            let src_buf = src_subrange.as_slice_mut();
+            let dst_buf = dst_subrange.as_slice_mut();
+            for (d, s) in dst_buf.iter_mut().zip(src_buf.iter()) {
+                *d = *s;
+            }
+        }
+        self.sub_ranges.insert(dst_idx, *dst_subrange);
+        self.munmap(src_subrange.start(), src_subrange.size())?;
+        Ok(dst_subrange.start())
+    }
+
     // Find the free subrange that satisfies the constraints of size and address
     fn find_free_subrange(&mut self, size: usize, addr: VMMapAddr) -> Result<(usize, VMRange)> {
         // TODO: reduce the complexity from O(N) to O(log(N)), where N is

src/libos/src/vm/vm_range.rs

@@ -71,6 +71,10 @@ impl VMRange {
         self.start() <= addr && addr < self.end()
     }
 
+    pub fn overlap_with(&self, other: &VMRange) -> bool {
+        self.start() < other.end() && other.start() < self.end()
+    }
+
     pub fn subtract(&self, other: &VMRange) -> Vec<VMRange> {
         let self_start = self.start();
         let self_end = self.end();

test/mmap/main.c

@@ -1,9 +1,11 @@
+#define _GNU_SOURCE
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <sys/mman.h>
 #include <unistd.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <errno.h>
 #include <assert.h>
 #include <string.h>
 #include <fcntl.h>
@@ -615,6 +617,119 @@ int test_munmap_with_non_page_aligned_len() {
     return 0;
 }
 
+// ============================================================================
+// Test cases for mremap
+// ============================================================================
+
+int test_mremap() {
+    int prot = PROT_READ | PROT_WRITE;
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+
+    for (size_t len = PAGE_SIZE; len < MAX_MMAP_USED_MEMORY; len *= 2) {
+        void *buf = mmap(NULL, len, prot, flags, -1, 0);
+        if (buf == MAP_FAILED) {
+            THROW_ERROR("mmap failed");
+        }
+
+        if (check_bytes_in_buf(buf, len, 0) < 0) {
+            THROW_ERROR("the buffer is not initialized to zeros");
+        }
+        void *expand_buf = mremap(buf, len, 2 * len, MREMAP_MAYMOVE);
+        if (expand_buf == MAP_FAILED) {
+            THROW_ERROR("mremap with big size failed");
+        }
+        if (check_bytes_in_buf(expand_buf, len, 0) < 0) {
+            THROW_ERROR("the old part of expand buffer is not zero");
+        }
+        memset(expand_buf, 'a', len * 2);
+        void *shrink_buf = mremap(expand_buf, 2 * len, len, 0);
+        if (shrink_buf == MAP_FAILED) {
+            THROW_ERROR("mmap with small size failed");
+        }
+        if (check_bytes_in_buf(shrink_buf, len, 'a') < 0) {
+            THROW_ERROR("the shrink buffer is not correct");
+        }
+        int ret = munmap(shrink_buf, len);
+        if (ret < 0) {
+            THROW_ERROR("munmap failed");
+        }
+    }
+    return 0;
+}
+
+int test_mremap_subrange() {
+    int prot = PROT_READ | PROT_WRITE;
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+
+    size_t len = PAGE_SIZE * 4;
+    void *buf = mmap(NULL, len, prot, flags, -1, 0);
+    if (buf == MAP_FAILED) {
+        THROW_ERROR("mmap failed");
+    }
+    if (check_bytes_in_buf(buf, len, 0) < 0) {
+        THROW_ERROR("the buffer is not initialized to zeros");
+    }
+    /* remap a subrange in the buffer */
+    void *new_part_buf = mremap(buf + len / 4, len / 4, len, MREMAP_MAYMOVE);
+    if (new_part_buf == MAP_FAILED) {
+        THROW_ERROR("mremap with subrange failed");
+    }
+    if (check_bytes_in_buf(new_part_buf, len / 4, 0) < 0) {
+        THROW_ERROR("the old part of buffer is not zero");
+    }
+    void *rear_buf = buf + len / 2;
+    /* now the length of rear buffer is (len / 2), remap the second part */
+    void *new_part_rear_buf = mremap(rear_buf + len / 4, len / 4, len, MREMAP_MAYMOVE);
+    if (new_part_rear_buf == MAP_FAILED) {
+        THROW_ERROR("mremap with rear subrange failed");
+    }
+    if (check_bytes_in_buf(new_part_rear_buf, len / 4, 0) < 0) {
+        THROW_ERROR("the old part of rear buffer is not zero");
+    }
+    int ret = munmap(buf, len / 4) || munmap(new_part_buf, len) ||
+              munmap(rear_buf, len / 4) || munmap(new_part_rear_buf, len);
+    if (ret < 0) {
+        THROW_ERROR("munmap failed");
+    }
+    return 0;
+}
+
+int test_mremap_with_fixed_addr() {
+    int prot = PROT_READ | PROT_WRITE;
+    int flags = MAP_PRIVATE | MAP_ANONYMOUS;
+
+    size_t len = PAGE_SIZE * 2;
+    void *buf = mmap(NULL, len, prot, flags, -1, 0);
+    if (buf == MAP_FAILED) {
+        THROW_ERROR("mmap failed");
+    }
+
+    if (check_bytes_in_buf(buf, len, 0) < 0) {
+        THROW_ERROR("the buffer is not initialized to zeros");
+    }
+    void *new_addr = buf + len * 2;
+    void *new_buf = mremap(buf, len, len, MREMAP_FIXED, new_addr);
+    if (new_buf != MAP_FAILED || errno != EINVAL) {
+        THROW_ERROR("check mremap with invalid flags failed");
+    }
+    new_buf = mremap(buf, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, buf);
+    if (new_buf != MAP_FAILED || errno != EINVAL) {
+        THROW_ERROR("check mremap with overlap addr failed");
+    }
+    new_buf = mremap(buf, len, len, MREMAP_FIXED | MREMAP_MAYMOVE, new_addr);
+    if (new_buf == MAP_FAILED) {
+        THROW_ERROR("mmap with a fixed address failed");
+    }
+    if (check_bytes_in_buf(new_buf, len, 0) < 0) {
+        THROW_ERROR("the new buffer is not zero");
+    }
+    int ret = munmap(new_buf, len);
+    if (ret < 0) {
+        THROW_ERROR("munmap failed");
+    }
+    return 0;
+}
+
 // ============================================================================
 // Test suite main
 // ============================================================================
@@ -640,7 +755,10 @@ static test_case_t test_cases[] = {
     TEST_CASE(test_munmap_whose_range_intersects_with_multiple_mmap_regions),
     TEST_CASE(test_munmap_with_null_addr),
     TEST_CASE(test_munmap_with_zero_len),
-    TEST_CASE(test_munmap_with_non_page_aligned_len)
+    TEST_CASE(test_munmap_with_non_page_aligned_len),
+    TEST_CASE(test_mremap),
+    TEST_CASE(test_mremap_subrange),
+    TEST_CASE(test_mremap_with_fixed_addr),
 };
 
 int main() {