Fix the error code and return logic for mmap failure

src/libos/src/vm/chunk.rs

@@ -92,7 +92,7 @@ impl Chunk {
         })
     }
 
-    pub fn new_single_vma_chunk(vm_range: VMRange, options: &VMMapOptions) -> Self {
+    pub fn new_single_vma_chunk(vm_range: &VMRange, options: &VMMapOptions) -> Result<Self> {
         let writeback_file = options.writeback_file().clone();
         let vm_area = VMArea::new(
             vm_range.clone(),
@@ -103,13 +103,13 @@ impl Chunk {
         // Initialize the memory of the new range
         unsafe {
             let buf = vm_range.as_slice_mut();
-            options.initializer().init_slice(buf);
+            options.initializer().init_slice(buf)?;
         }
         // Set memory permissions
         if !options.perms().is_default() {
             VMPerms::apply_perms(&vm_area, vm_area.perms());
         }
-        Self::new_chunk_with_vma(vm_area)
+        Ok(Self::new_chunk_with_vma(vm_area))
     }
 
     pub fn new_chunk_with_vma(vma: VMArea) -> Self {

src/libos/src/vm/vm_chunk_manager.rs

@@ -121,10 +121,15 @@ impl ChunkManager {
         let new_vma = VMArea::new(new_range, *options.perms(), writeback_file, current_pid);
 
         // Initialize the memory of the new range
-        unsafe {
+        let buf = unsafe { new_vma.as_slice_mut() };
-            let buf = new_vma.as_slice_mut();
+        let ret = options.initializer().init_slice(buf);
-            options.initializer().init_slice(buf)?;
+        if let Err(e) = ret {
+            // Return the free range before return with error
+            self.free_manager
+                .add_range_back_to_free_manager(new_vma.range());
+            return_errno!(e.errno(), "failed to mmap");
         }
+
         // Set memory permissions
         if !options.perms().is_default() {
             VMPerms::apply_perms(&new_vma, new_vma.perms());

src/libos/src/vm/vm_manager.rs

@@ -117,13 +117,13 @@ impl VMManager {
         // Allocate a new chunk with chunk default size.
         // Lock on ChunkManager.
         if let Ok(new_chunk) = self.internal().mmap_chunk_default(addr) {
-            // Allocate in the new chunk
+            // Add this new chunk to process' chunk list
-            let start = new_chunk.mmap(options);
-            debug_assert!(start.is_ok()); // We just allocate a chunk for you. You must succeed.
-                                          // Add this new chunk to process' chunk list
             new_chunk.add_process(&current);
-            current.vm().add_mem_chunk(new_chunk);
+            current.vm().add_mem_chunk(new_chunk.clone());
-            return start;
+
+            // Allocate in the new chunk
+            // This mmap could still fail due to invalid options
+            return new_chunk.mmap(options);
         }
 
         // Slow path: Sadly, there is no free chunk, iterate every chunk to find a range
@@ -548,7 +548,14 @@ impl InternalVMManager {
         let size = *options.size();
         let align = *options.align();
         let free_range = self.find_free_gaps(size, align, addr)?;
-        let chunk = Arc::new(Chunk::new_single_vma_chunk(free_range, options));
+        let free_chunk = Chunk::new_single_vma_chunk(&free_range, options);
+        if let Err(e) = free_chunk {
+            // Error when creating chunks. Must return the free space before returning error
+            self.free_manager
+                .add_range_back_to_free_manager(&free_range);
+            return_errno!(e.errno(), "mmap_chunk failure");
+        }
+        let chunk = Arc::new(free_chunk.unwrap());
         trace!("allocate a new single vma chunk: {:?}", chunk);
         self.chunks.insert(chunk.clone());
         Ok(chunk)

src/libos/src/vm/vm_util.rs

@@ -59,7 +59,7 @@ impl VMInitializer {
                 // TODO: make sure that read_at does not move file cursor
                 let len = file
                     .read_at(*offset, buf)
-                    .cause_err(|_| errno!(EIO, "failed to init memory from file"))?;
+                    .cause_err(|_| errno!(EACCES, "failed to init memory from file"))?;
                 for b in &mut buf[len..] {
                     *b = 0;
                 }
@@ -77,7 +77,7 @@ impl VMInitializer {
                 buf[..copy_len].copy_from_slice(&src_slice[..copy_len]);
                 let len = file
                     .read_at(*offset, &mut buf[copy_len..])
-                    .cause_err(|_| errno!(EIO, "failed to init memory from file"))?;
+                    .cause_err(|_| errno!(EACCES, "failed to init memory from file"))?;
                 for b in &mut buf[(copy_len + len)..] {
                     *b = 0;
                 }