From bcf8f5e93c5898f38cd8c8dee2c35eab6abff49b Mon Sep 17 00:00:00 2001
From: Marsman1996 <lqliuyuwei@outlook.com>
Date: Tue, 23 Dec 2025 17:10:26 +0800
Subject: [PATCH] Fix OOM by adding SYSCTL_NR_OPEN limit for setrlimit and
 prlimit64

---
 kernel/src/process/rlimit.rs    |  4 +++-
 kernel/src/syscall/prlimit64.rs | 11 ++++++++++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/kernel/src/process/rlimit.rs b/kernel/src/process/rlimit.rs
index c762a7871..1f9ab7512 100644
--- a/kernel/src/process/rlimit.rs
+++ b/kernel/src/process/rlimit.rs
@@ -25,6 +25,8 @@ const INIT_RLIMIT_NOFILE_MAX: u64 = 4096;
 const INIT_RLIMIT_MEMLOCK: u64 = 8 * 1024 * 1024;
 // https://github.com/torvalds/linux/blob/fac04efc5c793dccbd07e2d59af9f90b7fc0dca4/include/uapi/linux/mqueue.h#L26
 const INIT_RLIMIT_MSGQUEUE: u64 = 819200;
+// https://github.com/torvalds/linux/blob/fac04efc5c793dccbd07e2d59af9f90b7fc0dca4/fs/file.c#L90
+pub const SYSCTL_NR_OPEN: u64 = 1024 * 1024;
 
 #[derive(Clone)]
 pub struct ResourceLimits {
@@ -80,7 +82,7 @@ impl Default for ResourceLimits {
 }
 
 #[repr(u32)]
-#[derive(Debug, Clone, Copy, TryFromInt)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, TryFromInt)]
 pub enum ResourceType {
     RLIMIT_CPU = 0,
     RLIMIT_FSIZE = 1,
diff --git a/kernel/src/syscall/prlimit64.rs b/kernel/src/syscall/prlimit64.rs
index eb21384e5..ddc103e10 100644
--- a/kernel/src/syscall/prlimit64.rs
+++ b/kernel/src/syscall/prlimit64.rs
@@ -10,7 +10,7 @@ use crate::{
         credentials::capabilities::CapSet,
         posix_thread::{AsPosixThread, PosixThread},
         process_table,
-        rlimit::RawRLimit64,
+        rlimit::{RawRLimit64, SYSCTL_NR_OPEN},
     },
 };
 
@@ -33,6 +33,11 @@ pub fn sys_setrlimit(resource: u32, new_rlim_addr: Vaddr, ctx: &Context) -> Resu
     );
     let new_raw: RawRLimit64 = ctx.user_space().read_val(new_rlim_addr)?;
     let resource_limits = ctx.process.resource_limits();
+
+    if resource == ResourceType::RLIMIT_NOFILE && new_raw.max > SYSCTL_NR_OPEN {
+        return_errno_with_message!(Errno::EPERM, "the new limit exceeds the system limit");
+    }
+
     resource_limits
         .get_rlimit(resource)
         .set_cur_and_max(new_raw.cur, new_raw.max)?;
@@ -64,6 +69,10 @@ pub fn sys_prlimit64(
         if new_rlim_addr != 0 {
             let new_raw: RawRLimit64 = ctx.user_space().read_val(new_rlim_addr)?;
             debug!("new_rlimit = {:?}", new_raw);
+            if resource == ResourceType::RLIMIT_NOFILE && new_raw.max > SYSCTL_NR_OPEN {
+                return_errno_with_message!(Errno::EPERM, "the new limit exceeds the system limit");
+            }
+
             resource_limits
                 .get_rlimit(resource)
                 .set_cur_and_max(new_raw.cur, new_raw.max)?;