Add KDGETMODE and KDSETMODE ioctl support

Add support for virtual console TTY mode management ioctls required by
Xorg and other display servers. The implementation includes:

- KDGETMODE: Query the current virtual console mode
- KDSETMODE: Switch between text and graphics modes

Virtual consoles utilize the framebuffer with two distinct modes:
- Text mode (default): Kernel renders text characters to framebuffer
- Graphics mode: User space applications gain full framebuffer control

Xorg requires KDSETMODE during initialization to switch to graphics
mode, preventing conflicts where both the kernel and Xorg would
simultaneously attempt to write to the framebuffer. This ensures
proper display handling and eliminates rendering artifacts.

Co-authored-by: Ruihan Li <lrh2000@pku.edu.cn>

Cargo.lock

@@ -110,6 +110,7 @@ version = "0.1.0"
 dependencies = [
  "component",
  "font8x8",
+ "int-to-c-enum",
  "ostd",
  "spin",
 ]

kernel/comps/console/Cargo.toml

@@ -10,6 +10,7 @@ ostd = { path = "../../../ostd" }
 component = { path = "../../libs/comp-sys/component" }
 spin = "0.9.4"
 font8x8 = { version = "0.2.5", default-features = false, features = [ "unicode" ] }
+int-to-c-enum = { path = "../../libs/int-to-c-enum" }
 
 [lints]
 workspace = true

kernel/comps/console/src/lib.rs

@@ -8,6 +8,7 @@
 extern crate alloc;
 
 pub mod font;
+pub mod mode;
 
 use alloc::{collections::BTreeMap, fmt::Debug, string::String, sync::Arc, vec::Vec};
 use core::any::Any;
@@ -22,6 +23,7 @@ use spin::Once;
 pub type ConsoleCallback = dyn Fn(VmReader<Infallible>) + Send + Sync;
 
 /// An error returned by [`AnyConsoleDevice::set_font`].
+#[derive(Debug, Clone, Copy)]
 pub enum ConsoleSetFontError {
     InappropriateDevice,
     InvalidFont,
@@ -40,6 +42,23 @@ pub trait AnyConsoleDevice: Send + Sync + Any + Debug {
     fn set_font(&self, _font: font::BitmapFont) -> Result<(), ConsoleSetFontError> {
         Err(ConsoleSetFontError::InappropriateDevice)
     }
+
+    // TODO: Add support for getting the font of the console device.
+
+    /// Sets the console mode (text or graphics, see [`mode::ConsoleMode`]).
+    ///
+    /// Returns true if the mode was changed, false if the mode is not supported.
+    #[must_use]
+    fn set_mode(&self, _mode: mode::ConsoleMode) -> bool {
+        false
+    }
+
+    /// Gets the current console mode.
+    ///
+    /// Returns the current console mode, or `None` if mode switching is not supported.
+    fn mode(&self) -> Option<mode::ConsoleMode> {
+        None
+    }
 }
 
 pub fn register_device(name: String, device: Arc<dyn AnyConsoleDevice>) {

kernel/comps/console/src/mode.rs

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: MPL-2.0
+
+//! Mode types.
+
+use int_to_c_enum::TryFromInt;
+
+/// The console mode (text or graphics).
+///
+/// Reference: <https://elixir.bootlin.com/linux/v6.17.4/source/include/uapi/linux/kd.h#L45>.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, TryFromInt)]
+#[repr(i32)]
+pub enum ConsoleMode {
+    /// The text mode (`KD_TEXT` in Linux). The console will display text characters.
+    Text = 0,
+    /// The graphics mode (`KD_GRAPHICS` in Linux). The console will not display text characters
+    /// and may be used for graphical output (e.g., by X server).
+    Graphics = 1,
+}

kernel/comps/framebuffer/src/console.rs

@@ -2,7 +2,9 @@
 
 use alloc::{sync::Arc, vec::Vec};
 
-use aster_console::{font::BitmapFont, AnyConsoleDevice, ConsoleCallback, ConsoleSetFontError};
+use aster_console::{
+    font::BitmapFont, mode::ConsoleMode, AnyConsoleDevice, ConsoleCallback, ConsoleSetFontError,
+};
 use aster_keyboard::InputKey;
 use ostd::{
     mm::VmReader,
@@ -62,6 +64,15 @@ impl AnyConsoleDevice for FramebufferConsole {
     fn set_font(&self, font: BitmapFont) -> Result<(), ConsoleSetFontError> {
         self.inner.lock().0.set_font(font)
     }
+
+    fn set_mode(&self, mode: ConsoleMode) -> bool {
+        self.inner.lock().0.set_mode(mode);
+        true
+    }
+
+    fn mode(&self) -> Option<ConsoleMode> {
+        Some(self.inner.lock().0.mode())
+    }
 }
 
 impl FramebufferConsole {
@@ -73,6 +84,8 @@ impl FramebufferConsole {
             fg_color: Pixel::WHITE,
             bg_color: Pixel::BLACK,
             font: BitmapFont::new_basic8x8(),
+            is_output_enabled: true,
+
             bytes: alloc::vec![0u8; framebuffer.size()],
             backend: framebuffer,
         };
@@ -99,6 +112,9 @@ struct ConsoleState {
     fg_color: Pixel,
     bg_color: Pixel,
     font: BitmapFont,
+    /// Whether the output characters will be drawn in the framebuffer.
+    is_output_enabled: bool,
+
     bytes: Vec<u8>,
     backend: Arc<FrameBuffer>,
 }
@@ -140,7 +156,9 @@ impl ConsoleState {
         self.bytes.copy_within(offset.., 0);
         self.bytes[self.backend.size() - offset..].fill(0);
 
-        self.backend.write_bytes_at(0, &self.bytes).unwrap();
+        if self.is_output_enabled {
+            self.backend.write_bytes_at(0, &self.bytes).unwrap();
+        }
 
         self.y_pos -= self.font.height();
     }
@@ -184,7 +202,9 @@ impl ConsoleState {
             }
 
             // Write pixels to the framebuffer.
-            self.backend.write_bytes_at(off_st, render_buf).unwrap();
+            if self.is_output_enabled {
+                self.backend.write_bytes_at(off_st, render_buf).unwrap();
+            }
 
             offset.y_add(1);
         }
@@ -206,6 +226,32 @@ impl ConsoleState {
 
         Ok(())
     }
+
+    /// Sets the console mode (text or graphics).
+    pub(self) fn set_mode(&mut self, mode: ConsoleMode) {
+        if mode == ConsoleMode::Graphics {
+            self.is_output_enabled = false;
+            return;
+        }
+
+        if self.is_output_enabled {
+            return;
+        }
+
+        // We're switching from the graphics mode back to the text mode. The characters need to be
+        // redrawn in the framebuffer.
+        self.is_output_enabled = true;
+        self.backend.write_bytes_at(0, &self.bytes).unwrap();
+    }
+
+    /// Gets the current console mode.
+    pub(self) fn mode(&self) -> ConsoleMode {
+        if self.is_output_enabled {
+            ConsoleMode::Text
+        } else {
+            ConsoleMode::Graphics
+        }
+    }
 }
 
 impl EscapeOp for ConsoleState {

kernel/src/device/tty/mod.rs

@@ -1,6 +1,6 @@
 // SPDX-License-Identifier: MPL-2.0
 
-use aster_console::{font::BitmapFont, AnyConsoleDevice};
+use aster_console::{font::BitmapFont, mode::ConsoleMode, AnyConsoleDevice};
 use ostd::sync::LocalIrqDisabled;
 
 use self::{line_discipline::LineDiscipline, termio::CFontOp};
@@ -309,6 +309,20 @@ impl<D: TtyDriver> FileIo for Tty<D> {
 
                 self.handle_set_font(&font_op)?;
             }
+            IoctlCmd::KDSETMODE => {
+                let console = self.console()?;
+
+                let mode = ConsoleMode::try_from(arg as i32)?;
+                if !console.set_mode(mode) {
+                    return_errno_with_message!(Errno::EINVAL, "the console mode is not supported");
+                }
+            }
+            IoctlCmd::KDGETMODE => {
+                let console = self.console()?;
+
+                let mode = console.mode().unwrap_or(ConsoleMode::Text);
+                current_userspace!().write_val(arg, &(mode as i32))?;
+            }
             _ => (self.weak_self.upgrade().unwrap() as Arc<dyn Terminal>)
                 .job_ioctl(cmd, arg, false)?,
         }

kernel/src/fs/utils/ioctl.rs

@@ -44,6 +44,10 @@ pub enum IoctlCmd {
     TIOCGPTPEER = 0x40045441,
     /// font operations
     KDFONTOP = 0x4B72,
+    /// Get console mode
+    KDGETMODE = 0x4B3B,
+    /// Set console mode
+    KDSETMODE = 0x4B3A,
     /// Get tdx report using TDCALL
     TDXGETREPORT = 0xc4405401,
 }