feat: introduce HOT! executable format and Ring 3 process isolation (v0.5.2)

- Implement custom 'HOT!' binary format and Rust-based elf2hot converter.
- Upgrade kernel loader with segment-based loading and BSS zeroing.
- Refactor scheduler for Ring 3 IRET frames and fix CS/SS selector swap.
- Add user stack allocation (0x70000000) and linker scripts for binary cleanup.

.gitignore

@@ -2,4 +2,6 @@
 build*
 .venv
 initrd/image.cpio
-initramfs/*
+initramfs/*
+
+target

CMakeLists.txt

@@ -42,7 +42,7 @@ if(MCOPY_EXE AND MKFS_EXE AND CPIO_EXE)
         COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}
         COMMAND sh -c "find . -mindepth 1 ! -name '*.cpio' -print0 | ${CPIO_EXE} --null -ov -H newc > \"${INITRAMFS_CPIO_FILE}\""
         WORKING_DIRECTORY ${INITRAMFS_SRC_DIR}
-        DEPENDS ${INIT_FILES} init
+        DEPENDS ${INIT_FILES} init_elf
         VERBATIM
         COMMENT "Packing initramfs to cpio..."
     )

common/hot_header.h

@@ -0,0 +1,23 @@
+typedef unsigned char      u8;
+typedef unsigned short     u16;
+typedef unsigned int       u32;
+typedef unsigned long long u64; 
+
+#define HOT_MAGIC 0x21544F48
+
+typedef struct hot_segment {
+    u64 type; // 1 = rx 2 = rw
+    u64 vaddr; 
+    u64 offset;
+    u64 filesz;
+    u64 memsz;
+} hot_segment;
+
+typedef struct hot_header {
+    u32 magic; // "HOT!"
+    u8 version; // 1
+    u8 reserved_pad[3];
+    u64 entry_point;
+    u64 segments_count;
+    u64 reserved;
+} hot_header;

kernel/inc/core/hot.h

@@ -2,5 +2,7 @@
 // Copyright (c) 2026 0xKarinyash
 
 #pragma once
+#include <types.h>
+#include <core/scheduler.h>
 
-void jump_to_userspace(void* entry, void* user_stack_top);
+u64 load_hot(process* proc, u8* data);

kernel/inc/core/scheduler.h

@@ -27,6 +27,6 @@ typedef struct task {
 } task;
 
 void sched_init();
-task* sched_spawn(void(*entry)(), process* owner);
+task* sched_spawn(void(*entry)(), process* owner, bool is_user, u64 fixed_user_stack);
 u64 sched_next(u64 curr_rsp);
 void yield(u64 ticks);

kernel/inc/mm/vmm.h

@@ -39,4 +39,5 @@ void vmm_init(Bootinfo* info);
 u64* vmm_map_page(u64* pml4, u64 phys, u64 virt, u64 flags);
 u64 vmm_create_address_space();
 u64 vmm_get_current_cr3();
-void load_cr3(u64 pml4_addr);
+void load_cr3(u64 pml4_addr);
+void vmm_setup_user_stack(u64* pml4_phys);

kernel/src/core/hot.c

@@ -0,0 +1,46 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Copyright (c) 2026 0xKarinyash
+
+#include <core/hot.h>
+#include <core/scheduler.h>
+#include <core/string.h>
+
+#include <mm/pmm.h>
+#include <mm/vmm.h>
+#include <mm/heap.h>
+#include <mm/memory.h>
+
+#include "../../common/hot_header.h"
+
+u64 load_hot(process* proc, u8* data) {
+    hot_header* header = (hot_header*)data;
+    if (header->magic != HOT_MAGIC) {
+        return -1;
+    }
+
+    hot_segment* segments = (hot_segment*)(data + sizeof(hot_header));
+    u64 kernel_cr3 = vmm_get_current_cr3();
+    for (u64 i = 0; i < header->segments_count; i++) {
+        hot_segment* seg = &segments[i];
+        if (seg->memsz == 0) continue;
+
+        u64 start = seg->vaddr & ~(0xFFF);
+        u64 end = (seg->vaddr + seg->memsz + 0xFFF) & ~(0xFFF);
+        for (u64 addr = start; addr < end; addr += PAGE_SIZE) {
+            void* phys = pmm_alloc_page();
+            vmm_map_page((u64*)proc->pml4_phys, (u64)phys, addr, PTE_USER | PTE_RW | PTE_PRESENT);  
+        }
+
+        load_cr3(proc->pml4_phys);
+        if (seg->filesz > 0) memcpy((void*)seg->vaddr, data + seg->offset, seg->filesz);
+        if (seg->memsz > seg->filesz) {
+            u64 bss_start = seg->vaddr + seg->filesz;
+            u64 bss_len = seg->memsz - seg->filesz;
+            memset((void*)bss_start, 0, bss_len);
+        }
+
+        load_cr3(kernel_cr3);
+    }
+
+    return header->entry_point;
+}

kernel/src/core/loader.c

@@ -1,8 +1,9 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 // Copyright (c) 2026 0xKarinyash
 
+#include <core/panic.h>
+#include <core/hot.h>
 #include <core/loader.h>
-#include <core/userspace.h>
 #include <core/scheduler.h>
 #include <core/string.h>
 
@@ -20,75 +21,27 @@
 
 extern task* curr_task;
 
-bool exec_init(process* p, const char* path) {
-    kprintf("[Loader] loading %s...\n", path);
-    fs_node* file = vfs_open(path);
-    if (!file) {
-        kprintf("[Loader] Error: %s not found in initramfs!\n", path);
-        return false;
-    }
-
-    u64 virt_code = 0x400000;
-    u64 virt_stack = 0x800000;
-    u64 stack_size = 8192;     
-
-    u64 bytes_left = file->len;
-    u64 offset = 0;
-    u64 page_idx = 0;
-
-    while (bytes_left > 0) {
-        void* phys = pmm_alloc_page();
-        if (!phys) {
-            kprintf("Loader: OOM!\n");
-            return false;
-        }
-
-        vmm_map_page((u64*)p->pml4_phys, (u64)phys, virt_code + (page_idx * 4096), PTE_PRESENT | PTE_RW | PTE_USER);
-
-        void* k_ptr = (void*)(HHDM_OFFSET + (u64)phys);
-        memset(k_ptr, 0, 4096);
-        
-        u64 chunk = (bytes_left > 4096) ? 4096 : bytes_left;
-        vfs_read(file, offset, chunk, (u8*)k_ptr);
-
-        bytes_left -= chunk;
-        offset += chunk;
-        page_idx++;
-    }
-
-    for (u64 i = 0; i < (stack_size / 4096); i++) {
-        void* phys = pmm_alloc_page();
-        vmm_map_page((u64*)p->pml4_phys, (u64)phys, virt_stack + (i * 4096), PTE_PRESENT | PTE_RW | PTE_USER);
-        memset((void*)(HHDM_OFFSET + (u64)phys), 0, 4096);
-    }
-
-    load_cr3(p->pml4_phys);
-
-    // __asm__ volatile(
-    //     "mov %%cr3, %%rax\n\t"
-    //     "mov %%rax, %%cr3\n\t"
-    //     ::: "rax", "memory"
-    // );
-
-    kprintf("[Loader] Transferring control to userspace...\n");
-    jump_to_userspace((void*)virt_code, (void*)(virt_stack + stack_size));
-    
-    return true; // unreachable
-}
+#define USER_STACK_TOP  0x70000000
 
 void init_task_entry() {
     process* init_proc = (process*)malloc(sizeof(process));
-    init_proc->pid = 1;
-    init_proc->state = RUNNING;
+    init_proc->pid = 1; 
+    init_proc->state = RUNNING; 
     init_proc->pml4_phys = vmm_create_address_space();
     strcpy(init_proc->name, "init");
 
-    curr_task->proc = init_proc;
+    fs_node* file = vfs_open("/init");
+    if (!file) panic("FATAL: /init not found!");
+
+    u8* file_buffer = (u8*)malloc(file->len);
+    vfs_read(file, 0, file->len, file_buffer);
+
+    u64 entry = load_hot(init_proc, file_buffer);
+    if (!entry) panic("Invalid HOT executable");
     
-    if (!exec_init(init_proc, "/init")) {
-        kprintf("FATAL: Could not load /init\n");
-        sched_spawn(ksh, nullptr); 
-        
-        while(1) __asm__("hlt"); 
-    }
+    free(file_buffer); 
+    vmm_setup_user_stack((u64*)init_proc->pml4_phys);
+    sched_spawn((void(*)())entry, init_proc, true, USER_STACK_TOP);
+
+    while(1) { __asm__("sti; hlt"); } 
 }

kernel/src/core/scheduler.c

@@ -12,7 +12,6 @@
 task* curr_task = nullptr;
 u32 next_pid = 1;
 
-
 extern void irq0_handler();
 extern u64 pml4_kernel_phys;
 
@@ -33,7 +32,7 @@ void sched_init() {
     curr_task = kt;
 }
 
-task* sched_spawn(void(*entry)(), process* owner) {
+task* sched_spawn(void(*entry)(), process* owner, bool is_user, u64 fixed_user_stack) {
     task* t = (task*)malloc(sizeof(task));
     if (!t) return nullptr;
     if (!owner) owner = &kernel_process;
@@ -43,10 +42,17 @@ task* sched_spawn(void(*entry)(), process* owner) {
     if (!stack_base) panic("OOM for task stack");
     u64* rsp = (u64*)(stack_base + stack_size); 
 
-    *--rsp = 0x10; // SS -- Kernel data
-    *--rsp = (u64)stack_base + stack_size; // rsp
-    *--rsp = 0x202; // RFLAGS -- Interrupts Enabled | Reserved bit;
-    *--rsp = 0x08; // CS -- Kernel Code;
+    u64 cs = is_user ? 0x23 : 0x08;
+    u64 ss = is_user ? 0x1b : 0x10;
+    u64 rflags = 0x202;
+    u64 target_rsp = 0;
+    if (is_user) target_rsp = fixed_user_stack;
+    else target_rsp = (u64)stack_base + stack_size;
+
+    *--rsp = ss; // SS -- Kernel data
+    *--rsp = target_rsp; // rsp
+    *--rsp = rflags; // RFLAGS -- Interrupts Enabled | Reserved bit;
+    *--rsp = cs; // CS -- Kernel Code;
     *--rsp = (u64)entry; // RIP
 
     *--rsp = 0; // int no

kernel/src/core/userspace.c

@@ -1,34 +0,0 @@
-// SPDX-License-Identifier: GPL-3.0-or-later
-// Copyright (c) 2026 0xKarinyash
-
-#include <core/userspace.h>
-#include <types.h>
-
-#define USER_DS (0x18 | 3)
-#define USER_CS (0x20 | 3)
-#define RFLAGS_IF 0x202
-
-void jump_to_userspace(void* entry, void* user_stack_top) {
-    __asm__ volatile (
-        "mov %0, %%ds\n"
-        "mov %0, %%es\n"
-        "mov %0, %%fs\n"
-        :: "r" ((u64)USER_DS) : "memory"
-    );
-
-    __asm__ volatile (
-        "pushq %0\n" // SS (User Data Selector)
-        "pushq %1\n" // RSP (User Stack Pointer)
-        "pushq %2\n" // RFLAGS
-        "pushq %3\n" // CS (User Code Selector)
-        "pushq %4\n" // RIP (Entry Point)
-        "iretq\n"
-        : 
-        : "r" ((u64)USER_DS),
-          "r" ((u64)user_stack_top),
-          "r" ((u64)RFLAGS_IF),
-          "r" ((u64)USER_CS),
-          "r" ((u64)entry)
-        : "memory"
-    );
-}

kernel/src/kmain.c

@@ -101,8 +101,8 @@ void kmain(Bootinfo* info) {
     c = console_getc();
     if (c != '\n') staying_in_ksh = true;
     
-    if (staying_in_ksh) sched_spawn(ksh, nullptr);
-    else sched_spawn(init_task_entry, nullptr);
+    if (staying_in_ksh) sched_spawn(ksh, nullptr, false, 0);
+    else sched_spawn(init_task_entry, nullptr, false, 0);
     
     __asm__ volatile("sti");
     

kernel/src/mm/vmm.c

@@ -4,12 +4,17 @@
 #include <mm/pmm.h>
 #include <mm/memory.h>
 
+#include <core/panic.h>
+
 #include <gdt.h>
 #include <idt.h>
 
 #include <types.h>
 #include "bootinfo.h"
 
+#define USER_STACK_TOP  0x70000000
+#define USER_STACK_SIZE 0x4000
+
 u64* pml4_kernel = nullptr;
 u64 pml4_kernel_phys = 0;
 static bool is_initialized = false;
@@ -155,4 +160,15 @@ u64 vmm_get_current_cr3() {
     u64 cr3;
     __asm__ volatile("mov %%cr3, %0" : "=r"(cr3));
     return cr3;
-}
+}
+
+
+void vmm_setup_user_stack(u64* pml4_phys) {
+    u64 stack_bottom = USER_STACK_TOP - USER_STACK_SIZE;
+    for (u64 addr = stack_bottom; addr < USER_STACK_TOP; addr += 4096) {
+        void* phys = pmm_alloc_page();
+        if (!phys) panic("OOM in user stack setup");
+        memset((void*)PHYS_TO_HHDM((u64)phys), 0, 4096);
+        vmm_map_page((u64*)pml4_phys, (u64)phys, addr, PTE_PRESENT | PTE_RW | PTE_USER);
+    }
+}

kernel/src/shell/builtins.c

@@ -124,13 +124,5 @@ void cmd_ver() {
 }
 
 void cmd_userspace() {
-    process* init_proc = (process*)malloc(sizeof(process));
-    init_proc->pid = 1;
-    init_proc->state = RUNNING;
-    init_proc->pml4_phys = vmm_create_address_space();
-    strcpy(init_proc->name, "init");
-
-    curr_task->proc = init_proc;
-    kprintf("Trying to jump in ring 3...\n");
-    if (!exec_init(init_proc, "/init")) kprintf("Failed to jump.\n");
+    sched_spawn(init_task_entry, nullptr, false, 0);
 }

kernel/src/shell/ksh.c

@@ -86,7 +86,7 @@ ksh_token char2token(char* token) {
 }
 
 void ksh() {
-    sched_spawn(cursor_blinker_sched_task, nullptr);
+    sched_spawn(cursor_blinker_sched_task, nullptr, false, 0);
     while (true) {
         kprintf("ksh_> ");
         char cmdbuff[256];

tools/elf2hot/Cargo.lock

@@ -0,0 +1,88 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "elf2hot"
+version = "0.1.0"
+dependencies = [
+ "goblin",
+]
+
+[[package]]
+name = "goblin"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4db6758c546e6f81f265638c980e5e84dfbda80cfd8e89e02f83454c8e8124bd"
+dependencies = [
+ "log",
+ "plain",
+ "scroll",
+]
+
+[[package]]
+name = "log"
+version = "0.4.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
+
+[[package]]
+name = "plain"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.106"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "scroll"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1257cd4248b4132760d6524d6dda4e053bc648c9070b960929bf50cfb1e7add"
+dependencies = [
+ "scroll_derive",
+]
+
+[[package]]
+name = "scroll_derive"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.114"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"

tools/elf2hot/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "elf2hot"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+goblin = "0.10.4"

tools/elf2hot/src/main.rs

@@ -0,0 +1,98 @@
+use goblin::elf::program_header::PT_LOAD;
+use std::env;
+use std::fs::File;
+use std::io::{Read, Write};
+
+#[repr(C, packed)]
+#[derive(Default, Debug)]
+struct HotHeader {
+    magic: u32,
+    version: u8,
+    reserved_pad: [u8; 3],
+    entry_point: u64,
+    segments_count: u64,
+    reserved: u64,
+}
+
+#[repr(C, packed)]
+#[derive(Default, Debug, Clone, Copy)]
+struct HotSegment {
+    stype: u64,
+    vaddr: u64,
+    offset: u64, 
+    filesz: u64,
+    memsz: u64,
+}
+
+const HOT_MAGIC: u32 = 0x21544F48; // "HOT!"
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args: Vec<String> = env::args().collect();
+    if args.len() < 3 {
+        println!("Usage: elf2hot <input_elf> <output_hot>");
+        return Ok(());
+    }
+
+    let input_path = &args[1];
+    let output_path = &args[2];
+
+    let mut f = File::open(input_path)?;
+    let mut buffer = Vec::new();
+    f.read_to_end(&mut buffer)?;
+
+    let elf = goblin::elf::Elf::parse(&buffer)?;
+    
+    let mut hot_segments = Vec::new();
+    let mut segment_data = Vec::new();
+
+    let mut current_offset = std::mem::size_of::<HotHeader>() as u64;
+
+    let load_segments: Vec<_> = elf.program_headers.iter()
+        .filter(|ph| ph.p_type == PT_LOAD)
+        .collect();
+    
+    current_offset += (load_segments.len() * std::mem::size_of::<HotSegment>()) as u64;
+
+    for ph in load_segments {
+        let is_code = ph.is_executable();
+        
+        let data = &buffer[ph.p_offset as usize..(ph.p_offset + ph.p_filesz) as usize];
+        segment_data.push(data);
+
+        hot_segments.push(HotSegment {
+            stype: if is_code { 1 } else { 2 },
+            vaddr: ph.p_vaddr,
+            offset: current_offset,
+            filesz: ph.p_filesz,
+            memsz: ph.p_memsz,
+        });
+
+        current_offset += ph.p_filesz;
+    }
+
+    let header = HotHeader {
+        magic: HOT_MAGIC,
+        version: 1,
+        entry_point: elf.entry,
+        segments_count: hot_segments.len() as u64,
+        ..Default::default()
+    };
+
+    let mut out = File::create(output_path)?;
+
+    let header_bytes: [u8; std::mem::size_of::<HotHeader>()] = unsafe { std::mem::transmute(header) };
+    out.write_all(&header_bytes)?;
+
+    for seg in hot_segments {
+        let seg_bytes: [u8; std::mem::size_of::<HotSegment>()] = unsafe { std::mem::transmute(seg) };
+        out.write_all(&seg_bytes)?;
+    }
+
+    for data in segment_data {
+        out.write_all(data)?;
+    }
+
+    println!("Successfully converted {} to {}!", input_path, output_path);
+    println!("Entry point: 0x{:X}", elf.entry);
+    Ok(())
+}

userspace/init/CMakeLists.txt

@@ -5,28 +5,29 @@ set(CMAKE_C_STANDARD 23)
 set(CMAKE_C_STANDARD_REQUIRED ON)
 set(CMAKE_C_EXTENSIONS OFF)
 
-message(STATUS "Building termOS's init")
+get_filename_component(TOOLS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../tools/elf2hot" ABSOLUTE)
+set(FINAL_INIT_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../initramfs/init")
 
-file(GLOB_RECURSE INIT_SOURCES CMAKE_CONFIGURE_DEPENDS
-    "src/*.asm"
-    "src/*.c"
+file(GLOB_RECURSE INIT_SOURCES "src/*.asm" "src/*.c")
+
+add_executable(init_elf ${INIT_SOURCES})
+
+set_target_properties(init_elf PROPERTIES 
+    OUTPUT_NAME "init"
+    LINKER_LANGUAGE C
 )
 
-add_executable(init ${INIT_SOURCES})
-set_target_properties(init PROPERTIES
-    SUFFIX ""
-    LINKER_LANGUAGE C 
-)
-
-target_link_options(init PRIVATE
+target_link_options(init_elf PRIVATE
     -nostdlib
     -static
-    -Wl,--oformat=binary
-    -Wl,-Ttext=0x400000
-    -Wl,-e,start
+    -T "${CMAKE_CURRENT_SOURCE_DIR}/linker.ld"
 )
 
-add_custom_command(TARGET init POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:init> ${CMAKE_SOURCE_DIR}/initramfs/init
-    COMMENT "Installing init binary to initramfs"
+add_custom_command(TARGET init_elf POST_BUILD
+    COMMAND cargo run --release --quiet -- $<TARGET_FILE:init_elf> ${FINAL_INIT_PATH}
+    
+    WORKING_DIRECTORY ${TOOLS_DIR}
+    
+    COMMENT "Cargo is converting ELF to HOT! format..."
+    VERBATIM
 )

userspace/init/linker.ld

@@ -0,0 +1,35 @@
+ENTRY(start)
+
+SECTIONS
+{
+    . = 0x400000;
+
+    .text : {
+        *(.text)
+    }
+
+    .rodata : {
+        *(.rodata)
+    }
+
+    .data : {
+        *(.data)
+    }
+
+    .bss : {
+        *(.bss)
+        *(COMMON)
+    }
+
+    /DISCARD/ : {
+        *(.note.gnu.build-id)
+        *(.note.GNU-stack)
+        *(.note.gnu.property)
+        *(.comment)
+        *(.interp)
+        *(.dynsym)
+        *(.dynstr)
+        *(.hash)
+        *(.gnu.hash)
+    }
+}