Skip to content

Commit e64ad9d

Browse files
committed
UPLOAD
1 parent 5782131 commit e64ad9d

192 files changed

Lines changed: 12699 additions & 0 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

Makefile

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
CC := i686-elf-gcc
2+
AS := nasm
3+
LD := i686-elf-gcc
4+
OBJCOPY := i686-elf-objcopy
5+
6+
CFLAGS := -ffreestanding -O2 -Wall -Wextra -m32 -fno-pie -fno-stack-protector -nostdlib -nostdinc -Iinclude
7+
ASFLAGS := -f elf32
8+
LDFLAGS := -T linker.ld -ffreestanding -O2 -nostdlib -Wl,--build-id=none
9+
10+
SRC_DIR := src
11+
BUILD_DIR := build
12+
MODEL_BLOB := $(firstword $(wildcard assets/smollm-135m.gguf) $(wildcard assets/SmolLM2-135M-Instruct-Q4_K_M.gguf))
13+
MODEL_OBJ := $(BUILD_DIR)/model.o
14+
15+
ASM_SRCS := $(wildcard $(SRC_DIR)/*.s $(SRC_DIR)/*/*.s $(SRC_DIR)/*/*/*.s)
16+
C_SRCS := $(wildcard $(SRC_DIR)/*.c $(SRC_DIR)/*/*.c $(SRC_DIR)/*/*/*.c)
17+
18+
ASM_OBJS := $(patsubst $(SRC_DIR)/%.s,$(BUILD_DIR)/%.o,$(ASM_SRCS))
19+
C_OBJS := $(patsubst $(SRC_DIR)/%.c,$(BUILD_DIR)/%.o,$(C_SRCS))
20+
21+
OBJS := $(ASM_OBJS) $(C_OBJS)
22+
23+
ifneq ($(MODEL_BLOB),)
24+
OBJS += $(MODEL_OBJ)
25+
endif
26+
27+
all: $(BUILD_DIR) kernel.bin
28+
29+
$(BUILD_DIR):
30+
mkdir -p $(BUILD_DIR)
31+
32+
$(BUILD_DIR)/%.o: $(SRC_DIR)/%.s
33+
mkdir -p $(dir $@)
34+
$(AS) $(ASFLAGS) $< -o $@
35+
36+
$(BUILD_DIR)/%.o: $(SRC_DIR)/%.c
37+
mkdir -p $(dir $@)
38+
$(CC) $(CFLAGS) -c $< -o $@
39+
40+
kernel.bin: $(OBJS) linker.ld
41+
$(LD) $(LDFLAGS) -o $@ $(OBJS)
42+
43+
$(MODEL_OBJ): $(MODEL_BLOB)
44+
mkdir -p $(dir $@)
45+
$(OBJCOPY) -I binary -O elf32-i386 -B i386 --rename-section .data=.ai_model,alloc,load,readonly,data,contents $< $@
46+
47+
clean:
48+
rm -rf $(BUILD_DIR) kernel.bin
49+
50+
.PHONY: all clean

build.ps1

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
param(
2+
[ValidateSet("all", "clean")]
3+
[string]$Target = "all",
4+
[string]$ModelBlob
5+
)
6+
7+
$ErrorActionPreference = "Stop"
8+
9+
function Require-Command([string]$Name) {
10+
if (-not (Get-Command $Name -ErrorAction SilentlyContinue)) {
11+
throw "$Name not found in PATH"
12+
}
13+
}
14+
15+
function Invoke-Checked([string]$Exe, [string[]]$Args) {
16+
& $Exe @Args
17+
if ($LASTEXITCODE -ne 0) {
18+
throw "$Exe failed with exit code $LASTEXITCODE"
19+
}
20+
}
21+
22+
function Get-ObjPath([string]$SrcPath, [string]$SrcRoot, [string]$BuildRoot) {
23+
$rel = $SrcPath.Substring($SrcRoot.Length)
24+
$rel = $rel.TrimStart([System.IO.Path]::DirectorySeparatorChar, [System.IO.Path]::AltDirectorySeparatorChar)
25+
$dir = Split-Path $rel -Parent
26+
$base = [System.IO.Path]::GetFileNameWithoutExtension($rel)
27+
if ([string]::IsNullOrEmpty($dir)) {
28+
return (Join-Path $BuildRoot ($base + ".o"))
29+
}
30+
return (Join-Path $BuildRoot (Join-Path $dir ($base + ".o")))
31+
}
32+
33+
$root = $PSScriptRoot
34+
if ([string]::IsNullOrEmpty($root)) {
35+
$root = (Get-Location).Path
36+
}
37+
38+
$srcDir = Join-Path $root "src"
39+
$buildDir = Join-Path $root "build"
40+
$kernelBin = Join-Path $root "kernel.bin"
41+
$ldScript = Join-Path $root "linker.ld"
42+
$assetsDir = Join-Path $root "assets"
43+
44+
if ($Target -eq "clean") {
45+
if (Test-Path $buildDir) {
46+
Remove-Item -Recurse -Force $buildDir
47+
}
48+
if (Test-Path $kernelBin) {
49+
Remove-Item -Force $kernelBin
50+
}
51+
exit 0
52+
}
53+
54+
Require-Command "i686-elf-gcc"
55+
Require-Command "nasm"
56+
Require-Command "i686-elf-objcopy"
57+
58+
if (-not (Test-Path $ldScript)) {
59+
throw "linker.ld not found"
60+
}
61+
62+
if (-not (Test-Path $buildDir)) {
63+
New-Item -ItemType Directory -Force -Path $buildDir | Out-Null
64+
}
65+
66+
$asmSrcs = Get-ChildItem -Path $srcDir -Recurse -Filter "*.s"
67+
$cSrcs = Get-ChildItem -Path $srcDir -Recurse -Filter "*.c"
68+
69+
$objPaths = @()
70+
71+
$asFlags = @("-f", "elf32")
72+
$cFlags = @("-ffreestanding", "-O2", "-Wall", "-Wextra", "-m32", "-fno-pie", "-fno-stack-protector", "-nostdlib", "-nostdinc", "-Iinclude")
73+
74+
foreach ($src in $asmSrcs) {
75+
$obj = Get-ObjPath $src.FullName $srcDir $buildDir
76+
$objDir = Split-Path $obj -Parent
77+
if (-not (Test-Path $objDir)) {
78+
New-Item -ItemType Directory -Force -Path $objDir | Out-Null
79+
}
80+
Invoke-Checked "nasm" ($asFlags + @($src.FullName, "-o", $obj))
81+
$objPaths += $obj
82+
}
83+
84+
foreach ($src in $cSrcs) {
85+
$obj = Get-ObjPath $src.FullName $srcDir $buildDir
86+
$objDir = Split-Path $obj -Parent
87+
if (-not (Test-Path $objDir)) {
88+
New-Item -ItemType Directory -Force -Path $objDir | Out-Null
89+
}
90+
Invoke-Checked "i686-elf-gcc" ($cFlags + @("-c", $src.FullName, "-o", $obj))
91+
$objPaths += $obj
92+
}
93+
94+
$modelBlobPath = $ModelBlob
95+
if ([string]::IsNullOrEmpty($modelBlobPath)) {
96+
$candidateA = Join-Path $assetsDir "smollm-135m.gguf"
97+
$candidateB = Join-Path $assetsDir "SmolLM2-135M-Instruct-Q4_K_M.gguf"
98+
if (Test-Path $candidateA) {
99+
$modelBlobPath = $candidateA
100+
} elseif (Test-Path $candidateB) {
101+
$modelBlobPath = $candidateB
102+
}
103+
}
104+
105+
if (-not [string]::IsNullOrEmpty($modelBlobPath)) {
106+
if (-not (Test-Path $modelBlobPath)) {
107+
throw "Model blob not found: $modelBlobPath"
108+
}
109+
$modelObj = Join-Path $buildDir "model.o"
110+
Invoke-Checked "i686-elf-objcopy" @(
111+
"-I", "binary",
112+
"-O", "elf32-i386",
113+
"-B", "i386",
114+
"--rename-section", ".data=.ai_model,alloc,load,readonly,data,contents",
115+
$modelBlobPath,
116+
$modelObj
117+
)
118+
$objPaths += $modelObj
119+
}
120+
121+
$ldFlags = @("-T", $ldScript, "-ffreestanding", "-O2", "-nostdlib", "-Wl,--build-id=none")
122+
Invoke-Checked "i686-elf-gcc" ($ldFlags + @("-o", $kernelBin) + $objPaths)

docs/ai_architecture.md

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# AI Subsystem Architecture
2+
3+
## Overview
4+
The OS features a native, kernel-level Artificial Intelligence subsystem designed for running Quantized Large Language Models (LLMs) on embedded or resource-constrained x86 hardware. Unlike traditional user-space AI runtimes, this subsystem is integrated directly into the kernel to minimize latency and memory overhead.
5+
6+
## Core Components
7+
8+
### 1. The Model Loader (`src/ai/ai_model.c`)
9+
The loader handles the GGUF (GPT-Generated Unified Format) binary format, which is the industry standard for quantized local models.
10+
11+
- **Parsing**: Validates the `GGUF` magic bytes and parses the header version.
12+
- **Tensor Mapping**: Instead of loading the entire model into heap memory, the kernel maps the model blob (linked as `.ai_model` section) directly into the kernel address space.
13+
- **Quantization Support**: Native support for various quantization types:
14+
- `Q4_0`, `Q4_1` (4-bit weights)
15+
- `Q5_0`, `Q5_1` (5-bit)
16+
- `Q8_0` (8-bit)
17+
- `F16`, `F32` (Fallback)
18+
19+
### 2. The Neural Engine (`ai_matmul_q16_16`)
20+
The heart of the AI subsystem is the Matrix Multiplication (MatMul) engine, optimized for fixed-point arithmetic.
21+
22+
- **Data Type**: `q16_16_t` (32-bit fixed-point: 16-bit integer, 16-bit fractional). This avoids the overhead and unpredictability of floating-point units (FPU) in interrupt contexts.
23+
- **SIMD Acceleration**:
24+
- Uses inline assembly for **SSE4.1** instructions (`pmulld`, `psrad`).
25+
- **Vectorization**: Processes 4 tensor elements in parallel using `xmm` registers.
26+
- **Optimization**: Packs non-contiguous memory into temporary buffers for aligned SIMD loads.
27+
28+
```c
29+
// Example of the SIMD kernel loop (simplified)
30+
asm volatile(
31+
"movdqu (%1), %%xmm0\n" // Load 4 values from A
32+
"movdqu (%2), %%xmm1\n" // Load 4 values from B
33+
"pmulld %%xmm1, %%xmm0\n" // Multiply packed integers
34+
"psrad $16, %%xmm0\n" // Shift right to restore Q16.16 scale
35+
// ... accumulate result ...
36+
);
37+
```
38+
39+
### 3. Inference Pipeline
40+
The inference process follows a standard Transformer architecture pipeline:
41+
42+
1. **Tokenizer**: Encodes text prompts into integer tokens.
43+
2. **Embedding**: Looks up token vectors from the model weights.
44+
3. **Transformer Blocks**:
45+
- **RMSNorm**: Root Mean Square Normalization.
46+
- **QKV Attention**: Query-Key-Value attention mechanism with RoPE (Rotary Positional Embeddings).
47+
- **Feed Forward**: SwiGLU activation functions (`activation_silu`).
48+
4. **Sampling**: Top-P (Nucleus) sampling to select the next token.
49+
50+
### 4. KV Cache
51+
To speed up generation, the kernel maintains a Key-Value (KV) Cache.
52+
- **Structure**: Pre-allocated memory buffer storing previous attention contexts.
53+
- **Management**: Ring-buffer style management to handle context windows (e.g., 2048 tokens).
54+
55+
### 5. Neural Dispatch Scheduler
56+
To leverage multicore systems (SMP), the AI subsystem includes a dedicated job scheduler (`ai_scheduler`).
57+
- **Parallelism**: Large matrix multiplications are tiled into smaller chunks and dispatched to worker threads across available CPU cores.
58+
- **Lock-Free Queue**: Uses atomic operations for low-latency job submission.
59+
- **Fork-Join Model**: The main inference thread submits tasks and waits for completion (`ai_scheduler_wait`), ensuring synchronization for layer-by-layer execution.
60+
61+
## Integration with OS
62+
- **Direct Hardware Access**: The AI engine writes tokens directly to the VGA buffer for zero-latency display (`ai_stream_write`).
63+
- **Benchmarking**: Built-in `bench_matmul` command allows profiling the neural engine's TOPS (Trillions of Operations Per Second) equivalent.
64+
65+
## Future Roadmap
66+
- **AVX2 Support**: Extending the SIMD kernel to 256-bit registers (8 elements/cycle).
67+
- **Background Inference**: Moving inference to a low-priority background thread (`SCHED_CLASS_IDLE`) to keep the UI responsive.

docs/architecture.md

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Kernel Architecture Master Index
2+
3+
## 1. Introduction
4+
This OS represents a next-generation, professional-grade freestanding x86 kernel. It is engineered for high-assurance computing, featuring a hybrid architecture that combines the raw performance of a monolithic design with the modularity and security of a microkernel.
5+
6+
## 2. Documentation Hierarchy
7+
The documentation is structured into three tiers: **Architecture (Tier 1)**, **Internals (Tier 2)**, and **Developer Guides (Tier 3)**.
8+
9+
### Tier 1: Core Architecture & Concepts
10+
High-level overviews of system design, philosophy, and feature sets.
11+
- **[Memory Management Architecture](memory_management.md)**: Physical/Virtual memory philosophy, strategies.
12+
- **[Process & Scheduling Architecture](process_management.md)**: The task model, scheduling theory.
13+
- **[Device Subsystem Architecture](device_subsystems.md)**: Hardware abstraction layers and bus topologies.
14+
- **[Network Stack Architecture](network_stack.md)**: Protocol stack design and data flow.
15+
- **[Security Architecture](security.md)**: The capability model and threat model.
16+
- **[AI Subsystem Architecture](ai_architecture.md)**: Kernel-level inference engine design.
17+
18+
### Tier 2: Implementation Internals (Deep Dive)
19+
Exhaustive technical details, data structures, algorithms, and API references.
20+
- **[Memory Manager Internals](internals/memory_manager.md)**: PMM, VMM, Slab Allocator, and Kswapd implementation.
21+
- **[Process Scheduler Internals](internals/process_scheduler.md)**: Context switching, SMP load balancing, and runqueues.
22+
- **[Network Stack Internals](internals/network_stack.md)**: Socket buffers, protocol state machines, and driver interfaces.
23+
- **[Hardware Abstraction Internals](internals/hardware_abstraction.md)**: Interrupt routing (Apex), Driver Grid, and MSGI.
24+
- **[Storage & VFS Internals](internals/storage_vfs.md)**: Filesystem nodes, inode caching, journaling, and mounting.
25+
- **[System Call Interface](internals/syscall_interface.md)**: ABI conventions, interrupt vectors, and handler tables.
26+
- **[Security Subsystem Internals](internals/security_subsystem.md)**: Capability sets, policy engine, and audit logs.
27+
- **[Userland Loader & Init](internals/userland_init.md)**: ELF parsing, dynamic linking, and process bootstrapping.
28+
29+
### Tier 3: Developer & Operational Guides
30+
Practical manuals for building, debugging, and extending the kernel.
31+
- **[Kernel Developer Guide](kernel_dev_guide.md)**: Setup, build system, and contribution workflow.
32+
- **[Driver Interface Guide](driver_interface.md)**: Step-by-step driver creation tutorial.
33+
- **[Debugging & Diagnostics](debugging.md)**: Using Trace Forge, Perf Meter, and crash analysis.
34+
- **[Userland ABI Reference](userland_abi.md)**: System call reference for application developers.
35+
36+
## 3. Core Design Principles
37+
1. **Strict Isolation**: Hardware-enforced separation of kernel/user space with per-process page tables and guard pages.
38+
2. **Deterministic Execution**: O(1) scheduling and deadline-aware processing for real-time workloads.
39+
3. **Defense-in-Depth**: A security-first approach using capabilities, mandatory access control, and kernel self-protection.
40+
4. **Modular Extensibility**: Subsystems are decoupled via standardized interfaces (Driver Grid, VFS Ops), allowing hot-pluggable expansion.
41+
42+
## 4. Boot Sequence Overview
43+
1. **MBR/GRUB**: Loads kernel binary into memory.
44+
2. **`kernel_main`**:
45+
- **CPU**: GDT/IDT setup, ISR registration.
46+
- **Memory**: PMM initialization -> Paging enable -> Heap ready.
47+
- **Core**: Scheduler init, Timer calibration.
48+
- **Subsystems**: Apex INTC, PCI Enumeration, USB Nexus, Network Stack.
49+
- **Security**: Policy load, Capability system init.
50+
3. **Userland Handoff**: `init_orch` loads `/bin/init` and drops to Ring 3.
51+
52+
## 5. Build & Deploy
53+
- **Build System**: Cross-platform (PowerShell/Make) build automation.
54+
- **Target**: i686-elf (32-bit Protected Mode).
55+
- **Commands**:
56+
- `make`: Compiles the kernel.
57+
- `make iso`: Generates a bootable CD image.
58+
- `make qemu`: Runs the OS in emulation.

docs/debugging.md

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
# Debugging & Diagnostics
2+
3+
## Overview
4+
The OS provides a suite of tools for debugging kernel issues, tracing performance, and verifying system integrity.
5+
6+
## Trace Forge
7+
**Trace Forge** is a high-speed, low-overhead kernel tracing facility.
8+
9+
- **Purpose**: Record events (interrupts, context switches, syscalls) with minimal latency.
10+
- **Mechanism**: In-memory ring buffer (256 events).
11+
- **API**:
12+
```c
13+
trace_forge_emit(TAG_SCHED_SWITCH, pid);
14+
```
15+
- **Analysis**: Use `trace_forge_get()` to retrieve events from userland or a debugger.
16+
17+
### Common Tags
18+
- `0x01`: Context Switch
19+
- `0x02`: Page Fault
20+
- `0x03`: Syscall Entry
21+
- `0x04`: Interrupt Entry
22+
23+
## Diagnostics (`diag`)
24+
The `diag` subsystem provides structured logging.
25+
26+
- **Output**: Serial Port (COM1) by default.
27+
- **Format**: `[SEVERITY] Subsystem: Message`
28+
- **Usage**:
29+
```c
30+
diag_log(DIAG_INFO, "INIT", "Kernel started successfully.");
31+
```
32+
33+
## Perf Meter
34+
**Perf Meter** measures execution time in CPU cycles (RDTSC).
35+
36+
- **Usage**: Profiling critical sections.
37+
- **API**:
38+
```c
39+
perf_meter_start(METER_ID_SCHED);
40+
// ... critical code ...
41+
perf_meter_stop(METER_ID_SCHED);
42+
```
43+
44+
## Self-Tests
45+
The kernel includes built-in self-tests for critical algorithms.
46+
47+
- **Command**: Run `selftest` in the shell.
48+
- **Coverage**:
49+
- Fixed-point arithmetic
50+
- AI Model tensor operations
51+
- Memory allocation sanity checks
52+
53+
## Debugging Tips
54+
1. **QEMU Monitor**: Use `Ctrl+Alt+2` to access QEMU monitor.
55+
- `info registers`: Dump CPU state.
56+
- `xp /10i $eip`: Disassemble current instruction.
57+
2. **Serial Output**: Always redirect serial output to a file when running QEMU (`-serial file:log.txt`) to capture boot logs.
58+
3. **Panic**: If the kernel encounters an unrecoverable error, it will hang. Check the last serial log entry.

0 commit comments

Comments
 (0)