362 lines
10 KiB
ArmAsm
362 lines
10 KiB
ArmAsm
// SPDX-License-Identifier: (GPL-2.0+ OR MIT)
|
||
|
||
/*
|
||
* This file provides low-level assembly routines for cache and MMU control
|
||
* on ARM architectures, covering both AArch64 (ARMv8-A) and AArch32 (ARMv7-A).
|
||
*
|
||
* ================================================================
|
||
* AArch64 (ARMv8-A, EL2)
|
||
* ================================================================
|
||
* The original implementation targets AArch64 running at EL2 and
|
||
* relies on system registers and instructions such as:
|
||
*
|
||
* - System registers:
|
||
* SCTLR_EL2, TTBR0_EL2, TCR_EL2, MAIR_EL2
|
||
*
|
||
* - Instructions:
|
||
* MRS / MSR (system register access)
|
||
* TLBI (TLB invalidate)
|
||
* DC / IC (data/instruction cache operations)
|
||
*
|
||
* - Features:
|
||
* * Full MMU configuration via TTBR + TCR + MAIR
|
||
* * Fine-grained memory attribute control
|
||
* * Cache maintenance by level / set-way / address range
|
||
* * Exception level awareness (EL1 / EL2 / EL3)
|
||
*
|
||
* This version is typically used in ARM Trusted Firmware or bootloader
|
||
* stages (e.g., BL2/BL33) running in AArch64 mode.
|
||
*
|
||
*
|
||
* ================================================================
|
||
* AArch32 (ARMv7-A compatible version)
|
||
* ================================================================
|
||
* The adapted version replaces AArch64 system register access with
|
||
* ARMv7 CP15 interface:
|
||
*
|
||
* - System registers (CP15):
|
||
* SCTLR -> c1
|
||
* TTBR0 -> c2
|
||
* DACR -> c3
|
||
* TLB ops -> c8
|
||
* Cache ops -> c7
|
||
*
|
||
* - Instructions:
|
||
* MRC / MCR (CP15 register access)
|
||
*
|
||
* - Features:
|
||
* * MMU configuration via TTBR0 + DACR
|
||
* * No TCR/MAIR (memory attributes encoded in descriptors)
|
||
* * Cache maintenance via set/way or VA operations
|
||
* * Single privilege model (no EL abstraction)
|
||
*
|
||
* - Limitations vs AArch64:
|
||
* * No MAIR (uses TEX/C/B bits in page tables)
|
||
* * No TCR (address size & granularity fixed by design)
|
||
* * Less flexible cache and MMU configuration
|
||
*
|
||
*
|
||
* ================================================================
|
||
* Key Differences (AArch64 vs AArch32)
|
||
* ================================================================
|
||
*
|
||
* MMU Configuration:
|
||
* AArch64 : TTBR + TCR + MAIR
|
||
* AArch32 : TTBR0 + DACR (attributes in descriptors)
|
||
*
|
||
* Register Access:
|
||
* AArch64 : MRS / MSR
|
||
* AArch32 : MRC / MCR (CP15)
|
||
*
|
||
* Cache/TLB Ops:
|
||
* AArch64 : TLBI / DC / IC instructions
|
||
* AArch32 : CP15 c7/c8 operations
|
||
*
|
||
* Exception Model:
|
||
* AArch64 : EL0–EL3 hierarchy
|
||
* AArch32 : Privileged modes (SVC, HYP, etc.)
|
||
*
|
||
* Descriptor Format:
|
||
* AArch64 : 64-bit descriptors
|
||
* AArch32 : 32-bit section/page descriptors (or LPAE 64-bit)
|
||
*
|
||
*
|
||
* ================================================================
|
||
* Notes
|
||
* ================================================================
|
||
*
|
||
* - The AArch64 and AArch32 implementations are NOT compatible and
|
||
* must be compiled separately depending on target architecture.
|
||
*
|
||
* - When porting between them, special care must be taken for:
|
||
* * Page table format
|
||
* * Cache maintenance semantics
|
||
* * MMU enable sequence
|
||
*
|
||
* - This file is intended for early boot stages where cache/MMU
|
||
* must be initialized before C runtime is fully available.
|
||
*
|
||
*/
|
||
#include <arch.h>
|
||
|
||
.global __asm_get_ttbr0
|
||
.global __asm_set_ttbr0
|
||
.global __asm_get_dacr
|
||
.global __asm_set_dacr
|
||
.global __asm_invalidate_tlb_all
|
||
.global __asm_invalidate_icache_all
|
||
.global __asm_flush_dcache_all
|
||
.global __asm_invalidate_dcache_all
|
||
.global __asm_flush_dcache_range
|
||
|
||
/* ===================================== */
|
||
/* TTBR0 */
|
||
/* ===================================== */
|
||
__asm_get_ttbr0:
|
||
mrc p15, 0, r0, c2, c0, 0
|
||
bx lr
|
||
|
||
__asm_set_ttbr0:
|
||
mcr p15, 0, r0, c2, c0, 0
|
||
dsb
|
||
isb
|
||
bx lr
|
||
|
||
/* ===================================== */
|
||
/* DACR */
|
||
/* ===================================== */
|
||
__asm_get_dacr:
|
||
mrc p15, 0, r0, c3, c0, 0
|
||
bx lr
|
||
|
||
__asm_set_dacr:
|
||
mcr p15, 0, r0, c3, c0, 0
|
||
bx lr
|
||
|
||
/* ===================================== */
|
||
/* TLB invalidate */
|
||
/* ===================================== */
|
||
__asm_invalidate_tlb_all:
|
||
mov r0, #0
|
||
mcr p15, 0, r0, c8, c7, 0 // invalidate unified TLB
|
||
dsb
|
||
isb
|
||
bx lr
|
||
|
||
/* ===================================== */
|
||
/* I-Cache invalidate */
|
||
/* ===================================== */
|
||
__asm_invalidate_icache_all:
|
||
mov r0, #0
|
||
mcr p15, 0, r0, c7, c5, 0 // invalidate I-cache all
|
||
dsb
|
||
isb
|
||
bx lr
|
||
|
||
/* ===================================== */
|
||
/* D-Cache clean+invalidate all */
|
||
/* ===================================== */
|
||
__asm_flush_dcache_all:
|
||
push {r0-r1, lr}
|
||
|
||
dsb
|
||
|
||
/* Clean + invalidate entire data cache */
|
||
mov r0, #0
|
||
mcr p15, 0, r0, c7, c14, 0
|
||
|
||
/* Ensure completion */
|
||
dsb
|
||
isb
|
||
|
||
pop {r0-r1, pc}
|
||
|
||
/* ===================================== */
|
||
/* D-Cache invalidate all */
|
||
/* ===================================== */
|
||
__asm_invalidate_dcache_all:
|
||
push {r0-r1, r2, lr}
|
||
|
||
dsb
|
||
|
||
/* 1. disable D-cache first (critical) */
|
||
mrc p15, 0, r0, c1, c0, 0 @ SCTLR
|
||
bic r0, r0, #(1 << 2) @ clear C bit (D-cache disable)
|
||
mcr p15, 0, r0, c1, c0, 0
|
||
isb
|
||
|
||
dsb
|
||
|
||
/* 2. invalidate entire D-cache (safe now) */
|
||
mov r0, #0
|
||
mcr p15, 0, r0, c7, c6, 0 @ D-cache invalidate all (safe only when OFF)
|
||
|
||
dsb
|
||
isb
|
||
|
||
pop {r0-r1, r2, pc}
|
||
|
||
/* ===================================== */
|
||
/* flush range */
|
||
/* ===================================== */
|
||
__asm_flush_dcache_range:
|
||
push {r2-r4, lr}
|
||
|
||
mrc p15, 0, r2, c0, c0, 1 // CTR
|
||
lsr r2, r2, #16
|
||
and r2, r2, #0xf
|
||
|
||
mov r3, #4
|
||
lsl r3, r3, r2 // line size
|
||
|
||
sub r2, r3, #1
|
||
bic r0, r0, r2
|
||
|
||
1:
|
||
mcr p15, 0, r0, c7, c14, 1 // clean+invalidate by VA
|
||
add r0, r0, r3
|
||
cmp r0, r1
|
||
blo 1b
|
||
|
||
dsb
|
||
isb
|
||
pop {r2-r4, pc}
|
||
|
||
/*
|
||
* v7_flush_dcache_all()
|
||
*
|
||
* Flush the whole D-cache.
|
||
*
|
||
* Corrupted registers: r0-r6, r9-r10
|
||
*
|
||
* - mm - mm_struct describing address space
|
||
*/
|
||
.global __v7_flush_dcache_all
|
||
__v7_flush_dcache_all:
|
||
dmb @ ensure ordering with previous memory accesses
|
||
mrc p15, 1, r0, c0, c0, 1 @ read clidr
|
||
mov r3, r0, lsr #23 @ move LoC into position
|
||
ands r3, r3, #7 << 1 @ extract LoC*2 from clidr
|
||
beq finished @ if loc is 0, then no need to clean
|
||
start_flush_levels:
|
||
mov r10, #0 @ start clean at cache level 0
|
||
flush_levels:
|
||
add r2, r10, r10, lsr #1 @ work out 3x current cache level
|
||
mov r1, r0, lsr r2 @ extract cache type bits from clidr
|
||
and r1, r1, #7 @ mask of the bits for current cache only
|
||
cmp r1, #2 @ see what cache we have at this level
|
||
blt skip @ skip if no cache, or just i-cache
|
||
mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
|
||
isb @ isb to sych the new cssr&csidr
|
||
mrc p15, 1, r1, c0, c0, 0 @ read the new csidr
|
||
and r2, r1, #7 @ extract the length of the cache lines
|
||
add r2, r2, #4 @ add 4 (line length offset)
|
||
movw r4, #0x3ff
|
||
ands r4, r4, r1, lsr #3 @ find maximum number on the way size
|
||
clz r5, r4 @ find bit position of way size increment
|
||
movw r6, #0x7fff
|
||
and r1, r6, r1, lsr #13 @ extract max number of the index size
|
||
mov r6, #1
|
||
movne r4, r4, lsl r5 @ # of ways shifted into bits [31:...]
|
||
movne r6, r6, lsl r5 @ 1 shifted left by same amount
|
||
loop1:
|
||
mov r9, r1 @ create working copy of max index
|
||
loop2:
|
||
mov r5, r9, lsl r2 @ factor set number into r5
|
||
orr r5, r5, r4 @ factor way number into r5
|
||
orr r5, r5, r10 @ factor cache level into r5
|
||
mcr p15, 0, r5, c7, c14, 2 @ clean & invalidate by set/way
|
||
subs r9, r9, #1 @ decrement the index
|
||
bge loop2
|
||
subs r4, r4, r6 @ decrement the way
|
||
bcs loop1
|
||
skip:
|
||
add r10, r10, #2 @ increment cache number
|
||
cmp r3, r10
|
||
dsb
|
||
bgt flush_levels
|
||
finished:
|
||
mov r10, #0 @ switch back to cache level 0
|
||
mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
|
||
dsb st
|
||
isb
|
||
bx lr
|
||
|
||
.global v7_flush_dcache_all
|
||
v7_flush_dcache_all:
|
||
stmfd sp!, {r4-r6, r9-r10, lr}
|
||
bl __v7_flush_dcache_all
|
||
ldmfd sp!, {r4-r6, r9-r10, lr}
|
||
bx lr
|
||
|
||
/*
|
||
* v7_invalidate_dcache_all()
|
||
*
|
||
* Invalidate the whole D-cache.
|
||
*
|
||
* Corrupted registers: r0-r7, r9-r11 (r6 only in Thumb mode)
|
||
*
|
||
* Note: copied from __v7_flush_dcache_all above with
|
||
* mcr p15, 0, r11, c7, c14, 2
|
||
* Replaced with:
|
||
* mcr p15, 0, r11, c7, c6, 2
|
||
*/
|
||
.global __v7_invalidate_dcache_all
|
||
__v7_invalidate_dcache_all:
|
||
dmb @ ensure ordering with previous memory accesses
|
||
mrc p15, 1, r0, c0, c0, 1 @ read clidr
|
||
mov r3, r0, lsr #23 @ move LoC into position
|
||
ands r3, r3, #7 << 1 @ extract LoC*2 from clidr
|
||
beq inval_finished @ if loc is 0, then no need to clean
|
||
mov r10, #0 @ start clean at cache level 0
|
||
inval_levels:
|
||
add r2, r10, r10, lsr #1 @ work out 3x current cache level
|
||
mov r1, r0, lsr r2 @ extract cache type bits from clidr
|
||
and r1, r1, #7 @ mask of the bits for current cache only
|
||
cmp r1, #2 @ see what cache we have at this level
|
||
blt inval_skip @ skip if no cache, or just i-cache
|
||
mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
|
||
isb @ isb to sych the new cssr&csidr
|
||
mrc p15, 1, r1, c0, c0, 0 @ read the new csidr
|
||
and r2, r1, #7 @ extract the length of the cache lines
|
||
add r2, r2, #4 @ add 4 (line length offset)
|
||
movw r4, #0x3ff
|
||
ands r4, r4, r1, lsr #3 @ find maximum number on the way size
|
||
clz r5, r4 @ find bit position of way size increment
|
||
movw r6, #0x7fff
|
||
and r1, r6, r1, lsr #13 @ extract max number of the index size
|
||
mov r6, #1
|
||
movne r4, r4, lsl r5 @ # of ways shifted into bits [31:...]
|
||
movne r6, r6, lsl r5 @ 1 shifted left by same amount
|
||
inval_loop1:
|
||
mov r9, r1 @ create working copy of max index
|
||
inval_loop2:
|
||
mov r5, r9, lsl r2 @ factor set number into r5
|
||
orr r5, r5, r4 @ factor way number into r5
|
||
orr r5, r5, r10 @ factor cache level into r5
|
||
mcr p15, 0, r5, c7, c6, 2 @ invalidate by set/way
|
||
subs r9, r9, #1 @ decrement the index
|
||
bge inval_loop2
|
||
subs r4, r4, r6 @ decrement the way
|
||
bcs inval_loop1
|
||
inval_skip:
|
||
add r10, r10, #2 @ increment cache number
|
||
cmp r3, r10
|
||
dsb
|
||
bgt inval_levels
|
||
inval_finished:
|
||
mov r10, #0 @ switch back to cache level 0
|
||
mcr p15, 2, r10, c0, c0, 0 @ select current cache level in cssr
|
||
dsb st
|
||
isb
|
||
bx lr
|
||
|
||
.global v7_invalidate_dcache_all
|
||
v7_invalidate_dcache_all:
|
||
stmfd sp!, {r4-r6, r9-r10, lr}
|
||
bl __v7_invalidate_dcache_all
|
||
ldmfd sp!, {r4-r6, r9-r10, lr}
|
||
bx lr
|
||
|
||
.section .note.GNU-stack,"",%progbits
|