cdnutter/iBoot
Archived
1
0 Fork 0
Code Issues Packages Projects Releases Wiki Activity

first and last commit

Browse Source
This commit is contained in:
Chris Nutter committed 2023-07-08 13:03:17 -07:00
commit 1bf4efd552
2612 files changed
+636500
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

arch/arm64/asm.S
+276
View File
@@ -0,0 +1,276 @@
/*
* Copyright (c) 2011-2012, 2014 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <arch/arm64/proc_reg.h>
.text
.align 2
.globl _arm_read_sctlr
_arm_read_sctlr:
mrs x0, SCTLR_ELx
ret
.text
.align 2
.globl _arm_write_sctlr
_arm_write_sctlr:
msr SCTLR_ELx, x0
dsb sy
isb sy
// The ifdef should match the ifdef in arch_cpu_quiesce()
#if CPU_APPLE_CYCLONE
// Don't allow MMU or WXN to be disabled. Placing the check after the msr
// means that a ROP attack will fail unless an attacker can trigger an
// interrupt between the msr and check
mov x1, x0
orr x0, x0, #SCTLR_M_ENABLED
orr x0, x0, #SCTLR_WXN_ENABLED
cmp x0, x1
b.ne _arm_write_sctlr
#endif
ret
#if WITH_EL3
.text
.align 2
.globl _arm_write_scr
_arm_write_scr:
msr SCR_EL3, x0
isb sy
ret
#endif
#if WITH_L2_AS_RAM
.text
.align 2
.globl _arm_read_l2_cramconfig
_arm_read_l2_cramconfig:
mrs x0, s3_3_c15_c7_0
ret
#endif
.text
.align 2
.globl _arm_write_mair
_arm_write_mair:
msr MAIR_ELx, x0
isb sy
ret
.text
.align 2
.globl _arm_write_tcr
_arm_write_tcr:
msr TCR_ELx, x0
isb sy
ret
.text
.align 2
.globl _arm_write_ttbr0
_arm_write_ttbr0:
msr TTBR0_ELx, x0
isb sy
ret
.text
.align 2
.globl _arm_read_cpacr
_arm_read_cpacr:
mrs x0, CPACR_EL1
ret
.text
.align 2
.globl _arm_write_cpacr
_arm_write_cpacr:
msr CPACR_EL1, x0
isb sy
ret
.text
.align 2
.globl _arm_read_cntp_ctl
_arm_read_cntp_ctl:
isb sy
mrs x0, CNTP_CTL_EL0
ret
.text
.align 2
.globl _arm_write_cntp_ctl
_arm_write_cntp_ctl:
msr CNTP_CTL_EL0, x0
isb sy
ret
.text
.align 2
.globl _arm_read_cntpct
_arm_read_cntpct:
isb sy
mrs x0, CNTPCT_EL0
ret
.text
.align 2
.globl _arm_write_cntp_tval
_arm_write_cntp_tval:
msr CNTP_TVAL_EL0, x0
isb sy
ret
.text
.align 2
.globl _arm_flush_tlbs
_arm_flush_tlbs:
dsb sy
#if WITH_EL3
tlbi alle3
#else
tlbi vmalle1
#endif
dsb sy
isb sy
ret
.text
.align 2
.globl _arm_invalidate_icache
_arm_invalidate_icache:
ic iallu
dsb sy
isb sy
ret
.text
.align 2
.globl _arm_invalidate_dcache_line
_arm_invalidate_dcache_line:
ret
.text
.align 2
.globl _arm_clean_dcache_line
_arm_clean_dcache_line:
ret
.text
.align 2
.globl _arm_clean_invalidate_dcache
_arm_clean_invalidate_dcache:
ret
.text
.align 2
.globl _arm_clean_dcache
_arm_clean_dcache:
ret
.text
.align 2
.globl _arm_invalidate_dcache
_arm_invalidate_dcache:
ret
.text
.align 2
.globl _arm_clean_invalidate_dcache_line
_arm_clean_invalidate_dcache_line:
dc civac, x0
ret
.text
.align 2
.globl _arm_drain_write_buffer
_arm_drain_write_buffer:
ret
.text
.align 2
.globl _arm_memory_barrier
_arm_memory_barrier:
dmb sy
ret
.text
.align 2
.globl _arm_vtop
_arm_vtop:
mrs x2, DAIF // Load current DAIF
msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF) // Disable IRQ
isb sy
mov x9, x0 // Saved address
at s1e3r, x0 // Translation Stage 1 EL1
isb sy
mrs x1, PAR_EL1 // Read result
tst x1, #1 // Test Translation not valid
cset x0, eq
b.ne L_arm_vtop_ret
movk x3, #0xf000
movk x3, #0xffff, lsl #16
movk x3, #0xffff, lsl #32
and x1, x1, x3 // Get physical address base
and x0, x9, #0xfff
orr x0, x1, x0 // Add page offset
L_arm_vtop_ret:
msr DAIF, x2 // Restore interrupt state, return
isb sy
ret
.text
.align 2
.globl _arm_disable_async_aborts
_arm_disable_async_aborts:
msr DAIFSet, #(DAIFSC_ASYNCF)
isb sy
ret
.text
.align 2
.globl _arm_enable_async_aborts
_arm_enable_async_aborts:
msr DAIFClr, #(DAIFSC_ASYNCF)
isb sy
ret
.text
.align 2
.globl _arch_disable_ints
_arch_disable_ints:
msr DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)
isb sy
ret
.text
.align 2
.globl _arch_enable_ints
_arch_enable_ints:
msr DAIFClr, #(DAIFSC_IRQF | DAIFSC_FIQF)
isb sy
ret
.text
.align 2
.globl _arch_halt
_arch_halt:
dsb sy
isb sy
wfi
ret
.text
.align 2
.globl _arch_spin
_arch_spin:
hint 0x45 // make fastsim drop to debugger (nop on hardware)
b .
arch/arm64/context.S
+107
View File
@@ -0,0 +1,107 @@
/*
* Copyright (c) 2011-2012 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
/*
* Task context switch support.
*
*/
/*
* Code here must remain synchronized with the structure and offsets defined in arch_task.h:
*
* struct arch_task {
* uint64_t regs[29]; // x0-x28
* uint64_t fp;
* uint64_t lr;
* uint64_t sp;
*
* union {
* uint128_t q;
* uint64_t d;
* uint32_t s;
* } vregs[32]; // v0-v31
* uint32_t fpsr;
* uint32_t fpcr;
* };
*/
/* AAPCS-64 5.1.1 Page 13:
* Preserve r16-17, r19-r28, FP, and SP.
*
* AAPCS-64 5.1.2 Page 14:
* Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
* calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
* (or should be preserved by the caller).
*/
/*
* void arch_task_context_switch(struct arch_task *old, struct arch_task *new_task);
*/
.text
.align 2
.globl _arch_task_context_switch
_arch_task_context_switch:
stp x16, x17, [x0, #120]
stp x19, x20, [x0, #152]
stp x21, x22, [x0, #168]
stp x23, x24, [x0, #184]
stp x25, x26, [x0, #200]
stp x27, x28, [x0, #216]
stp fp, lr, [x0, #232]
mov x2, sp
str x2, [x0, #248]
#if WITH_VFP
str d8, [x0, #384]
str d9, [x0, #400]
str d10, [x0, #416]
str d11, [x0, #432]
str d12, [x0, #448]
str d13, [x0, #464]
str d14, [x0, #480]
str d15, [x0, #496]
#endif
mov x0, x1
b _arch_task_context_restore
/*
* void arch_task_context_restore(struct arch_task *restore_context)
*/
.text
.align 2
.globl _arch_task_context_restore
_arch_task_context_restore:
ldp x16, x17, [x0, #120]
ldp x19, x20, [x0, #152]
ldp x21, x22, [x0, #168]
ldp x23, x24, [x0, #184]
ldp x25, x26, [x0, #200]
ldp x27, x28, [x0, #216]
ldp fp, lr, [x0, #232]
ldr x1, [x0, #248]
mov sp, x1
#if WITH_VFP
ldr d8, [x0, #384]
ldr d9, [x0, #400]
ldr d10, [x0, #416]
ldr d11, [x0, #432]
ldr d12, [x0, #448]
ldr d13, [x0, #464]
ldr d14, [x0, #480]
ldr d15, [x0, #496]
#endif
ret
arch/arm64/cpu.c
+100
View File
@@ -0,0 +1,100 @@
/*
* Copyright (C) 2011-2012, 2014 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <arch.h>
#include <arch/arm/arm.h>
#include <arch/arm64/proc_reg.h>
#include <debug.h>
#include <platform.h>
#include <platform/memmap.h>
extern void arm_write_sctlr(uint64_t);
extern uint64_t arm_read_sctlr();
extern void arm_enable_async_aborts();
#if WITH_EL3
extern void arm_write_scr(uint64_t);
#endif
int arch_cpu_init(bool resume)
{
uint64_t sctlr;
#if WITH_EL3
uint64_t scr;
/* enable interrupts forwarding to EL3 */
scr = (SCR_EA | SCR_FIQ | SCR_IRQ);
arm_write_scr(scr);
#endif
/* enable async aborts */
arm_enable_async_aborts();
/* invalidate caches */
arm_invalidate_icache();
arm_invalidate_dcache();
sctlr = arm_read_sctlr();
/* turn on Stack Alignment check */
sctlr |= SCTLR_SA_ENABLED;
/* turn on the mmu */
arm_mmu_init(resume);
sctlr |= SCTLR_M_ENABLED;
/* disallow executing from writeable pages */
sctlr |= SCTLR_WXN_ENABLED;
/* turn on d-cache */
sctlr |= SCTLR_D_ENABLED;
/* turn on i-cache */
sctlr |= SCTLR_I_ENABLED;
/* enable the MMU, caches and etc. */
arm_write_sctlr(sctlr);
#if WITH_VFP
/* initialize VFP */
arm_fp_init();
#endif
return 0;
}
int arch_cpu_quiesce(void)
{
#if !CPU_APPLE_CYCLONE
uint64_t sctlr;
/* clean the d-cache */
arm_clean_dcache();
/* read the control regiser base value */
sctlr = arm_read_sctlr();
sctlr &= ~(SCTLR_D_ENABLED | SCTLR_SA_ENABLED | SCTLR_I_ENABLED | SCTLR_M_ENABLED | SCTLR_WXN_ENABLED);
/* disable the MMU, caches and etc. */
arm_write_sctlr(sctlr);
/* invalidate i-cache */
arm_invalidate_icache();
#endif /* !CPU_APPLE_CYCLONE */
return 0;
}
int arch_cpu_init_posttasks(void)
{
return 0;
}
arch/arm64/entropy.S
+41
View File
@@ -0,0 +1,41 @@
/*
* Copyright (C) 2012 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <arch/arm/assembler.h>
.text
.balign 4
.globl _arch_get_entropy
// u_int32_t arch_get_entropy(volatile u_int32_t *counter_address)
//
// arch_get_entropy exploits the uncertainty in the asynchronous boundary crossing
// between the CPU and a counter. To work correctly, the CPU and counter must be
// derived from clocks with relative jitter (PLL vs. OSC). Ideally the inner loop,
// up to 32 iterations, should consume a large part of the effective read period of
// the counter.
//
_arch_get_entropy:
mov x1, x0
mov w0, #0
mov w2, #32
1:
mov w3, w2
2:
subs w3, w3, #1
b.ne 2b
ldr w3, [x1]
and w3, w3, #1
orr w0, w3, w0, lsl #1
subs w2, w2, #1
b.ne 1b
ret
arch/arm64/exceptions_el1.S
+306
View File
@@ -0,0 +1,306 @@
/*
* Copyright (C) 2011-2012, 2014 Apple Computer, Inc. All rights reserved.
*
* This document is the property of Apple Computer, Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Computer, Inc.
*/
#if WITH_EL3
#error "This file should not be built for platforms WITH_EL3"
#endif
////////////////////////////////////////////////////////////
// This is what exception frame looks like:
//
// struct arm_exception_frame64 {
// uint64_t regs[29]; // x0-x28
// uint64_t fp;
// uint64_t lr;
// uint64_t sp;
// uint32_t spsr;
// uint32_t reserved0;
// uint64_t pc;
// uint64_t far;
// uint32_t esr;
// uint32_t reserved1;
// uint64_t reserved2; // stp requires multiple of 16 in imm field
// uint64_t reserved3; // stp requires multiple of 16 in imm field
// union {
// uint128_t q;
// uint64_t d;
// uint32_t s;
// } vregs[32]; // v0-v31
// uint32_t fpsr;
// uint32_t reserved4;
// uint32_t fpcr;
// uint32_t reserved5;
// };
////////////////////////////////////////////////////////////
.macro EL1_SP0_VECTOR
msr SPSel, #0 // Switch to SP0
sub sp, sp, #832 // Create exception frame
stp x0, x1, [sp, #0] // Save x0, x1 to exception frame
add x0, sp, #832 // Calculate the original stack pointer
str x0, [sp, #248] // Save stack pointer to exception frame
stp fp, lr, [sp, #232] // Save fp and lr to exception frame
mrs lr, ELR_EL1 // Get exception link reg so _dispatch64 can make a frame
mov x0, sp // Copy saved state pointer to x0
.endmacro
.macro FASTSIM_DEBUG_HINT
hint 0x45 // make fastsim drop to debugger (nop on hardware)
.endmacro
.text
.align 12
.globl _exception_vector_base
_exception_vector_base:
L__el1_sp0_synchronous_vector:
EL1_SP0_VECTOR
mrs x1, ESR_EL1 // Load exception syndrome
str x1, [x0, #280]
mrs x1, FAR_EL1 // Load fault address
str x1, [x0, #272]
adrp x1, _arm_synchronous_exception@page
add x1, x1, _arm_synchronous_exception@pageoff
b __dispatch64
.text
.align 7
L__el1_sp0_irq_vector:
EL1_SP0_VECTOR
adrp x1, _interrupt_stack_top@page
add x1, x1, _interrupt_stack_top@pageoff
ldr x1, [x1]
mov sp, x1
adrp x1, _arm_irq@page
add x1, x1, _arm_irq@pageoff
b __dispatch64
.text
.align 7
L__el1_sp0_fiq_vector:
EL1_SP0_VECTOR
adrp x1, _interrupt_stack_top@page
add x1, x1, _interrupt_stack_top@pageoff
ldr x1, [x1]
mov sp, x1
adrp x1, _arm_fiq@page
add x1, x1, _arm_fiq@pageoff
b __dispatch64
.text
.align 7
L__el1_sp0_serror_vector:
EL1_SP0_VECTOR
mrs x1, ESR_EL1 // Load exception syndrome
str x1, [x0, #280]
mrs x1, FAR_EL1 // Load fault address
str x1, [x0, #272]
adrp x1, _arm_serror_exception@page
add x1, x1, _arm_serror_exception@pageoff
b __dispatch64
.text
.align 7
L__el1_sp1_synchronous_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__el1_sp1_irq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__el1_sp1_fiq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__el1_sp1_serror_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_synchronous_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_irq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_fiq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_serror_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_synchronous_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_irq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_fiq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_serror_vector:
FASTSIM_DEBUG_HINT
b .
/* Placeholder for macho_post_process.py. Placing the UUID here
allows debuggers to locate the UUID based on the value of VBAR */
.text
.align 7
_UUID:
.space 32
/* 64-bit first level exception handler dispatcher.
* Completes register context saving and branches to FLEH.
* Expects:
* {x0, x1, fp, lr, sp} - saved
* x0 - arm_context_t
* x1 - address of FLEH
* fp - previous stack frame if EL1
* lr - unused
* sp - kernel stack
*/
.text
.align 2
__dispatch64:
stp fp, lr, [sp, #-16]! // Create a frame to help debuggers unwind
mov fp, sp // past the exception handler
stp x2, x3, [x0, #16] // Save remaining GPRs
stp x4, x5, [x0, #32]
stp x6, x7, [x0, #48]
stp x8, x9, [x0, #64]
stp x10, x11, [x0, #80]
stp x12, x13, [x0, #96]
stp x14, x15, [x0, #112]
stp x16, x17, [x0, #128]
stp x18, x19, [x0, #144]
stp x20, x21, [x0, #160]
stp x22, x23, [x0, #176]
stp x24, x25, [x0, #192]
stp x26, x27, [x0, #208]
str x28, [x0, #224]
mrs lr, ELR_EL1 // Get exception link register
str lr, [x0, #264] // Save ELR to PCB
mrs x2, SPSR_EL1 // Get CPSR
str w2, [x0, #256] // Save CPSR to PCB
#if WITH_VFP
stp q0, q1, [x0, #304]
stp q2, q3, [x0, #336]
stp q4, q5, [x0, #368]
stp q6, q7, [x0, #400]
stp q8, q9, [x0, #432]
stp q10, q11, [x0, #464]
stp q12, q13, [x0, #496]
stp q14, q15, [x0, #528]
stp q16, q17, [x0, #560]
stp q18, q19, [x0, #592]
stp q20, q21, [x0, #624]
stp q22, q23, [x0, #656]
stp q24, q25, [x0, #688]
stp q26, q27, [x0, #720]
stp q28, q29, [x0, #752]
stp q30, q31, [x0, #784]
mrs x2, FPSR
str w2, [x0, #816]
mrs x2, FPCR
str w2, [x0, #820]
#endif
mov x21, x0 // Copy arm_context_t pointer to x21
blr x1
b __exception_return
.text
.align 2
__exception_return:
mov sp, x21 // Reload the arm_context_t pointer
#if WITH_VFP
ldp q0, q1, [sp, #304]
ldp q2, q3, [sp, #336]
ldp q4, q5, [sp, #368]
ldp q6, q7, [sp, #400]
ldp q8, q9, [sp, #432]
ldp q10, q11, [sp, #464]
ldp q12, q13, [sp, #496]
ldp q14, q15, [sp, #528]
ldp q16, q17, [sp, #560]
ldp q18, q19, [sp, #592]
ldp q20, q21, [sp, #624]
ldp q22, q23, [sp, #656]
ldp q24, q25, [sp, #688]
ldp q26, q27, [sp, #720]
ldp q28, q29, [sp, #752]
ldp q30, q31, [sp, #784]
ldr w3, [sp, #816]
msr FPSR, x3
ldr w4, [sp, #820]
msr FPCR, x4
#endif
ldr x0, [sp, #264] // Get the return address
msr ELR_EL1, x0 // Load the return address into ELR
ldr w1, [sp, #256] // Get the return CPSR
msr SPSR_EL1, x1 // Load the return CPSR into SPSR
ldp x0, x1, [sp, #0] // Restore the GPRs
ldp x2, x3, [sp, #16]
ldp x4, x5, [sp, #32]
ldp x6, x7, [sp, #48]
ldp x8, x9, [sp, #64]
ldp x10, x11, [sp, #80]
ldp x12, x13, [sp, #96]
ldp x14, x15, [sp, #112]
ldp x16, x17, [sp, #128]
ldp x18, x19, [sp, #144]
ldp x20, x21, [sp, #160]
ldp x22, x23, [sp, #176]
ldp x24, x25, [sp, #192]
ldp x26, x27, [sp, #208]
ldr x28, [sp, #224]
ldp fp, lr, [sp, #232]
/* Use exception stack to restore SP0 */
msr SPSel, #1 // Switch to SPx
stp x0, x1, [sp, #-16]! // Save x0 and x1
mrs x0, SP_EL0 // Reload the pcb pointer from SP0
ldr x1, [x0, #248] // Get the saved SP from the pcb
msr SP_EL0, x1 // Restore SP0
ldp x0, x1, [sp], #16 // Restore x0 and x1
eret
arch/arm64/exceptions_el3.S
+315
View File
@@ -0,0 +1,315 @@
/*
* Copyright (C) 2011-2012, 2014 Apple Computer, Inc. All rights reserved.
*
* This document is the property of Apple Computer, Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Computer, Inc.
*/
#if !WITH_EL3
#error "This file should only be built for platforms WITH_EL3"
#endif
////////////////////////////////////////////////////////////
// This is what exception frame looks like:
//
// struct arm_exception_frame64 {
// uint64_t regs[29]; // x0-x28
// uint64_t fp;
// uint64_t lr;
// uint64_t sp;
// uint32_t spsr;
// uint32_t reserved0;
// uint64_t pc;
// uint64_t far;
// uint32_t esr;
// uint32_t reserved1;
// uint64_t reserved2; // stp requires multiple of 16 in imm field
// uint64_t reserved3; // stp requires multiple of 16 in imm field
// union {
// uint128_t q;
// uint64_t d;
// uint32_t s;
// } vregs[32]; // v0-v31
// uint32_t fpsr;
// uint32_t reserved4;
// uint32_t fpcr;
// uint32_t reserved5;
// };
////////////////////////////////////////////////////////////
.macro EL3_SP0_VECTOR
msr SPSel, #0 // Switch to SP0
sub sp, sp, #832 // Create exception frame
stp x0, x1, [sp, #0] // Save x0, x1 to exception frame
add x0, sp, #832 // Calculate the original stack pointer
str x0, [sp, #248] // Save stack pointer to exception frame
stp fp, lr, [sp, #232] // Save fp and lr to exception frame
mrs lr, ELR_EL3 // Get exception link reg so _dispatch64 can make a frame
mov x0, sp // Copy saved state pointer to x0
.endmacro
.macro FASTSIM_DEBUG_HINT
hint 0x45 // make fastsim drop to debugger (nop on hardware)
.endmacro
.text
.align 12
.globl _exception_vector_base
_exception_vector_base:
L__el3_sp0_synchronous_vector:
EL3_SP0_VECTOR
mrs x1, ESR_EL3 // Load exception syndrome
str x1, [x0, #280]
mrs x1, FAR_EL3 // Load fault address
str x1, [x0, #272]
adrp x1, _arm_synchronous_exception@page
add x1, x1, _arm_synchronous_exception@pageoff
b __dispatch64
.text
.align 7
L__el3_sp0_irq_vector:
EL3_SP0_VECTOR
adrp x1, _interrupt_stack_top@page
add x1, x1, _interrupt_stack_top@pageoff
ldr x1, [x1]
mov sp, x1
adrp x1, _arm_irq@page
add x1, x1, _arm_irq@pageoff
b __dispatch64
.text
.align 7
L__el3_sp0_fiq_vector:
EL3_SP0_VECTOR
adrp x1, _interrupt_stack_top@page
add x1, x1, _interrupt_stack_top@pageoff
ldr x1, [x1]
mov sp, x1
adrp x1, _arm_fiq@page
add x1, x1, _arm_fiq@pageoff
b __dispatch64
.text
.align 7
L__el3_sp0_serror_vector:
EL3_SP0_VECTOR
mrs x1, ESR_EL3 // Load exception syndrome
str x1, [x0, #280]
mrs x1, FAR_EL3 // Load fault address
str x1, [x0, #272]
adrp x1, _arm_serror_exception@page
add x1, x1, _arm_serror_exception@pageoff
b __dispatch64
.text
.align 7
L__el3_sp3_synchronous_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__el3_sp3_irq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__el3_sp3_fiq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__el3_sp3_serror_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_synchronous_vector:
// Monitor (EL3) call.
// Check "smc #0x5ec3" was executed and no other reason code.
movz x1, #0x5ec3
movk x1, #0x5e00, lsl #16
mrs x2, ESR_EL3
cmp x1, x2
b.eq 1f
FASTSIM_DEBUG_HINT
b .
1: // Got here from an "smc". Branch to x0.
br x0
.text
.align 7
L__elx_64_irq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_fiq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_64_serror_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_synchronous_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_irq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_fiq_vector:
FASTSIM_DEBUG_HINT
b .
.text
.align 7
L__elx_32_serror_vector:
FASTSIM_DEBUG_HINT
b .
/* Placeholder for macho_post_process.py. Placing the UUID here
allows debuggers to locate the UUID based on the value of VBAR */
.text
.align 7
_UUID:
.space 32
/* 64-bit first level exception handler dispatcher.
* Completes register context saving and branches to FLEH.
* Expects:
* {x0, x1, fp, lr, sp} - saved
* x0 - arm_context_t
* x1 - address of FLEH
* fp - previous stack frame if EL1
* lr - unused
* sp - kernel stack
*/
.text
.align 2
__dispatch64:
stp fp, lr, [sp, #-16]! // Create a frame to help debuggers unwind
mov fp, sp // past the exception handler
stp x2, x3, [x0, #16] // Save remaining GPRs
stp x4, x5, [x0, #32]
stp x6, x7, [x0, #48]
stp x8, x9, [x0, #64]
stp x10, x11, [x0, #80]
stp x12, x13, [x0, #96]
stp x14, x15, [x0, #112]
stp x16, x17, [x0, #128]
stp x18, x19, [x0, #144]
stp x20, x21, [x0, #160]
stp x22, x23, [x0, #176]
stp x24, x25, [x0, #192]
stp x26, x27, [x0, #208]
str x28, [x0, #224]
mrs lr, ELR_EL3 // Get exception link register
str lr, [x0, #264] // Save ELR to PCB
mrs x2, SPSR_EL3 // Get CPSR
str w2, [x0, #256] // Save CPSR to PCB
#if WITH_VFP
stp q0, q1, [x0, #304]
stp q2, q3, [x0, #336]
stp q4, q5, [x0, #368]
stp q6, q7, [x0, #400]
stp q8, q9, [x0, #432]
stp q10, q11, [x0, #464]
stp q12, q13, [x0, #496]
stp q14, q15, [x0, #528]
stp q16, q17, [x0, #560]
stp q18, q19, [x0, #592]
stp q20, q21, [x0, #624]
stp q22, q23, [x0, #656]
stp q24, q25, [x0, #688]
stp q26, q27, [x0, #720]
stp q28, q29, [x0, #752]
stp q30, q31, [x0, #784]
mrs x2, FPSR
str w2, [x0, #816]
mrs x2, FPCR
str w2, [x0, #820]
#endif
mov x21, x0 // Copy arm_context_t pointer to x21
blr x1
b __exception_return
.text
.align 2
__exception_return:
mov sp, x21 // Reload the arm_context_t pointer
#if WITH_VFP
ldp q0, q1, [sp, #304]
ldp q2, q3, [sp, #336]
ldp q4, q5, [sp, #368]
ldp q6, q7, [sp, #400]
ldp q8, q9, [sp, #432]
ldp q10, q11, [sp, #464]
ldp q12, q13, [sp, #496]
ldp q14, q15, [sp, #528]
ldp q16, q17, [sp, #560]
ldp q18, q19, [sp, #592]
ldp q20, q21, [sp, #624]
ldp q22, q23, [sp, #656]
ldp q24, q25, [sp, #688]
ldp q26, q27, [sp, #720]
ldp q28, q29, [sp, #752]
ldp q30, q31, [sp, #784]
ldr w3, [sp, #816]
msr FPSR, x3
ldr w4, [sp, #820]
msr FPCR, x4
#endif
ldr x0, [sp, #264] // Get the return address
msr ELR_EL3, x0 // Load the return address into ELR
ldr w1, [sp, #256] // Get the return CPSR
msr SPSR_EL3, x1 // Load the return CPSR into SPSR
ldp x0, x1, [sp, #0] // Restore the GPRs
ldp x2, x3, [sp, #16]
ldp x4, x5, [sp, #32]
ldp x6, x7, [sp, #48]
ldp x8, x9, [sp, #64]
ldp x10, x11, [sp, #80]
ldp x12, x13, [sp, #96]
ldp x14, x15, [sp, #112]
ldp x16, x17, [sp, #128]
ldp x18, x19, [sp, #144]
ldp x20, x21, [sp, #160]
ldp x22, x23, [sp, #176]
ldp x24, x25, [sp, #192]
ldp x26, x27, [sp, #208]
ldr x28, [sp, #224]
ldp fp, lr, [sp, #232]
/* Use exception stack to restore SP0 */
msr SPSel, #1 // Switch to SPx
stp x0, x1, [sp, #-16]! // Save x0 and x1
mrs x0, SP_EL0 // Reload the pcb pointer from SP0
ldr x1, [x0, #248] // Get the saved SP from the pcb
msr SP_EL0, x1 // Restore SP0
ldp x0, x1, [sp], #16 // Restore x0 and x1
eret
arch/arm64/fp.c
+40
View File
@@ -0,0 +1,40 @@
/*
* Copyright (C) 2012-2013 Apple Computer, Inc. All rights reserved.
*
* This document is the property of Apple Computer, Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Computer, Inc.
*/
#include <arch.h>
#include <arch/arm/arm.h>
#include <debug.h>
#include <sys/task.h>
/*
* VFP/Neon support
*/
#if WITH_VFP
extern uint32_t arm_read_cpacr();
extern void arm_write_cpacr(uint32_t);
void arm_fp_init(void)
{
uint32_t cpacr;
/* v8: CPACR FPEN bits [21:20] needs to be programmed correctly to avoid a trap */
cpacr = arm_read_cpacr();
cpacr |= (3 << 20);
arm_write_cpacr(cpacr);
}
bool arch_task_fp_enable(bool enable)
{
return true;
}
#endif /* WITH_VFP */
arch/arm64/handlers.c
+138
View File
@@ -0,0 +1,138 @@
/*
* Copyright (C) 2011-2012 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <debug.h>
#include <arch.h>
#include <arch/arch_task.h>
#include <arch/arm/arm.h>
#include <arch/arm64/proc_reg.h>
#include <sys.h>
#include <lib/libc.h>
#include <platform.h>
extern void platform_irq(void);
typedef enum {
ESR_EC_UNCATEGORIZED = 0x00,
ESR_EC_TRAP_SIMD_FP = 0x07,
ESR_EC_ILLEGAL_INSTR_SET = 0x0c,
ESR_EC_SVC_32 = 0x11,
ESR_EC_SVC_64 = 0x15,
ESR_EC_MSR_TRAP = 0x18,
ESR_EC_IABORT_EL0 = 0x20,
ESR_EC_IABORT_EL1 = 0x21,
ESR_EC_PC_ALIGN = 0x22,
ESR_EC_DABORT_EL0 = 0x24,
ESR_EC_DABORT_EL1 = 0x25,
ESR_EC_SP_ALIGN = 0x26
} esr_exception_class_t;
typedef enum {
FSC_TRANSLATION_FAULT_L0 = 0x04,
FSC_TRANSLATION_FAULT_L1 = 0x05,
FSC_TRANSLATION_FAULT_L2 = 0x06,
FSC_TRANSLATION_FAULT_L3 = 0x07,
FSC_ACCESS_FLAG_FAULT_L1 = 0x09,
FSC_ACCESS_FLAG_FAULT_L2 = 0x0A,
FSC_ACCESS_FLAG_FAULT_L3 = 0x0B,
FSC_PERMISSION_FAULT_L1 = 0x0D,
FSC_PERMISSION_FAULT_L2 = 0x0E,
FSC_PERMISSION_FAULT_L3 = 0x0F,
FSC_SYNC_EXT_ABORT = 0x10,
FSC_ASYNC_EXT_ABORT = 0x11,
FSC_SYNC_EXT_ABORT_TT_L1 = 0x15,
FSC_SYNC_EXT_ABORT_TT_L2 = 0x16,
FSC_SYNC_EXT_ABORT_TT_L3 = 0x17,
FSC_SYNC_PARITY = 0x18,
FSC_ASYNC_PARITY = 0x19,
FSC_SYNC_PARITY_TT_L1 = 0x1D,
FSC_SYNC_PARITY_TT_L2 = 0x1E,
FSC_SYNC_PARITY_TT_L3 = 0x1F,
FSC_ALIGNMENT_FAULT = 0x21,
FSC_DEBUG_FAULT = 0x22
} fault_status_t;
static const char *exception_codes[50] = {
[0x00] = "Unknown",
[0x0E] = "Illegal Instruction Set",
[0x21] = "Instruction Abort",
[0x22] = "PC Alignment",
[0x25] = "Data Abort",
[0x26] = "Stack Pointer Alignment",
[0x2F] = "SError",
};
void
arm_synchronous_exception(struct arm_exception_frame64 *frame)
{
const char **kind;
kind = &exception_codes[ESR_EC(frame->esr)];
panic("ARM synchronous abort at 0x%016llx:\n"
" esr 0x%08x (ec: %s, iss: 0x%08x) far 0x%016llx\n"
" x0 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x4 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x8 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x12 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x16 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x20 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x24 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x28 0x%016llx sp 0x%016llx lr 0x%016llx spsr 0x%08x\n",
frame->pc,
frame->esr, *kind, ESR_ISS(frame->esr), frame->far,
frame->regs[0], frame->regs[1], frame->regs[2], frame->regs[3],
frame->regs[4], frame->regs[5], frame->regs[6], frame->regs[7],
frame->regs[8], frame->regs[9], frame->regs[10], frame->regs[11],
frame->regs[12], frame->regs[13], frame->regs[14], frame->regs[15],
frame->regs[16], frame->regs[17], frame->regs[18], frame->regs[19],
frame->regs[20], frame->regs[21], frame->regs[22], frame->regs[23],
frame->regs[24], frame->regs[25], frame->regs[26], frame->regs[27],
frame->regs[28], frame->sp, frame->lr, frame->spsr);
}
void
arm_serror_exception(struct arm_exception_frame64 *frame)
{
const char **kind;
kind = &exception_codes[ESR_EC(frame->esr)];
platform_asynchronous_exception();
panic("ARM SError abort at 0x%016llx:\n"
" esr 0x%08x (ec: %s, iss: 0x%08x) far 0x%016llx\n"
" x0 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x4 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x8 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x12 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x16 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x20 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x24 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
" x28 0x%016llx sp 0x%016llx lr 0x%016llx spsr 0x%08x\n",
frame->pc,
frame->esr, *kind, ESR_ISS(frame->esr), frame->far,
frame->regs[0], frame->regs[1], frame->regs[2], frame->regs[3],
frame->regs[4], frame->regs[5], frame->regs[6], frame->regs[7],
frame->regs[8], frame->regs[9], frame->regs[10], frame->regs[11],
frame->regs[12], frame->regs[13], frame->regs[14], frame->regs[15],
frame->regs[16], frame->regs[17], frame->regs[18], frame->regs[19],
frame->regs[20], frame->regs[21], frame->regs[22], frame->regs[23],
frame->regs[24], frame->regs[25], frame->regs[26], frame->regs[27],
frame->regs[28], frame->sp, frame->lr, frame->spsr);
}
void
arm_irq()
{
_irq_enter_critical_section();
platform_irq();
_irq_exit_critical_section();
}
arch/arm64/include/arch/arm64/proc_reg.h
+599
View File
@@ -0,0 +1,599 @@
/*
* Copyright (c) 2007-2011, 2014 Apple Inc. All rights reserved.
*/
/*
* Processor registers for ARM64
*/
#ifndef _ARM64_PROC_REG_H_
#define _ARM64_PROC_REG_H_
// Pick the right registers depending on the exception level we're booting into.
#if WITH_EL3
#define SCTLR_ELx SCTLR_EL3
#define MAIR_ELx MAIR_EL3
#define TCR_ELx TCR_EL3
#define TTBR0_ELx TTBR0_EL3
#define VBAR_ELx VBAR_EL3
#else
#define SCTLR_ELx SCTLR_EL1
#define MAIR_ELx MAIR_EL1
#define TCR_ELx TCR_EL1
#define TTBR0_ELx TTBR0_EL1
#define VBAR_ELx VBAR_EL1
#endif
/*
* 64-bit Program Status Register (PSR64)
*
* 31 27 22 21 20 19 10 9 5 4 0
* +-+-+-+-+------+--+--+----------+-+-+-+-+-+-----+
* |N|Z|C|V|000000|SS|IL|0000000000|D|A|I|F|0| M |
* +-+-+-+-+-+----+--+--+----------+-+-+-+-+-+-----+
*
* where:
* NZCV Comparison flags
* SS Single step
* IL Illegal state
* DAIF Interrupt masks
* M Mode field
*/
#define PSR64_NZCV_SHIFT 28
#define PSR64_NZCV_MASK (1 << PSR64_NZCV_SHIFT)
#define PSR64_N_SHIFT 31
#define PSR64_N (1 << PSR64_N_SHIFT)
#define PSR64_Z_SHIFT 30
#define PSR64_Z (1 << PSR64_Z_SHIFT)
#define PSR64_C_SHIFT 29
#define PSR64_C (1 << PSR64_C_SHIFT)
#define PSR64_V_SHIFT 28
#define PSR64_V (1 << PSR64_V_SHIFT)
#define PSR64_SS_SHIFT 21
#define PSR64_SS (1 << PSR64_SS_SHIFT)
#define PSR64_IL_SHIFT 20
#define PSR64_IL (1 << PSR64_IL_SHIFT)
/*
* msr DAIF, Xn and mrs Xn, DAIF transfer into
* and out of bits 9:6
*/
#define DAIF_DEBUG_SHIFT 9
#define DAIF_DEBUGF (1 << DAIF_DEBUG_SHIFT)
#define DAIF_ASYNC_SHIFT 8
#define DAIF_ASYNCF (1 << DAIF_ASYNC_SHIFT)
#define DAIF_IRQF_SHIFT 7
#define DAIF_IRQF (1 << DAIF_IRQF_SHIFT)
#define DAIF_FIQF_SHIFT 6
#define DAIF_FIQF (1 << DAIF_FIQF_SHIFT)
#define DAIF_ALL (DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
#define DAIF_STANDARD_DISABLE (DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
#define SPSR_INTERRUPTS_ENABLED(x) (!(x & DAIF_FIQF))
/*
* msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
* from bits 3:0.
*/
#define DAIFSC_DEBUGF (1 << 3)
#define DAIFSC_ASYNCF (1 << 2)
#define DAIFSC_IRQF (1 << 1)
#define DAIFSC_FIQF (1 << 0)
#define DAIFSC_ALL (DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
#define DAIFSC_STANDARD_DISABLE (DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
/*
* ARM64_TODO: unify with ARM?
*/
#define PSR64_CF 0x20000000 /* Carry/Borrow/Extend */
#define PSR64_MODE_MASK 0x1F
#define PSR64_MODE_USER32_THUMB 0x20
#define PSR64_MODE_RW_SHIFT 4
#define PSR64_MODE_RW_64 0
#define PSR64_MODE_RW_32 (0x1 << PSR64_MODE_RW_SHIFT)
#define PSR64_MODE_EL_SHIFT 2
#define PSR64_MODE_EL_MASK (0x3 << PSR64_MODE_EL_SHIFT)
#define PSR64_MODE_EL3 (0x3 << PSR64_MODE_EL_SHIFT)
#define PSR64_MODE_EL1 (0x1 << PSR64_MODE_EL_SHIFT)
#define PSR64_MODE_EL0 0
#define PSR64_MODE_SPX 0x1
#define PSR64_MODE_SP0 0
#define PSR64_USER32_DEFAULT (PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
#define PSR64_USER64_DEFAULT (PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
#define PSR64_KERNEL_DEFAULT (DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
#define PSR64_IS_KERNEL(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL1)
#define PSR64_IS_USER(x) ((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
/*
* Floating Point Control Register (FPCR)
*/
#define FPCR_DN (1 << 25)
#define FPCR_DEFAULT (FPCR_DN)
/*
* System Control Register (SCTLR)
*
* 31 26 25 24 23 19 18 15 14 13 12 11 7 6 5 4 3 2 1 0
* +-----+---+--+---+----+---+---+---+---+--+--+----+---+----+------+---+--+-+-+-+
* |00110|UCI|EE|E0E|1101|WXN|101|UCT|DZE| 0|I |1000|ITD|THEE|C15BEN|SA0|SA|D|A|M|
* +-----+---+--+---+----+---+---+---+---+--+--+----+---+----+------+---+--+-+-+-+
*
* UCI User Cache Instructions
* EE Exception Endianness
* E0E EL0 Endianness
* WXN Writeable implies eXecute Never
* UCT User Cache Type register (CTR_EL0)
* DZE user Data Cache Zero (DC ZVA)
* I Instruction cache enable
* ITD IT Disable
* THEE THumb EE enable
* CP15BEN CP15 Barrier ENable
* SA0 Stack Alignment check for EL0
* SA Stack Alignment check enable
* D Data/unified cache enable
* A Alignment check enable
* M MMU enable
*/
#define SCTLR_RESERVED 0x30d50800
#define SCTLR_UCI_ENABLED (1 << 26)
#define SCTLR_EE_ENABLED (1 << 25)
#define SCTLR_E0E_ENABLED (1 << 24)
#define SCTLR_WXN_ENABLED (1 << 19)
#define SCTLR_UCT_ENABLED (1 << 15)
#define SCTLR_DZE_ENABLED (1 << 14)
#define SCTLR_I_ENABLED (1 << 12)
#define SCTLR_ITD_DISABLED (1 << 7)
#define SCTLR_THEE_ENABLED (1 << 6)
#define SCTLR_CP15BEN_ENABLED (1 << 5)
#define SCTLR_SA0_ENABLED (1 << 4)
#define SCTLR_SA_ENABLED (1 << 3)
#define SCTLR_D_ENABLED (1 << 2)
#define SCTLR_A_ENABLED (1 << 1)
#define SCTLR_M_ENABLED (1 << 0)
#define SCTLR_EL1_DEFAULT (SCTLR_RESERVED | SCTLR_I_ENABLED | SCTLR_SA_ENABLED | SCTLR_D_ENABLED | SCTLR_M_ENABLED)
/*
* Secure Configuratio Register (SCR)
*
* 31 12 0
* +---+---+------+------+--------------------+
* |00000000000000000000 ||
* +---+---+------+------+--------------------+
*
* where:
* TTA Trace trap
* FPEN Floating point enable
*/
#define SCR_EA_SHIFT (3)
#define SCR_EA (1 << SCR_EA_SHIFT)
#define SCR_FIQ_SHIFT (2)
#define SCR_FIQ (1 << SCR_FIQ_SHIFT)
#define SCR_IRQ_SHIFT (1)
#define SCR_IRQ (1 << SCR_IRQ_SHIFT)
#define SCR_DEFAULT (SCR_EA | SCR_FIQ | SCR_IRQ)
/*
* Coprocessor Access Control Register (CPACR)
*
* 31 28 27 22 21 20 19 0
* +---+---+------+------+--------------------+
* |000|TTA|000000| FPEN |00000000000000000000|
* +---+---+------+------+--------------------+
*
* where:
* TTA Trace trap
* FPEN Floating point enable
*/
#define CPACR_TTA_SHIFT 28
#define CPACR_TTA (1 << CPACR_TTA_SHIFT)
#define CPACR_FPEN_SHIFT 20
#define CPACR_FPEN_EL0_TRAP (0x1 << CPACR_FPEN_SHIFT)
#define CPACR_FPEN_ENABLE (0x3 << CPACR_FPEN_SHIFT)
/*
* Translation Control Register (TCR) - EL3
*
* 31 30 24 23 22 21 20 19 18 16 14 13 12 11 10 9 8 7 6 5 0
* +-+-------+--+-----+---+--+-----+---+-----+-----+-----+-+-+----+
* |1| 0 | 1| 0 |TBI| 0| PS |TG0| SH0 |ORGN0|IRGN0|0|0|T0SZ|
* +-+-------+--+-----+---+--+-----+---+-----+-----+-----+-+-+----+
*
* TBI Top Byte Ignored for TTBR0 region
* PS Physical Address Size limit
* TG0 Granule Size for TTBR0 region
* SH0 Shareability for TTBR0 region
* ORGN0 Outer Cacheability for TTBR0 region
* IRGN0 Inner Cacheability for TTBR0 region
* T0SZ Virtual address size for TTBR0
* Translation Control Register (TCR) - EL1
*
* 63 39 38 37 36 34 32 30 29 28 27 26 25 24 23 22 21 16 14 13 12 11 10 9 8 7 5 0
* +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
* | 0 |TBI1|TBI0|AS|0| IPS |0|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |0|TG0| SH0 |ORGN0|IRGN0|EPD0|0|T0SZ|
* +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
*
* TBI1 Top Byte Ignored for TTBR1 region
* TBI0 Top Byte Ignored for TTBR0 region
* AS ASID Size
* IPS Physical Address Size limit
* TG1 Granule Size for TTBR1 region
* SH1 Shareability for TTBR1 region
* ORGN1 Outer Cacheability for TTBR1 region
* IRGN1 Inner Cacheability for TTBR1 region
* EPD1 Translation table walk disable for TTBR1
* A1 ASID selection from TTBR1 enable
* T1SZ Virtual address size for TTBR1
* TG0 Granule Size for TTBR0 region
* SH0 Shareability for TTBR0 region
* ORGN0 Outer Cacheability for TTBR0 region
* IRGN0 Inner Cacheability for TTBR0 region
* T0SZ Virtual address size for TTBR0
*/
// Common defintions for EL1 and EL3 versions of this register
#define TCR_ELx_T0SZ_SHIFT 0
#define TCR_ELx_IRGN0_SHIFT 8
#define TCR_ELx_IRGN0_DISABLED (0ULL << TCR_ELx_IRGN0_SHIFT)
#define TCR_ELx_IRGN0_WRITEBACK (1ULL << TCR_ELx_IRGN0_SHIFT)
#define TCR_ELx_IRGN0_WRITETHRU (2ULL << TCR_ELx_IRGN0_SHIFT)
#define TCR_ELx_IRGN0_WRITEBACKNO (3ULL << TCR_ELx_IRGN0_SHIFT)
#define TCR_ELx_ORGN0_SHIFT 10
#define TCR_ELx_ORGN0_DISABLED (0ULL << TCR_ELx_ORGN0_SHIFT)
#define TCR_ELx_ORGN0_WRITEBACK (1ULL << TCR_ELx_ORGN0_SHIFT)
#define TCR_ELx_ORGN0_WRITETHRU (2ULL << TCR_ELx_ORGN0_SHIFT)
#define TCR_ELx_ORGN0_WRITEBACKNO (3ULL << TCR_ELx_ORGN0_SHIFT)
#define TCR_ELx_SH0_SHIFT 12
#define TCR_ELx_SH0_NONE (0ULL << TCR_ELx_SH0_SHIFT)
#define TCR_ELx_SH0_OUTER (2ULL << TCR_ELx_SH0_SHIFT)
#define TCR_ELx_SH0_INNER (3ULL << TCR_ELx_SH0_SHIFT)
#define TCR_ELx_TG0_GRANULE_SHIFT 14
#define TCR_ELx_TG0_GRANULE_4KB (0ULL << TCR_ELx_TG0_GRANULE_SHIFT)
#define TCR_ELx_TG0_GRANULE_64KB (1ULL << TCR_ELx_TG0_GRANULE_SHIFT)
#define TCR_ELx_TG0_GRANULE_16KB (2ULL << TCR_ELx_TG0_GRANULE_SHIFT)
#if WITH_EL3
// EL3-specific TCR register definitions
#define TCR_EL3_PS_SHIFT 16
#define TCR_EL3_PS_32BITS (0ULL << TCR_EL3_PS_SHIFT)
#define TCR_EL3_PS_36BITS (1ULL << TCR_EL3_PS_SHIFT)
#define TCR_EL3_PS_40BITS (2ULL << TCR_EL3_PS_SHIFT)
#define TCR_EL3_PS_42BITS (3ULL << TCR_EL3_PS_SHIFT)
#define TCR_EL3_PS_44BITS (4ULL << TCR_EL3_PS_SHIFT)
#define TCR_EL3_PS_48BITS (5ULL << TCR_EL3_PS_SHIFT)
#define TCR_EL3_TBI_TOPBYTE_IGNORED (1ULL << 20)
#else
// EL1-specific TCR register definitions
#define TCR_EL1_EPD0_TTBR0_DISABLED (1ULL << 7)
#define TCR_EL1_T1SZ_SHIFT 16
#define TCR_EL1_A1_ASID1 (1ULL << 22)
#define TCR_EL1_EPD1_TTBR1_DISABLED (1ULL << 23)
#define TCR_EL1_IRGN1_SHIFT 24
#define TCR_EL1_IRGN1_DISABLED (0ULL << TCR_EL1_IRGN1_SHIFT)
#define TCR_EL1_IRGN1_WRITEBACK (1ULL << TCR_EL1_IRGN1_SHIFT)
#define TCR_EL1_IRGN1_WRITETHRU (2ULL << TCR_EL1_IRGN1_SHIFT)
#define TCR_EL1_IRGN1_WRITEBACKNO (3ULL << TCR_EL1_IRGN1_SHIFT)
#define TCR_EL1_ORGN1_SHIFT 26
#define TCR_EL1_ORGN1_DISABLED (0ULL << TCR_EL1_ORGN1_SHIFT)
#define TCR_EL1_ORGN1_WRITEBACK (1ULL << TCR_EL1_ORGN1_SHIFT)
#define TCR_EL1_ORGN1_WRITETHRU (2ULL << TCR_EL1_ORGN1_SHIFT)
#define TCR_EL1_ORGN1_WRITEBACKNO (3ULL << TCR_EL1_ORGN1_SHIFT)
#define TCR_EL1_SH1_SHIFT 28
#define TCR_EL1_SH1_NONE (0ULL << TCR_EL1_SH1_SHIFT)
#define TCR_EL1_SH1_OUTER (2ULL << TCR_EL1_SH1_SHIFT)
#define TCR_EL1_SH1_INNER (3ULL << TCR_EL1_SH1_SHIFT)
#define TCR_EL1_TG1_GRANULE_SHIFT 30
#define TCR_EL1_TG1_GRANULE_16KB (1ULL << TCR_EL1_TG1_GRANULE_SHIFT)
#define TCR_EL1_TG1_GRANULE_4KB (2ULL << TCR_EL1_TG1_GRANULE_SHIFT)
#define TCR_EL1_TG1_GRANULE_64KB (3ULL << TCR_EL1_TG1_GRANULE_SHIFT)
#define TCR_EL1_IPS_SHIFT 32
#define TCR_EL1_IPS_32BITS (0ULL << TCR_EL1_IPS_SHIFT)
#define TCR_EL1_IPS_36BITS (1ULL << TCR_EL1_IPS_SHIFT)
#define TCR_EL1_IPS_40BITS (2ULL << TCR_EL1_IPS_SHIFT)
#define TCR_EL1_IPS_42BITS (3ULL << TCR_EL1_IPS_SHIFT)
#define TCR_EL1_IPS_44BITS (4ULL << TCR_EL1_IPS_SHIFT)
#define TCR_EL1_IPS_48BITS (5ULL << TCR_EL1_IPS_SHIFT)
#define TCR_EL1_AS_16BIT_ASID (1ULL << 36)
#define TCR_EL1_TBI0_TOPBYTE_IGNORED (1ULL << 37)
#define TCR_EL1_TBI1_TOPBYTE_IGNORED (1ULL << 38)
#endif
/*
* Translation Table Base Register (TTBR)
*
* 63 48 47 x x-1 0
* +--------+------------------+------+
* | ASID | Base Address | zero |
* +--------+------------------+------+
*
*/
#define TTBR_ASID_SHIFT 48
#define TTBR_ASID_MASK 0xffff000000000000
#define TTBR_BADDR_MASK 0x0000ffffffffffff
/*
* Memory Attribute Indirection Register
*
* 63 56 55 48 47 40 39 32 31 24 23 16 15 8 7 0
* +-------+-------+-------+-------+-------+-------+-------+-------+
* | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
* +-------+-------+-------+-------+-------+-------+-------+-------+
*
*/
#define MAIR_ATTR_SHIFT(x) (8*(x))
/* Strongly ordered or device memory attributes */
#define MAIR_OUTER_STRONGLY_ORDERED 0x0
#define MAIR_OUTER_DEVICE 0x0
#define MAIR_INNER_STRONGLY_ORDERED 0x0
#define MAIR_INNER_DEVICE 0x4
/* Normal memory attributes */
#define MAIR_OUTER_NON_CACHEABLE 0x40
#define MAIR_OUTER_WRITE_THROUGH 0x80
#define MAIR_OUTER_WRITE_BACK 0xc0
#define MAIR_INNER_NON_CACHEABLE 0x4
#define MAIR_INNER_WRITE_THROUGH 0x8
#define MAIR_INNER_WRITE_BACK 0xc
/* Allocate policy for cacheable memory */
#define MAIR_OUTER_WRITE_ALLOCATE 0x10
#define MAIR_OUTER_READ_ALLOCATE 0x20
#define MAIR_INNER_WRITE_ALLOCATE 0x1
#define MAIR_INNER_READ_ALLOCATE 0x2
/* Memory Atribute Encoding */
#define MAIR_DISABLE 0x04 /* Strongly Ordered */
#define MAIR_WRITECOMB 0x44 /* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
#define MAIR_WRITETHRU 0xBB /* Normal Memory, Outer Write-through, Inner Write-through */
#define MAIR_WRITEBACK 0xFF /* Normal Memory, Outer Write-back, Inner Write-back */
#define MAIR_INNERWRITEBACK 0x4F /* Normal Memory, Outer Non-Cacheable, Inner Write-back */
/*
* ARM 4-level Page Table support - 2*1024TB (2^48) of address space
*/
/*
* Memory Attribute Index
*/
#define CACHE_ATTRINDX_DISABLE 0x0 /* no cache, no buffer */
#define CACHE_ATTRINDX_WRITEBACK 0x1 /* cache enabled, buffer enabled */
#define CACHE_ATTRINDX_WRITECOMB 0x1 /* no cache, buffered writes */
#define CACHE_ATTRINDX_WRITETHRU 0x2 /* cache enabled, buffer disabled */
#define CACHE_ATTRINDX_INNERWRITEBACK 0x4 /* inner cache enabled, buffer enabled, write allocate */
#define CACHE_ATTRINDX_DEFAULT CACHE_ATTRINDX_WRITEBACK
/*
* Access protection bit values (TTEs and PTEs)
*/
#define AP_RWNA 0x0 /* priv=read-write, user=no-access */
#define AP_RWRW 0x1 /* priv=read-write, user=read-write */
#define AP_RONA 0x2 /* priv=read-only, user=no-access */
#define AP_RORO 0x3 /* priv=read-only, user=read-only */
#define AP_MASK 0x3 /* mask to find ap bits */
/*
* Shareability attributes
*/
#define SH_NONE 0x0 /* Non shareable */
#define SH_OUTER_MEMORY 0x2 /* Normal memory Inner shareable - Outer shareable */
#define SH_INNER_MEMORY 0x3 /* Normal memory Inner shareable - Outer non shareable */
/*
* LPAE/ARMv8 Table Entry
*
* 63 62 61 60 59 58 52 51 48 47 12 11 2 1 0
* ,--+-----+--+---+-------+------+----------------------+-------+-+-.
* |NS| AP |XN|PXN|ignored| zero | TableOutputAddress |ignored|1|V|
* `--+-----+--+---+-------+------+----------------------+-------+-+-'
*
* LPAE/ARMv8 Block Entry
*
* 63 59 58 55 54 53 52 51 48 47 12 11 10 9 8 7 6 5 4 2 1 0
* ,-----+-----+--+---+----+-----+-------------+--+--+----+----+--+-------+-+-.
* | ign | SW |XN|PXN|HINT| zero|OutputAddress|nG|AF| SH | AP |NS|AttrIdx|W|V|
* `-----+-----+--+---+----+-----+-------------+--+--+----+----+--+-------+-+-'
*
* NS (NSTable) Non-Secure: NS=1 for everything outside of TZ region
* AP (APTable) Access Permission: AP[2:1]=0 for RW, 2 for RO
* XN (XNTable) eXecute Never: XN=0 for code, 1 for data
* PXN (PXNTable) Privileged eXecute Never: 1=dont execute when EL3
* TableOutputAddress Next physical base: Lower bits 0 as alignment requires.
* SW Reserved for software use (unused, can have metadata)
* HINT Contiguous hint. Optimization only, leave as 0.
* OutputAddress Block physical base: Lower bits 0 as alignment requires.
* nG notGlobal: nG=1 restricts TLB hit to matching ASID
* AF Access Flag: 0=never,1=hit. ARMv8 may fault if 0.
* SH Shareability, SH[1:0]=2 for normal memory
* AttrIdx Memory Attribute Index, 0 for device, 1 for normal
* W "Walk" flag: L0-2 0=Block,1=Table; L3 0=Fault,1=Block
*/
#define ARM_LPAE_TABLE_MASK (0x0000fffffffff000ULL) /* mask for extracting pointer to next table (works at any level) */
#define ARM_LPAE_NSTABLE 0x8000000000000000ULL /* value for a secure mapping */
#define ARM_LPAE_NSTABLE_MASK 0x8000000000000000ULL /* notSecure mapping mask */
#define ARM_LPAE_APTABLE_SHIFT 61
#define ARM_LPAE_APTABLE(x) ((x)<<ARM_LPAE_APTABLE_SHIFT) /* access protection */
#define ARM_LPAE_XNTABLE 0x1000000000000000ULL /* value for no execute */
#define ARM_LPAE_XNTABLE_MASK 0x1000000000000000ULL /* no execute mask */
#define ARM_LPAE_PXNTABLE 0x0800000000000000ULL /* value for privilege no execute bit */
#define ARM_LPAE_PXNTABLE_MASK 0x0800000000000000ULL /* privilege execute mask */
#define ARM_LPAE_XN 0x0040000000000000ULL /* value for no execute */
#define ARM_LPAE_XN_MASK 0x0040000000000000ULL /* no execute mask */
#define ARM_LPAE_PXN 0x0020000000000000ULL /* value for privilege no execute bit */
#define ARM_LPAE_PXN_MASK 0x0020000000000000ULL /* privilege execute mask */
#define ARM_LPAE_NG 0x0000000000000800ULL /* value for a global mapping */
#define ARM_LPAE_NG_MASK 0x0000000000000800ULL /* notGlobal mapping mask */
#define ARM_LPAE_AF 0x0000000000000400ULL /* value for access */
#define ARM_LPAE_AF_MASK 0x0000000000000400ULL /* access mask */
#define ARM_LPAE_SH_SHIFT 8
#define ARM_LPAE_SH(x) ((x) << ARM_LPAE_SH_SHIFT) /* access shared */
#define ARM_LPAE_SH_MASK (0x3ULL << ARM_LPAE_SH_SHIFT) /* mask access shared */
#define ARM_LPAE_AP_SHIFT 6
#define ARM_LPAE_AP(x) ((x)<<ARM_LPAE_AP_SHIFT) /* access protection */
#define ARM_LPAE_AP_MASK (0x3ULL<<ARM_LPAE_AP_SHIFT)
#define ARM_LPAE_NS 0x0000000000000020ULL /* value for a secure mapping */
#define ARM_LPAE_MASK 0x0000000000000020ULL /* notSecure mapping mask */
#define ARM_LPAE_ATTRINDX_SHIFT 2 /* memory attributes index */
#define ARM_LPAE_ATTRINDX(x) ((x) << ARM_LPAE_ATTRINDX_SHIFT) /* memory attributes index */
#define ARM_LPAE_ATTRINDX_MASK (0x3ULL << ARM_LPAE_ATTRINDX_SHIFT) /* mask memory attributes index */
#define ARM_LPAE_TYPE_MASK 0x0000000000000002ULL /* mask for extracting the type */
#define ARM_LPAE_TYPE_TABLE 0x0000000000000002ULL /* page table type */
#define ARM_LPAE_TYPE_L0L1L2BLOCK 0x0000000000000000ULL /* block entry type */
#define ARM_LPAE_TYPE_L3BLOCK 0x0000000000000002ULL /* L3 is different */
#define ARM_LPAE_VALID 0x0000000000000001ULL /* valid entry */
/*
* Exception Syndrome Register
*
* 31 26 25 24 0
* +------+--+------------------+
* | EC |IL| ISS |
* +------+--+------------------+
*
* EC - Exception Class
* IL - Instruction Length
* ISS- Instruction Specific Syndrome
*
* Note: The ISS can have many forms. These are defined separately below.
*/
#define ESR_EC_SHIFT 26
#define ESR_EC_MASK (0x3F << ESR_EC_SHIFT)
#define ESR_EC(x) ((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
#define ESR_IL_SHIFT 25
#define ESR_IL (1 < ESR_IL_SHIFT)
#define ESR_IS_THUMB(x) (!(x & ESR_IL))
#define ESR_ISS_MASK 0x01FFFFFF
#define ESR_ISS(x) (x & ESR_ISS_MASK)
/*
* Instruction Abort ISS (EL1)
* 24 10 9 5 0
* +---------------+--+---+------+
* |000000000000000|EA|000| IFSC |
* +---------------+--+---+------+
*
* where:
* EA External Abort type
* IFSC Instruction Fault Status Code
*/
#define ISS_IA_EA_SHIFT 9
#define ISS_IA_EA (0x1 << ISS_IA_EA_SHIFT)
#define ISS_IA_FSC_MASK 0x2F
#define ISS_IA_FSC(x) (x & ISS_DA_FSC_MASK)
/*
* Data Abort ISS (EL1)
*
* 24 9 8 7 6 5 0
* +---------------+--+--+-+---+----+
* |000000000000000|EA|CM|0|WnR|DFSC|
* +---------------+--+--+-+---+----+
*
* where:
* EA External Abort type
* CM Cache Maintenance operation
* WnR Write not Read
* DFSC Data Fault Status Code
*/
#define ISS_DA_EA_SHIFT 9
#define ISS_DA_EA (0x1 << ISS_DA_EA_SHIFT)
#define ISS_DA_CM_SHIFT 8
#define ISS_DA_CM (0x1 << ISS_DA_CM_SHIFT)
#define ISS_DA_WNR_SHIFT 6
#define ISS_DA_WNR (0x1 << ISS_DA_WNR_SHIFT)
#define ISS_DA_FSC_MASK 0x2F
#define ISS_DA_FSC(x) (x & ISS_DA_FSC_MASK)
#define PLATFORM_SYSCALL_TRAP_NO 0x80000000
#define ARM64_SYSCALL_CODE_REG_NUM (16)
#define ARM64_CLINE_SHIFT 6
/*
* Timer definitions.
*/
#define CNTKCTL_EL1_PL0PTEN (0x1 << 9) /* 1: EL0 access to physical timer regs permitted */
#define CNTKCTL_EL1_PL0VTEN (0x1 << 8) /* 1: EL0 access to virtual timer regs permitted */
#define CNTKCTL_EL1_EVENTI_MASK (0x000000f0) /* Mask for bits describing which bit to use for triggering event stream */
#define CNTKCTL_EL1_EVENTI_SHIFT (0x4) /* Shift for same */
#define CNTKCTL_EL1_EVENTDIR (0x1 << 3) /* 1: one-to-zero transition of specified bit causes event */
#define CNTKCTL_EL1_EVNTEN (0x1 << 2) /* 1: enable event stream */
#define CNTKCTL_EL1_PL0VCTEN (0x1 << 1) /* 1: EL0 access to physical timebase + frequency reg enabled */
#define CNTKCTL_EL1_PL0PCTEN (0x1 << 0) /* 1: EL0 access to virtual timebase + frequency reg enabled */
#endif /* _ARM64_PROC_REG_H_ */
arch/arm64/link.exp
+1
View File
@@ -0,0 +1 @@
_start
arch/arm64/mmu.c
+543
View File
@@ -0,0 +1,543 @@
/*
* Copyright (C) 2012-2014 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
/*
* See docs/mmu64.txt for an in-depth version of what is going on here.
* Short version:
* - H6, H7 use 4KB page granule.
* - H8 uses 16KB page granule (4KB not supported by hardware).
* - Start at L1 for 4KB page granule.
* - Start at L2 for 16KB page granule.
* - Set virtual memory limit so the first level (L0/L1) has just two entries.
* - ...then never touch the top level so we don't suffer from NS/S coherency.
*/
#include <debug.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <arch/arm/arm.h>
#include <arch/arm64/proc_reg.h>
#ifndef TEST
# include <sys.h>
# include <arch.h>
# include <platform.h>
# include <platform/memmap.h>
#endif
#define LAST_TABLE_LEVEL 3
extern uint64_t arm_read_sctlr();
// Start level page table entries
static uint64_t *tt_top_level;
// Allocation area and current allocation pointer.
static addr_t tt_alloc_start, tt_alloc_next, tt_alloc_end;
#ifdef TEST
// Mock test interfaces for mmu.c, implemented in mmu_test.c
void mmu_printf(int verbosity, const char *fmt, ...);
unsigned get_page_granule_shift(void);
void mmu_get_memmap(addr_t *page_tables_base,
size_t *page_tables_size);
void platform_mmu_setup(bool resume);
void get_table_entry_hook(unsigned level, addr_t table_base, size_t index);
void set_table_entry_hook(unsigned level, addr_t table_base, size_t index,
uint64_t value);
#else // !defined(TEST)
// Real interfaces, designed to be inlined. These have no host-side coverage,
// so they should be no-ops or simple constants.
// No printf spew.
static void mmu_printf(int verbosity, const char *fmt, ...)
{
(void) verbosity;
(void) fmt;
}
// Constants.
static unsigned get_page_granule_shift(void) { return PAGE_GRANULE_SHIFT; }
static void mmu_get_memmap(addr_t *page_tables_base,
size_t *page_tables_size)
{
*page_tables_base = PAGE_TABLES_BASE;
*page_tables_size = PAGE_TABLES_SIZE;
}
// Remove expensive validity checking.
static void get_table_entry_hook(unsigned level, addr_t table_base,
size_t index)
{
(void) level; (void) table_base; (void) index;
}
static void set_table_entry_hook(unsigned level, addr_t table_base,
size_t index, uint64_t value)
{
(void) level; (void) table_base; (void) index; (void) value;
}
#endif // !defined(TEST)
#define mmu_spew(x...) mmu_printf(1, x)
#define mmu_extra_spew(x...) mmu_printf(2, x)
static void assert_valid_level(unsigned level)
{
ASSERT((get_page_granule_shift() == 12 || level >= 1) &&
level <= LAST_TABLE_LEVEL);
}
static bool level_permits_blocks(unsigned level)
{
assert_valid_level(level);
// No granule supports L0 blocks.
// 4KB granule supports L1 blocks.
// All granules support L2 and L3 blocks.
return (level == 1 && get_page_granule_shift() == 12) || level >= 2;
}
static uint64_t get_table_entry(unsigned level, addr_t table_base,
size_t index)
{
get_table_entry_hook(level, table_base, index);
mmu_extra_spew("get_table_entry(%u, 0x%llx, %u) ",
level, (unsigned long long) table_base,
(unsigned) index);
// No possible aliases, known aligned.
uint64_t result = *(uint64_t *) (table_base + index * 8);
mmu_extra_spew("= 0x%016llx\n", (unsigned long long) result);
return result;
}
static void set_table_entry(unsigned level, addr_t table_base, size_t index,
uint64_t value)
{
set_table_entry_hook(level, table_base, index, value);
mmu_extra_spew("set_table_entry(%u, 0x%llx, %u, 0x%llx)\n",
level, (unsigned long long) table_base, (unsigned) index,
(unsigned long long) value);
*((volatile uint64_t *) (table_base + index * 8)) = value;
}
static unsigned get_start_level(void)
{
switch (get_page_granule_shift()) {
case 12: return 1;
case 14: return 2;
case 16: return 2;
default: panic("Bad granule");
}
}
static unsigned get_virt_address_bits(void)
{
switch (get_page_granule_shift()) {
case 12: return 36;
case 14: return 36;
case 16: return 36;
default: panic("Bad granule");
}
}
static uint64_t get_mair(void)
{
return (MAIR_WRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITEBACK)) |
(MAIR_DISABLE << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_DISABLE));
}
static uint64_t get_tcr(void)
{
// Form the contents of the TCR register for the selected
// virtual address limit and page granule.
uint64_t tcr;
uint32_t txsz = 64 - get_virt_address_bits();
#if WITH_EL3
switch (get_page_granule_shift()) {
case 12: tcr = TCR_ELx_TG0_GRANULE_4KB; break;
case 14: tcr = TCR_ELx_TG0_GRANULE_16KB; break;
case 16: tcr = TCR_ELx_TG0_GRANULE_64KB; break;
default: panic("Bad granule");
}
tcr |= TCR_EL3_PS_36BITS |
TCR_ELx_SH0_OUTER | TCR_ELx_ORGN0_WRITEBACK | TCR_ELx_IRGN0_WRITEBACK |
(txsz << TCR_ELx_T0SZ_SHIFT);
#else
switch (get_page_granule_shift()) {
case 12: tcr = TCR_ELx_TG0_GRANULE_4KB | TCR_EL1_TG1_GRANULE_4KB; break;
case 14: tcr = TCR_ELx_TG0_GRANULE_16KB | TCR_EL1_TG1_GRANULE_16KB; break;
case 16: tcr = TCR_ELx_TG0_GRANULE_64KB | TCR_EL1_TG1_GRANULE_64KB; break;
default: panic("Bad granule");
}
tcr |= TCR_EL1_IPS_36BITS |
TCR_ELx_SH0_OUTER | TCR_ELx_ORGN0_WRITEBACK | TCR_ELx_IRGN0_WRITEBACK |
(txsz << TCR_ELx_T0SZ_SHIFT) |
TCR_EL1_EPD1_TTBR1_DISABLED |
TCR_EL1_SH1_OUTER | TCR_EL1_ORGN1_WRITEBACK | TCR_EL1_IRGN1_WRITEBACK |
(txsz << TCR_EL1_T1SZ_SHIFT);
#endif
return tcr;
}
static uint64_t get_table_flags(void)
{
// All table walks after the start level are non-secure.
return ARM_LPAE_NSTABLE | ARM_LPAE_TYPE_TABLE | ARM_LPAE_VALID;
}
static uint64_t get_block_flags(unsigned level, arm_mmu_extended_attr_t attr)
{
assert_valid_level(level);
RELEASE_ASSERT(level_permits_blocks(level));
bool normal, writable, executable;
switch (attr) {
case kARMMMUDeviceR:
normal = false; writable = false; executable = false; break;
case kARMMMUDeviceRX:
normal = false; writable = false; executable = true; break;
case kARMMMUDeviceRW:
normal = false; writable = true; executable = false; break;
case kARMMMUNormalR:
normal = true; writable = false; executable = false; break;
case kARMMMUNormalRX:
normal = true; writable = false; executable = true; break;
case kARMMMUNormalRW:
normal = true; writable = true; executable = false; break;
default:
panic("Bad attr %d", (int) attr);
}
// Always set Access Flag, Non-Secure and Valid.
uint64_t value = ARM_LPAE_AF | ARM_LPAE_NS | ARM_LPAE_VALID;
if (level < LAST_TABLE_LEVEL) {
value |= ARM_LPAE_TYPE_L0L1L2BLOCK;
} else {
value |= ARM_LPAE_TYPE_L3BLOCK;
}
uint64_t sh, attr_indx;
if (normal) {
sh = SH_OUTER_MEMORY;
attr_indx = CACHE_ATTRINDX_WRITEBACK;
} else {
sh = SH_NONE;
attr_indx = CACHE_ATTRINDX_DISABLE;
}
value |= ARM_LPAE_SH(sh);
value |= ARM_LPAE_ATTRINDX(attr_indx);
uint64_t ap = writable ? 0 : 2;
value |= ARM_LPAE_AP(ap);
if (!executable) {
value |= ARM_LPAE_PXN | ARM_LPAE_XN;
}
return value;
}
static uint64_t get_virt_size_bytes(void)
{
return 1ULL << get_virt_address_bits();
}
static uint64_t get_phys_size_bytes(void)
{
return 1ULL << 40;
}
static size_t get_level_shift(unsigned level)
{
assert_valid_level(level);
switch (get_page_granule_shift()) {
case 12: return 39 - (level * 9);
case 14: return 47 - (level * 11);
case 16: return 55 - (level * 13);
default: panic("Granule 2^%u not supported\n",
get_page_granule_shift());
}
}
static size_t get_level_entries(unsigned level)
{
assert_valid_level(level);
// 1 granule per sub-table, 8 bytes per entry.
return ((size_t) 1) << (get_page_granule_shift() - 3);
}
static addr_t tt_alloc(unsigned level)
{
// Allocate a translation table. The start level was
// allocated as tt_top_level in arm_mmu_init and should not be
// requested to be allocated. All levels are the
// same size: one granule.
RELEASE_ASSERT(level > get_start_level() && level <= LAST_TABLE_LEVEL);
size_t bytes = ((size_t) 1) << get_page_granule_shift();
mmu_spew("Allocate L%u size 0x%llx\n", level,
(unsigned long long) bytes);
if (bytes > tt_alloc_end || tt_alloc_next > tt_alloc_end - bytes) {
panic("Out of table allocation space L%u\n", level);
}
addr_t ret = tt_alloc_next;
// Alignment is guaranteed by arm_mmu_init() and all
// allocations being the same size (1 granule), but it's cheap
// to double-check allocation alignment.
if ((ret & (bytes - 1)) != 0) {
panic("Bad aligned pte alloc 0x%llx\n",
(unsigned long long) ret);
}
tt_alloc_next += bytes;
mmu_spew("Allocated L%u size 0x%llx at 0x%llx\n",
level, (unsigned long long) bytes, (unsigned long long) ret);
return ret;
}
static size_t map_at_level(unsigned level, addr_t table_base,
addr_t vaddr, addr_t paddr, size_t size,
arm_mmu_extended_attr_t attr,
bool blank_table)
{
mmu_spew("map_at_level(level=%u,\n"
" table_base=0x%llx,\n"
" vaddr=0x%llx,\n"
" paddr=0x%llx,\n"
" size=0x%llx,\n"
" attr=%u\n",
level, (unsigned long long) table_base,
(unsigned long long) vaddr, (unsigned long long) paddr,
(unsigned long long) size, (unsigned) attr);
// Check this mapping is bounded by virtual and physical address limits.
assert_valid_level(level);
RELEASE_ASSERT(size <= get_virt_size_bytes() &&
vaddr <= get_virt_size_bytes() - size);
RELEASE_ASSERT(size <= get_phys_size_bytes() &&
paddr <= get_phys_size_bytes() - size);
// Calculate constants about this translation level.
size_t level_shift = get_level_shift(level);
size_t level_entries = get_level_entries(level);
size_t level_size = 1ULL << level_shift;
size_t level_mask = level_size - 1;
bool permit_block = level_permits_blocks(level);
size_t index = (vaddr >> level_shift) & (level_entries - 1);
mmu_spew("shift:%d entries:%d size:0x%llx mask:0x%llx "
"block:%d index:%d\n",
(int) level_shift,
(int) level_entries,
(unsigned long long) level_size,
(unsigned long long) level_mask,
(int) permit_block,
(int) index);
// Can we make block entries here? Must be permitted at this
// level, have enough bytes remaining, and both virtual and
// physical addresses aligned to a block.
if (permit_block &&
size >= level_size &&
((vaddr | paddr) & level_mask) == 0) {
// Map contiguous blocks.
mmu_spew("Array of blocks\n");
size_t mapped_size = 0;
size_t remaining_size = size;
// Common block flags for all entries at this level.
uint64_t block_flags = get_block_flags(level, attr);
while (remaining_size >= level_size && index < level_entries) {
// Check the existing entry is unused.
uint64_t entry = get_table_entry(level, table_base,
index);
if (entry != 0) {
// Not difficult but no use case.
panic("Remapping an existing L%u at 0x%016llx",
level, (unsigned long long) vaddr);
}
// Form an entry.
uint64_t ent = block_flags | paddr;
// Write a block entry to the table.
set_table_entry(level, table_base, index, ent);
// Next index.
mapped_size += level_size;
remaining_size -= level_size;
paddr += level_size;
vaddr += level_size;
++index;
}
// Must make forward progress... and not too much.
RELEASE_ASSERT(mapped_size > 0 && mapped_size <= size);
return mapped_size;
} else {
// Sub-divide into a next level table.
mmu_spew("Sub-divide\n");
// Only so much granularity available.
RELEASE_ASSERT(level < LAST_TABLE_LEVEL);
// Get the existing entry.
uint64_t entry = blank_table ? 0 : get_table_entry(level, table_base, index);
if ((entry & ARM_LPAE_VALID) &&
(entry & ARM_LPAE_TYPE_MASK) == ARM_LPAE_TYPE_L0L1L2BLOCK) {
// Not hard but no use case.
panic("Breaking down blocks not implemented");
}
unsigned sub_level = level + 1;
bool blank_sub_table;
addr_t sub_base;
if (entry & ARM_LPAE_VALID) {
mmu_spew("Existing entry\n");
sub_base = entry & ARM_LPAE_TABLE_MASK;
blank_sub_table = false;
} else {
mmu_spew("Allocating sub-level table\n");
sub_base = tt_alloc(sub_level);
// Form an entry.
uint64_t ent = get_table_flags() | sub_base;
// Store an entry pointing to the sub-level table.
set_table_entry(level, table_base, index, ent);
blank_sub_table = true;
}
// Recurse at next level.
return map_at_level(sub_level, sub_base,
vaddr, paddr, size, attr, blank_sub_table);
}
}
void arm_mmu_map_range(addr_t vaddr, addr_t paddr, size_t size,
arm_mmu_extended_attr_t attr)
{
// Ensure all addresses and sizes are at least L3 aligned.
size_t l3_align = ((size_t) 1) << get_level_shift(LAST_TABLE_LEVEL);
RELEASE_ASSERT(((vaddr | paddr | size) & (l3_align - 1)) == 0);
while (size > 0) {
// Map at the top level first, allowing the mapper function
// to break it down into smaller chunks as necessary.
size_t mapped = map_at_level(get_start_level(),
(addr_t) tt_top_level,
vaddr, paddr, size, attr, false);
RELEASE_ASSERT(mapped <= size);
size -= mapped;
vaddr += mapped;
paddr += mapped;
}
if ((arm_read_sctlr() & SCTLR_M_ENABLED) != 0) {
arm_flush_tlbs();
}
}
void arm_mmu_map_ro(addr_t addr, size_t size)
{
arm_mmu_map_range(addr, addr, size, kARMMMUNormalR);
}
void arm_mmu_map_rw(addr_t addr, size_t size)
{
arm_mmu_map_range(addr, addr, size, kARMMMUNormalRW);
}
void arm_mmu_map_rx(addr_t addr, size_t size)
{
arm_mmu_map_range(addr, addr, size, kARMMMUNormalRX);
}
void arm_mmu_map_device_ro(addr_t addr, size_t size)
{
arm_mmu_map_range(addr, addr, size, kARMMMUDeviceR);
}
void arm_mmu_map_device_rw(addr_t addr, size_t size)
{
arm_mmu_map_range(addr, addr, size, kARMMMUDeviceRW);
}
void arm_mmu_map_device_rx(addr_t addr, size_t size)
{
arm_mmu_map_range(addr, addr, size, kARMMMUDeviceRX);
}
void arm_mmu_init(bool resume)
{
// Setup memory attributes.
arm_write_mair(get_mair());
// Setup translation control register.
arm_write_tcr(get_tcr());
/*
* Sub-level page table allocations starts in the book-keeping
* region after all stacks, rounded up. Allocation rounded down.
*/
size_t granule_mask = (((size_t) 1) << get_page_granule_shift()) - 1;
addr_t page_tables_base;
size_t page_tables_size;
mmu_get_memmap(&page_tables_base,
&page_tables_size);
RELEASE_ASSERT((page_tables_base & granule_mask) == 0);
RELEASE_ASSERT((page_tables_size & granule_mask) == 0);
tt_alloc_start = page_tables_base;
tt_alloc_next = tt_alloc_start;
tt_alloc_end = (page_tables_base + page_tables_size) & ~granule_mask;
// allocate top-level page table
tt_top_level = (uint64_t *)tt_alloc_start;
tt_alloc_next += 1 << get_page_granule_shift();
// Defer to platform code for the memory map setup.
platform_mmu_setup(resume);
// Write the translation table base.
arm_write_ttbr0(tt_top_level);
// Flush TLBs.
arm_flush_tlbs();
}
#ifdef TEST
// static->global stubs for mmu_test.c
uint64_t *mmu_get_tt_top_level(void) { return tt_top_level; }
unsigned mmu_get_start_level(void) { return get_start_level(); }
uint64_t mmu_get_tcr(void) { return get_tcr(); }
uint64_t mmu_tt_alloc(unsigned level) { return tt_alloc(level); }
size_t mmu_get_level_entries(unsigned level) {
return get_level_entries(level);
}
uint64_t mmu_get_table_flags(void) {
return get_table_flags();
}
uint64_t mmu_get_block_flags(unsigned level, arm_mmu_extended_attr_t attr) {
return get_block_flags(level, attr);
}
size_t mmu_map_at_level(unsigned level, addr_t table_base,
addr_t vaddr, addr_t paddr, size_t size,
arm_mmu_extended_attr_t attr)
{
return map_at_level(level, table_base, vaddr, paddr, size, attr, false);
}
void mmu_get_tt_alloc_range(addr_t *start, addr_t *next, addr_t *end)
{
*start = tt_alloc_start;
*next = tt_alloc_next;
*end = tt_alloc_end;
}
void mmu_reset_state(void)
{
tt_top_level = 0;
tt_alloc_start = 0;
tt_alloc_next = 0;
tt_alloc_end = 0;
}
#endif // defined(TEST)
arch/arm64/mmu_test.c
+846
View File
@@ -0,0 +1,846 @@
/*
* Copyright (C) 2014 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <stdbool.h>
#include <assert.h>
#include <inttypes.h>
#include <setjmp.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <arch/arm/arm.h>
#include <arch/arm64/proc_reg.h>
#define VERBOSITY FAILURE
#define FAILURE 0
#define INFO 1
#define SPEW 2
static size_t tests_run, tests_passed;
enum Reason {
REASON_OK = 0,
REASON_EXPECTED_PANIC,
REASON_UNEXPECTED_PANIC,
REASON_PARAMS,
REASON_LEVEL,
REASON_OOM,
REASON_PTE_ALIGNMENT,
REASON_PTE_BASE,
REASON_PTE_INDEX,
REASON_PTE_ACCESS,
REASON_PTE_MISMATCH
};
static void test_unwind(enum Reason reason) __attribute__((noreturn));
#define TEST_ASSERT(reason, c) \
do { \
if (!(c)) { \
printf("!!! Test assertion failure: %s\n", \
#c ); \
test_unwind(reason); \
} \
} while (0)
struct State {
// Environment constants for an individual test.
const char *test_name;
const char *expect_panic_func;
const char *expect_panic_substr;
size_t level_entries[4];
void *page_tables_base;
size_t page_tables_size;
size_t total_stack_size;
unsigned page_granule_shift;
jmp_buf jmp_env;
// Updated during test.
bool arm_flush_tlbs_called;
// Callbacks from test.
void (^mmu_setup_func)();
void (^get_table_entry_hook)();
void (^set_table_entry_hook)();
};
static struct State state;
unsigned get_page_granule_shift(void) { return state.page_granule_shift; }
void mmu_get_memmap(addr_t *page_tables_base,
size_t *page_tables_size)
{
*page_tables_base = (addr_t) state.page_tables_base;
*page_tables_size = state.page_tables_size;
}
// Access to mmu.c static functions and variables.
uint64_t *mmu_get_tt_top_level(void);
unsigned mmu_get_start_level(void);
uint64_t mmu_get_tcr(void);
addr_t mmu_tt_alloc(unsigned level);
size_t mmu_get_level_entries(unsigned level);
uint64_t mmu_get_table_flags(void);
uint64_t mmu_get_block_flags(unsigned level, arm_mmu_extended_attr_t attr);
size_t mmu_map_at_level(unsigned level, addr_t table_base,
addr_t vaddr, addr_t paddr, size_t size,
arm_mmu_extended_attr_t attr);
void mmu_get_tt_alloc_range(addr_t *start, addr_t *next, addr_t *end);
void mmu_reset_state(void);
const char *reason_to_string(int reason)
{
switch (reason) {
case REASON_OK: return "OK";
case REASON_EXPECTED_PANIC: return "Expected panic() called";
case REASON_UNEXPECTED_PANIC: return "Unexpected panic() called";
case REASON_PARAMS: return "Bad parameters detected";
case REASON_LEVEL: return "Invalid level used";
case REASON_OOM: return "Test ran out of memory for structures";
case REASON_PTE_ALIGNMENT: return "Bad page table alignment";
case REASON_PTE_BASE: return "Page table base does not match allocated";
case REASON_PTE_INDEX: return "Page table index out of bounds";
case REASON_PTE_ACCESS: return "Page table unexpected access";
case REASON_PTE_MISMATCH: return "Page table value mismatch";
default:
printf("Bad reason code %d\n", (int) reason);
abort();
}
}
void mmu_printf(int verbosity, const char *fmt, ...)
{
if (VERBOSITY >= verbosity) {
va_list va;
va_start(va, fmt);
vprintf(fmt, va);
va_end(va);
}
}
static void test_unwind(enum Reason reason)
{
assert(reason != REASON_OK);
longjmp(state.jmp_env, (int) reason);
}
void _panic(const char *func, const char *fmt, ...)
{
char *buf = malloc(4096);
va_list va;
va_start(va, fmt);
vsnprintf(buf, 4096, fmt, va);
va_end(va);
bool expected = state.expect_panic_func && state.expect_panic_substr &&
strstr(func, state.expect_panic_func) != NULL &&
strstr(buf, state.expect_panic_substr) != NULL;
if (!expected) {
mmu_printf(FAILURE, "!!! Unexpected panic in function %s: %s\n", func, buf);
} else {
mmu_printf(INFO, "Expected panic in function %s: %s\n", func, buf);
}
free(buf);
if (expected) {
test_unwind(REASON_EXPECTED_PANIC);
} else if (strstr(buf, "Out of table allocation space")) {
test_unwind(REASON_OOM);
} else {
test_unwind(REASON_UNEXPECTED_PANIC);
}
}
static void expect_panic(const char *func, const char *substr)
{
state.expect_panic_func = func;
state.expect_panic_substr = substr;
}
uint64_t check_valid_index(unsigned level, addr_t table_base, size_t index)
{
if (level < mmu_get_start_level() || level > 3) {
mmu_printf(FAILURE, "L%u invalid level\n", level);
test_unwind(REASON_LEVEL);
}
void *p = NULL;
size_t table_size =
mmu_get_level_entries(level) * sizeof(uint64_t);
size_t table_entries = mmu_get_level_entries(level);
uint64_t alloc_start, alloc_next, alloc_end;
mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
if (table_base < alloc_start ||
table_size > alloc_next ||
table_base > alloc_next - table_size) {
mmu_printf(FAILURE, "L%u base 0x%" PRIx64 " size 0x%" PRIx64
" outside allocated space\n",
level,
(uint64_t) table_base,
(uint64_t) table_size);
test_unwind(REASON_PTE_BASE);
}
if (index > table_entries) {
mmu_printf(FAILURE, "L%u entry %u >= table_entries %u\n",
level, (unsigned) index,
(unsigned) table_entries);
test_unwind(REASON_PTE_INDEX);
}
p = (void *) (table_base + index * 8);
uint64_t result;
memcpy(&result, p, sizeof(result));
return result;
}
static void get_table_entry_internal(unsigned level, addr_t table_base,
size_t index)
{
mmu_printf(SPEW, "get_table_entry(%u, 0x%" PRIx64 ", %u) ",
level, (uint64_t) table_base, (unsigned) index);
uint64_t result = check_valid_index(level, table_base, index);
mmu_printf(SPEW, "= 0x%016" PRIx64 "\n", result);
}
void get_table_entry_hook(unsigned level, addr_t table_base, size_t index)
{
if (state.get_table_entry_hook) {
state.get_table_entry_hook(level, table_base, index);
return;
} else {
get_table_entry_internal(level, table_base, index);
}
}
static void set_table_entry_internal(unsigned level, addr_t table_base,
size_t index, uint64_t value)
{
mmu_printf(SPEW, "set_table_entry(%u, 0x%" PRIx64 ", %u, "
"0x%" PRIx64 ") ",
level, (uint64_t) table_base, (unsigned) index,
value);
uint64_t old = check_valid_index(level, table_base, index);
mmu_printf(SPEW, " was 0x%016" PRIx64 "\n", old);
}
void set_table_entry_hook(unsigned level, addr_t table_base, size_t index,
uint64_t value)
{
if (state.set_table_entry_hook) {
state.set_table_entry_hook(level, table_base, index, value);
return;
} else {
set_table_entry_internal(level, table_base, index, value);
}
}
// Mock HAL.
uint64_t arm_read_sctlr(void)
{
mmu_printf(INFO, "arm_read_sctlr()\n");
return 0;
}
void arm_write_mair(uint64_t mair)
{
mmu_printf(INFO, "arm_write_mair(0x%016" PRIx64 ")\n", (uint64_t) mair);
}
void arm_write_tcr(uint64_t tcr)
{
mmu_printf(INFO, "arm_write_tcr(0x%016" PRIx64 ")\n", (uint64_t) tcr);
}
void arm_write_ttbr0(void *ttbr0)
{
mmu_printf(INFO, "arm_write_ttbr0(%p)\n", ttbr0);
if ((addr_t) ttbr0 != (addr_t) mmu_get_tt_top_level()) {
mmu_printf(FAILURE, "TTBR0 written with %p expected %p\n",
ttbr0, mmu_get_tt_top_level());
test_unwind(REASON_PTE_BASE);
}
}
void arm_flush_tlbs(void)
{
mmu_printf(INFO, "arm_flush_tlbs()\n");
state.arm_flush_tlbs_called = true;
}
void platform_mmu_setup(bool resume)
{
(void) resume;
uint64_t alloc_start, alloc_next, alloc_end;
mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
mmu_printf(INFO, "platform_mmu_setup(%d)\n", (int) resume);
mmu_printf(INFO, "tt_top_level = %p\n", mmu_get_tt_top_level());
mmu_printf(INFO, "page_tables_base = %p\n", state.page_tables_base);
mmu_printf(INFO, "page_tables_size = 0x%" PRIx64 "\n",
(uint64_t) state.page_tables_size);
mmu_printf(INFO, "tt_alloc_start = 0x%" PRIx64 "\n"
"tt_alloc_next = 0x%" PRIx64 "\n"
"tt_alloc_end = 0x%" PRIx64 "\n",
(uint64_t) alloc_start,
(uint64_t) alloc_next,
(uint64_t) alloc_end);
if (state.mmu_setup_func) {
state.mmu_setup_func();
}
}
// Test setup functions.
static void reset_state(void)
{
mmu_reset_state();
assert(state.page_tables_base == NULL); // Prevent leak.
memset(&state, 0, sizeof(state));
}
static void mmu_setup_flat(void)
{
uint64_t io_base = 0x200000000ULL;
uint64_t io_size = 256ULL * 1024 * 1024;
uint64_t sdram_base = 0x800000000ULL;
uint64_t sdram_len = 1ULL << 30;
arm_mmu_map_rw(sdram_base, sdram_len);
arm_mmu_map_device_rw(io_base, io_size);
}
static void mmu_setup_fail_remap_existing(void)
{
uint64_t io_base = 0x200000000ULL;
uint64_t io_size = 256ULL * 1024 * 1024;
arm_mmu_map_rw(io_base, io_size);
// Map again
arm_mmu_map_device_rw(io_base, io_size);
}
static void mmu_setup_fail_breaking_down(void)
{
uint64_t io_base = 0x200000000ULL;
uint64_t io_size = 256ULL * 1024 * 1024;
arm_mmu_map_rw(io_base, io_size);
// Map one L3 page again
arm_mmu_map_rw(io_base, 4096);
}
static void mmu_setup_misaligned(void)
{
uint64_t sdram_base = 0x800000000ULL;
uint64_t sdram_len = 1ULL << 30;
arm_mmu_map_rw(sdram_base, sdram_len + 512);
}
static void test_delete(void)
{
uint64_t alloc_start, alloc_next, alloc_end;
mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
mmu_printf(INFO, "Allocation space used: 0x%" PRIx64 "\n",
alloc_next - alloc_start);
mmu_printf(INFO, "Allocation space remaining: 0x%" PRIx64 "\n",
alloc_end - alloc_next);
if (state.page_tables_base) {
free(state.page_tables_base);
state.page_tables_base = NULL;
}
reset_state();
}
static int mmu_test(void)
{
mmu_printf(INFO, "Test \"%s\" running\n", state.test_name);
int ret = posix_memalign(&state.page_tables_base,
1 << state.page_granule_shift,
state.page_tables_size);
assert(ret == 0);
memset(state.page_tables_base, 0, state.page_tables_size);
arm_mmu_init(false);
// Calls back on platform_mmu_setup(resume)
if (state.arm_flush_tlbs_called) {
return true;
} else {
mmu_printf(FAILURE, "!!! arm_flush_tlbs() was not called\n");
return false;
}
}
static void do_test(enum Reason expect_reason, int (^test_func)(void))
{
assert(state.test_name != NULL);
mmu_printf(INFO, "----------------------------------------"
"--------------------------------------\n");
mmu_printf(INFO, "Test: %s\n", state.test_name);
// Increment number of tests run.
++tests_run;
// Set a jump environment for panic/abort exit path.
int reason_code = setjmp(state.jmp_env);
if (reason_code != 0) {
// Returns non-zero if longjmp() invoked to return here.
if (reason_code == (int) expect_reason) {
// Failure for an expected reason: exercising error
// detection.
mmu_printf(INFO, "Test \"%s\" aborted for expected reason: %s\n",
state.test_name,
reason_to_string(expect_reason));
++tests_passed;
} else {
mmu_printf(FAILURE, "!!! Test \"%s\" failed: %s\n",
state.test_name,
reason_to_string(reason_code));
mmu_printf(FAILURE, "!!! Expected outcome: %s\n",
reason_to_string(expect_reason));
}
test_delete();
reset_state();
return;
}
if (test_func()) {
mmu_printf(INFO, "Test \"%s\" completed, passed\n",
state.test_name);
++tests_passed;
} else {
mmu_printf(FAILURE, "!!! Test \"%s\" failed at completion\n",
state.test_name);
mmu_printf(FAILURE, "!!! Expected outcome: %s\n",
reason_to_string(expect_reason));
}
test_delete();
reset_state();
}
static void setup_standard_granule(size_t granule)
{
switch (granule) {
case 4096:
state.page_granule_shift = 12;
state.page_tables_size = 4096 * 4;
break;
case 16384:
state.page_granule_shift = 14;
state.page_tables_size = 16384 * 4;
break;
case 65536:
state.page_granule_shift = 16;
state.page_tables_size = 65536 * 4;
break;
default:
mmu_printf(FAILURE, "Bad granule %u\n", (unsigned) granule);
abort();
}
}
static void test_granule(size_t bytes)
{
state.mmu_setup_func = ^{ mmu_setup_flat(); };
setup_standard_granule(bytes);
do_test(REASON_OK, ^{ return mmu_test(); });
}
static void test_granule_4kb(void)
{
state.test_name = "4KB granule";
test_granule(4096);
}
static void test_granule_16kb(void)
{
state.test_name = "16KB granule";
test_granule(16384);
}
static void test_granule_64kb(void)
{
state.test_name = "64KB granule";
test_granule(65536);
}
static void test_fail_oom(void)
{
state.test_name = "Detect out-of-memory PTE allocation";
state.mmu_setup_func = ^{ mmu_setup_flat(); };
setup_standard_granule(4096);
// Too small.
state.page_tables_size = 16384;
expect_panic("tt_alloc", "Out of table allocation space L2");
do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
}
static void test_fail_range_misaligned(void)
{
state.test_name = "Detect misaligned mapping request";
state.mmu_setup_func = ^{ mmu_setup_misaligned(); };
setup_standard_granule(4096);
// Expect vaddr, paddr, size to misaligned.
expect_panic("arm_mmu_map_range", "vaddr | paddr | size");
do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
}
static void test_fail_bad_granule(void)
{
state.test_name = "Detect bad granule size";
state.mmu_setup_func = ^{
// Alter the granule to an illegal value.
++state.page_granule_shift;
mmu_setup_flat();
};
setup_standard_granule(4096);
expect_panic("get_level_shift", "Granule 2^13");
do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
}
static void test_fail_remap_existing(void)
{
state.test_name = "Detect remapping existing page";
state.mmu_setup_func = ^{ mmu_setup_fail_remap_existing(); };
setup_standard_granule(4096);
expect_panic("map_at_level", "Remapping an existing L2");
do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
}
static void test_fail_breaking_down(void)
{
state.test_name = "Detect breaking down existing mapping";
state.mmu_setup_func = ^{ mmu_setup_fail_breaking_down(); };
setup_standard_granule(4096);
expect_panic("map_at_level", "Breaking down");
do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
}
static void test_get_tcr(void)
{
state.test_name = "get_tcr()";
do_test(REASON_OK,
^{
state.page_granule_shift = 12;
mmu_get_tcr();
state.page_granule_shift = 14;
mmu_get_tcr();
state.page_granule_shift = 16;
mmu_get_tcr();
return true;
});
}
static void test_fail_get_tcr_bad_granule(void)
{
state.test_name = "Detect get_tcr() with bad granule";
expect_panic("get_tcr", "Bad granule");
state.page_granule_shift = 13;
do_test(REASON_EXPECTED_PANIC, ^{ mmu_get_tcr(); return false; });
}
static void test_get_block_flags_for_level(unsigned level)
{
// [63:59] SBZ
// [58:55] S/W use SBZ
// [54:53] XN,PXN = execute never, 0 for RX, 3 for R/RW
// [52] Contiguous hint, SBZ
// [51:12] SBZ and address bits, also SBZ
// [11] nG - non-global, we don't use ASID - SBZ
// [10] AF - access flag, should set - SBO
// [9:8] SH - shareability - 0 for device, 2 for normal
// [7:6] AP - access permission - 0 for RW, 2 for RO
// [5] NS - non-secure, should set - SBO
// [4:2] AttrIndx - 0 for device, 1 for normal
// [1] Walk - L0,L1,L2 0=Block; L3 1=Block
// [0] V - 0=fault, 1=valid
const uint64_t sbo = (1 << 10) | (1 << 5) | (1 << 0);
const uint64_t normal = (2 << 8) | (1 << 2);
const uint64_t device = (0 << 8) | (0 << 2);
const uint64_t read_write = 0 << 6;
const uint64_t read_only = 2 << 6;
const uint64_t execute = 0ULL << 53;
const uint64_t no_execute = 3ULL << 53;
static const struct {
arm_mmu_extended_attr_t attr;
uint64_t expect;
} tests[] = {
{ kARMMMUDeviceR, sbo | device | read_only | no_execute },
{ kARMMMUDeviceRX, sbo | device | read_only | execute },
{ kARMMMUDeviceRW, sbo | device | read_write | no_execute },
{ kARMMMUNormalR, sbo | normal | read_only | no_execute },
{ kARMMMUNormalRX, sbo | normal | read_only | execute },
{ kARMMMUNormalRW, sbo | normal | read_write | no_execute },
};
for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); ++i) {
arm_mmu_extended_attr_t attr = tests[i].attr;
uint64_t expect = tests[i].expect;
if (level == 3) {
expect |= 1 << 1; // "Walk" flag
}
uint64_t value = mmu_get_block_flags(level, attr);
if (value != expect) {
mmu_printf(FAILURE, "get_block_flags level %u test %u enum %u\n",
level, (unsigned) i, (unsigned) attr);
mmu_printf(FAILURE, "got 0x%" PRIx64 "\n", value);
mmu_printf(FAILURE, "expect 0x%" PRIx64 "\n", expect);
test_unwind(REASON_PTE_MISMATCH);
}
}
}
static void test_get_block_flags(void)
{
state.test_name = "get_block_flags()";
state.page_granule_shift = 12;
do_test(REASON_OK, ^{
for (unsigned level = 1; level <= 3; ++level) {
test_get_block_flags_for_level(level);
}
return true;
});
}
static void test_get_table_flags(void)
{
state.test_name = "get_tte_flags()";
state.page_granule_shift = 12;
// [63] NS
// [62:61] AP
// [60] XN
// [59] PXN
// [58:52] ignored
// [51:48] zero, sbz
// [47:12] TableOutputAddress
// [11:2] ignored
// [1] type=1, sbo
// [0] valid=1, sbo
const uint64_t expect = (1ULL << 63) | (1ULL << 1) | (1ULL << 0);
do_test(REASON_OK, ^{
uint64_t value = mmu_get_table_flags();
if (value != expect) {
mmu_printf(FAILURE, "get_table_flags() mismatch\n");
mmu_printf(FAILURE, "Got 0x%" PRIx64 "\n", value);
mmu_printf(FAILURE, "Expect 0x%" PRIx64 "\n", expect);
test_unwind(REASON_PTE_MISMATCH);
}
return true;
});
}
static void test_fail_get_block_flags_bad_attr(void)
{
state.test_name = "Detect get_block_flags() bad attribute";
state.page_granule_shift = 12;
expect_panic("get_block_flags", "Bad attr");
do_test(REASON_EXPECTED_PANIC,
^{ mmu_get_block_flags(3, -1); return false; });
}
static void test_fail_get_block_flags_bad_level(void)
{
state.test_name = "Detect get_pte_flags() bad level";
state.page_granule_shift = 16;
expect_panic("get_block_flags", "level_permits_blocks");
// Should not allow a block at L1 with 64KB granule.
do_test(REASON_EXPECTED_PANIC,
^{ mmu_get_block_flags(1, kARMMMUNormalRW); return false; });
}
static void test_tt_alloc(unsigned granule_shift)
{
state.test_name = "tt_alloc()";
state.page_granule_shift = granule_shift;
state.page_tables_size = (1 << granule_shift) * 3;
state.mmu_setup_func = ^{};
do_test(REASON_OK,
^{
arm_mmu_init(false);
if (granule_shift == 12)
mmu_tt_alloc(2);
mmu_tt_alloc(3);
return true;
});
}
static void test_tt_alloc_bad_level(void)
{
state.test_name = "Detect tt_alloc() bad level";
state.page_granule_shift = 14;
state.page_tables_size = 16384 * 3;
state.mmu_setup_func = ^{};
expect_panic("tt_alloc", "level > ");
do_test(REASON_EXPECTED_PANIC,
^{
arm_mmu_init(false);
mmu_tt_alloc(1);
return false;
});
}
static void test_tt_alloc_bad_alignment(void)
{
state.test_name = "Detect tt_alloc() bad alignment";
state.page_granule_shift = 14;
state.page_tables_size = 65536 * 3; // Enough space.
state.mmu_setup_func = ^{};
expect_panic("tt_alloc", "Bad aligned pte alloc");
do_test(REASON_EXPECTED_PANIC,
^{
arm_mmu_init(false);
mmu_tt_alloc(3);
// Allocation is aligned, so only way to trip it up
// is to artificially tamper with the granule.
state.page_granule_shift = 16;
mmu_tt_alloc(3);
return false;
});
}
static void test_map_at_top_level_large(void)
{
state.test_name = "map_at_level() at top level traverses only L1";
setup_standard_granule(4096);
__block uint64_t alloc_start, alloc_next, alloc_end;
state.mmu_setup_func = ^{
mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
mmu_map_at_level(1,
(addr_t) mmu_get_tt_top_level(),
1ULL << 30, 1ULL << 30, 1ULL << 30,
kARMMMUNormalRW);
};
__block size_t calls = 0;
state.set_table_entry_hook = ^(unsigned level,
uint64_t base,
size_t index,
uint64_t value) {
(void) value;
++calls;
mmu_printf(INFO, "calls:%u %u 0x%" PRIx64 " 0x%" PRIx64
" 0x%" PRIx64 "\n",
(unsigned) calls,
level, base, (uint64_t) index, value);
if (calls == 1) {
if (level != 1 ||
base != (uint64_t) mmu_get_tt_top_level() ||
index != 1) {
mmu_printf(FAILURE, "Did not write L0 index 0\n");
test_unwind(REASON_PTE_ACCESS);
}
} else {
mmu_printf(FAILURE, "Too many calls to get_table_entry()\n");
test_unwind(REASON_PTE_ACCESS);
}
};
do_test(REASON_OK,
^{
if (!mmu_test()) {
return false;
}
return true;
});
}
static void test_map_at_l3(void)
{
state.test_name = "map_at_level() at l3";
setup_standard_granule(16384);
__block uint64_t alloc_start = 0, alloc_next = 0, alloc_end = 0;
__block uint64_t l3_base = 0;
state.mmu_setup_func = ^{
mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
l3_base = mmu_tt_alloc(3);
mmu_map_at_level(3, l3_base,
(1ULL << 30) + 16384 * 3,
16384 * 7,
16384,
kARMMMUDeviceRW);
};
__block size_t calls = 0;
state.set_table_entry_hook = ^(unsigned level,
uint64_t base,
size_t index,
uint64_t value) {
(void) value;
++calls;
mmu_printf(INFO, "calls:%u %u 0x%" PRIx64 " 0x%" PRIx64
" 0x%" PRIx64 "\n",
(unsigned) calls,
level, base, (uint64_t) index, value);
if (calls == 1) {
if (level != 3 ||
base != l3_base ||
index != 3) {
mmu_printf(FAILURE, "Did not write L3 index 3\n");
test_unwind(REASON_PTE_ACCESS);
}
} else {
mmu_printf(FAILURE, "Too many calls to get_table_entry()\n");
test_unwind(REASON_PTE_ACCESS);
}
};
do_test(REASON_OK,
^{
if (!mmu_test()) {
return false;
}
return true;
});
}
int main()
{
reset_state();
test_granule_4kb();
test_granule_16kb();
test_granule_64kb();
test_fail_oom();
test_fail_range_misaligned();
test_fail_bad_granule();
test_fail_remap_existing();
test_fail_breaking_down();
test_get_tcr();
// test_fail_get_tcr_bad_granule(); !!!FIXME!!! <rdar://problem/19225114> Unexpected panic in function get_virt_address_bits: Bad granule
test_get_block_flags();
test_get_table_flags();
test_fail_get_block_flags_bad_attr();
test_fail_get_block_flags_bad_level();
test_tt_alloc(12);
test_tt_alloc(14);
test_tt_alloc(16);
test_tt_alloc_bad_level();
test_tt_alloc_bad_alignment();
test_map_at_top_level_large();
test_map_at_l3();
mmu_printf(INFO, "%" PRIu64 "/%" PRIu64 " tests passed\n",
(uint64_t) tests_passed, (uint64_t) tests_run);
assert(tests_passed <= tests_run);
if (tests_passed == tests_run) {
mmu_printf(INFO, "Pass\n");
return 0;
} else {
mmu_printf(INFO, "!!! Fail\n");
return 1;
}
}
arch/arm64/rules.mk
+176
View File
@@ -0,0 +1,176 @@
# Copyright (C) 2011-2014 Apple Inc. All rights reserved.
#
# This document is the property of Apple Computer, Inc.
# It is considered confidential and proprietary.
#
# This document may not be reproduced or transmitted in any form,
# in whole or in part, without the express written permission of
# Apple Computer, Inc.
#
LOCAL_DIR := $(GET_LOCAL_DIR)
#
# ARM CPU config
#
# sanity check
ifeq ($(ARM_CPU),)
$(error ARM_CPU is not set)
endif
ifneq ($(ARM_ARCH),)
$(info ARM_ARCH does not need to be set at the platform level)
endif
ARCH_ARMV8_CPUS := apple-cyclone apple-typhoon apple-typhoon-capri apple-twister apple-hurricane-zephyr
################################################################################
# Per-CPU configuration
################################################################################
#
# Apple Cyclone/Typhoon
#
ifneq (,$(filter $(ARCH_ARMV8_CPUS),$(ARM_CPU)))
ARM_ARCH ?= arm64
ARM_ARCH_OPTIONS := ARCH_ARMv8=1 WITH_VFP=1 WITH_VFP_ALWAYS_ON=1
OPTIONS += \
CPU_APPLE_CYCLONE=1 \
L1_CACHELINE_SIZE=64 \
L1_CACHELINE_SHIFT=6 \
CPU_CACHELINE_SIZE=L1_CACHELINE_SIZE \
CPU_CACHELINE_SHIFT=L1_CACHELINE_SHIFT \
WITH_HW_TIMER=1
ALL_OBJS += \
$(LOCAL_DIR)/mmu.o \
$(LOCAL_DIR)/timer.o
ifneq (,$(filter apple-cyclone apple-typhoon,$(ARM_CPU)))
WITH_EL3 := 1
OPTIONS += \
L1_CACHEINDEX_SHIFT=9 \
L1_CACHEWAY_SHIFT=1 \
L2_CACHE_SIZE=1048576 \
L2_CACHEINDEX_SHIFT=11 \
L2_CACHEWAY_SHIFT=3 \
L2_CACHEWAY_COUNT=8 \
L2_CACHELINE_SHIFT=6 \
PAGE_GRANULE_SHIFT=12 \
PAGE_SIZE=4096 \
WITH_EL3=1 \
WITH_MCC_AS_RAM=1
#4k-algined linker pages
GLOBAL_LDFLAGS += -Wl,-segalign,0x1000
else ifneq (,$(filter apple-typhoon-capri,$(ARM_CPU)))
WITH_EL3 := 1
OPTIONS += \
L1_CACHEINDEX_SHIFT=9 \
L1_CACHEWAY_SHIFT=1 \
L2_CACHE_SIZE=2097152 \
L2_CACHEINDEX_SHIFT=12 \
L2_CACHEWAY_SHIFT=3 \
L2_CACHEWAY_COUNT=8 \
L2_CACHELINE_SHIFT=6 \
PAGE_GRANULE_SHIFT=12 \
PAGE_SIZE=4096 \
WITH_EL3=1 \
WITH_MCC_AS_RAM=1
#4k-algined linker pages
GLOBAL_LDFLAGS += -Wl,-segalign,0x1000
else ifneq (,$(filter apple-twister,$(ARM_CPU)))
WITH_EL3 := 1
OPTIONS += \
L1_CACHEINDEX_SHIFT=8 \
L1_CACHEWAY_SHIFT=2 \
L2_CACHE_SIZE=3145728 \
L2_CACHEINDEX_SHIFT=12 \
L2_CACHEWAY_SHIFT=4 \
L2_CACHEWAY_COUNT=12 \
L2_CACHELINE_SHIFT=6 \
PAGE_GRANULE_SHIFT=14 \
PAGE_SIZE=16384 \
WITH_EL3=1 \
WITH_MCC_AS_RAM=1
#16k-algined linker pages
GLOBAL_LDFLAGS += -Wl,-segalign,0x4000
else ifneq (,$(filter apple-hurricane-zephyr,$(ARM_CPU)))
OPTIONS += \
L1_CACHEINDEX_SHIFT=8 \
L1_CACHEWAY_SHIFT=2 \
L2_CACHE_SIZE=3145728 \
L2_CACHEINDEX_SHIFT=11 \
L2_CACHEWAY_SHIFT=4 \
L2_CACHEWAY_COUNT=12 \
L2_CACHELINE_SHIFT=7 \
PAGE_GRANULE_SHIFT=14 \
PAGE_SIZE=16384 \
WITH_L2_AS_RAM=1
#16k-algined linker pages
GLOBAL_LDFLAGS += -Wl,-segalign,0x4000
endif
endif
################################################################################
# General and conditional configuration
################################################################################
# sanity check
ifeq ($(ARM_ARCH),)
$(error ARM_ARCH not set, no valid CPU selected)
endif
# build for the proper cpu
OPTIONS += ARCH_ARM=1
GLOBAL_ALLFLAGS += -arch $(ARM_ARCH) $(addprefix -D,$(ARM_ARCH_OPTIONS))
GLOBAL_LDFLAGS += -arch $(ARM_ARCH)
# Tell the build system how libraries should be tagged
LIBRARY_TAG += $(ARM_ARCH)
LIBRARY_OPTIONS += ARCH=arm64 ARM_ARCH=$(ARM_ARCH)
GLOBAL_INCLUDES += arch/arm/include
GLOBAL_INCLUDES += $(LOCAL_DIR)/include
# Defaults for stack sizes (stacks laid out from top to bottom as listed)
INTERRUPT_STACK_SIZE ?= 8192
BOOTSTRAP_STACK_SIZE ?= 4096
EXCEPTION_STACK_SIZE ?= 4096
OPTIONS += \
INTERRUPT_STACK_SIZE=$(INTERRUPT_STACK_SIZE) \
EXCEPTION_STACK_SIZE=$(EXCEPTION_STACK_SIZE) \
BOOTSTRAP_STACK_SIZE=$(BOOTSTRAP_STACK_SIZE)
# Force start.o and asm.o to link first
ALL_OBJS := \
$(LOCAL_DIR)/start.o \
$(LOCAL_DIR)/asm.o \
$(ALL_OBJS)
# Code common to all cores
ALL_OBJS += \
$(LOCAL_DIR)/cpu.o \
$(LOCAL_DIR)/context.o \
$(LOCAL_DIR)/entropy.o \
$(LOCAL_DIR)/fp.o \
$(LOCAL_DIR)/handlers.o \
$(LOCAL_DIR)/task.o
ifeq ($(WITH_EL3),1)
ALL_OBJS += \
$(LOCAL_DIR)/exceptions_el3.o
else
ALL_OBJS += \
$(LOCAL_DIR)/exceptions_el1.o
endif
LINKER_EXPORTS := $(LOCAL_DIR)/link.exp
arch/arm64/start.S
+262
View File
@@ -0,0 +1,262 @@
/*
* Copyright (c) 2011-2014 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <arch/arm64/proc_reg.h>
#include <platform/memmap.h>
.text
.align 12
.no_dead_strip _start
.globl _start
_start:
// Relocate, if needed
adrp x0, _start@page
add x0, x0, _start@pageoff
ldr x1, L_TEXT_BASE
#ifdef PLATFORM_START_FUNCTION
/*
* x0 and x1 must be preserved
* 'eq' will be true if no text relocation is required
*/
bl PLATFORM_START_FUNCTION
#endif
cmp x1, x0
b.eq L__continue_start
#if !CPU_APPLE_CYCLONE
mov x2, #(SCTLR_I_ENABLED)
msr SCTLR_ELx, x2
ic iallu
#endif
mov lr, x1
ldr x2, L_TEXT_END
ldr x3, L_TEXT_BASE
sub x2, x2, x3
L__relocate_loop:
ldp x3, x4, [x0], #16
stp x3, x4, [x1], #16
subs x2, x2, #16
b.ne L__relocate_loop
ret
L__continue_start:
// Disable all interrupts
msr DAIFSet, #(DAIFSC_ALL)
// Setup return address (virtual)
adrp lr, __main@page
add lr, lr, __main@pageoff
// Set up exception vectors
adrp x10, _exception_vector_base@page
add x10, x10, _exception_vector_base@pageoff
msr VBAR_ELx, x10
// bzero stacks region
ldr x10, L_STACKS_BASE
ldr x11, L_STACKS_SIZE
add x11, x11, x10
mov x12, #0
L__stacks_zero_loop:
stp x12, x12, [x10], #16
cmp x10, x11
b.ne L__stacks_zero_loop
// bzero page tables region
ldr x10, L_PAGE_TABLES_BASE
ldr x11, L_PAGE_TABLES_SIZE
add x11, x11, x10
mov x12, #0
L__tables_zero_loop:
stp x12, x12, [x10], #16
cmp x10, x11
b.ne L__tables_zero_loop
// Set up stack pointers
ldr x10, L_STACKS_BASE
// exception stack goes at very bottom
add x10, x10, #(EXCEPTION_STACK_SIZE)
mov sp, x10 // set SPx to exception stack
// bootstrap stack above exception stack
msr SPSel, #0 // switch to SP0
add x10, x10, #(BOOTSTRAP_STACK_SIZE)
mov sp, x10 // set SP0 to bootstrap stack
// Save interrupt stack top, to be stored after bss is cleared
// interrupt stack goes at top so that it can share with bootstrap
// stack after bootstrap task exits
add x20, x10, #(INTERRUPT_STACK_SIZE)
// SecureROM: relocate data to SRAM
#if APPLICATION_SECUREROM
ldr x10, L_DATA_RO_START
mov x11, #(PAGE_SIZE - 1)
add x10, x10, x11
bic x10, x10, x11
ldr x11, L_DATA_BASE
cmp x10, x11
b.eq L__do_bss
ldr x12, L_DATA_END
L__relocate_data_loop:
ldp x13, x14, [x10], #16
stp x13, x14, [x11], #16
cmp x11, x12
b.ne L__relocate_data_loop
#endif // APPLICATION_SECUREROM
// Clear the bss section
// We requested all the sections to be aligned by 64.
// Section length should be 4 byte aligned.
L__do_bss:
ldr x10, L_BSS_START
ldr x11, L_BSS_END
mov x12, #15
bic x12, x11, x12
mov x13, #0
L__bss_loop_stp:
stp x13, x13, [x10], #16
cmp x10, x12
b.ne L__bss_loop_stp
cmp x11, x12
b.eq L__bss_done
L__bss_loop_word:
str w13, [x10], #4
cmp x10, x11
b.lt L__bss_loop_word
L__bss_done:
// Store interrupt stack top
adrp x11, _interrupt_stack_top@page
add x11, x11, _interrupt_stack_top@pageoff
str x20, [x11]
#if defined(HEAP_GUARD) && (HEAP_GUARD != SRAM_END)
// Clear the heap guard page, which will never be mapped
mov x13, #0
ldr x10, L_HEAP_GUARD
mov x11, #(PAGE_SIZE)
add x11, x10, x11
L__loop_heap_guard:
stp x13, x13, [x10], #16
stp x13, x13, [x10], #16
stp x13, x13, [x10], #16
stp x13, x13, [x10], #16
cmp x10, x11
b.ne L__loop_heap_guard
#endif
#ifdef BOOT_TRAMPOLINE_BASE
// copy the trampoline into a non-writeable page
ldr x3, L_boot_trampoline_start
ldr x4, L_boot_trampoline_end
ldr x5, L_boot_trampoline_dst
1:
ldp x6, x7, [x3], #16
stp x6, x7, [x5], #16
cmp x3, x4
b.lo 1b
#endif
// Branch to main
ret
.org 0x200
L__build_banner_string:
.ascii CONFIG_PROGNAME_STRING
.ascii " for "
.ascii CONFIG_BOARD_STRING
.ascii ", Copyright 2007-2016, Apple Inc."
.org 0x240
L__build_style_string:
.ascii BUILD_STYLE
/* This will get replaced by macho_post_process.py so that the build tag
is accurate even for iterative desktop builds that don't rebuild start.o */
.org 0x280
_build_tag_string_contents:
.org 0x300
.globl _build_banner_string
_build_banner_string:
.8byte L__build_banner_string
.balign 8
.globl _build_style_string
_build_style_string:
.8byte L__build_style_string
.balign 8
.globl _build_tag_string
_build_tag_string:
.8byte _build_tag_string_contents
.balign 8
// Linked start and end addresses of the text
L_TEXT_BASE:
.8byte (TEXT_BASE)
L_TEXT_END:
.8byte section$start$__DATA$__zerofill
L_TEXT_SIZE:
.8byte segment$end$__TEXT - segment$start$__TEXT
// start address of the data in the read only text */
L_DATA_RO_START:
.8byte segment$end$__TEXT
// Linked start and end addresses of the data
L_DATA_BASE:
.8byte segment$start$__DATA
L_DATA_END:
.8byte section$start$__DATA$__zerofill
// Linked start and end addresses of the zero-filled range
L_BSS_START:
.8byte section$start$__DATA$__zerofill
L_BSS_END:
.8byte segment$end$__DATA
L_STACKS_BASE:
.8byte (STACKS_BASE)
L_STACKS_SIZE:
.8byte (STACKS_SIZE)
L_PAGE_TABLES_BASE:
.8byte (PAGE_TABLES_BASE)
L_PAGE_TABLES_SIZE:
.8byte (PAGE_TABLES_SIZE)
#if defined(HEAP_GUARD) && (HEAP_GUARD != SRAM_END)
L_HEAP_GUARD:
.8byte (HEAP_GUARD)
#endif
#ifdef BOOT_TRAMPOLINE_BASE
L_boot_trampoline_start:
.8byte _boot_handoff_trampoline
L_boot_trampoline_end:
.8byte _boot_handoff_trampoline_end
L_boot_trampoline_dst:
.8byte BOOT_TRAMPOLINE_BASE
#endif
.data
.globl _interrupt_stack_top
_interrupt_stack_top:
.space 8
arch/arm64/task.c
+52
View File
@@ -0,0 +1,52 @@
/*
* Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
*
* This document is the property of Apple Computer, Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Computer, Inc.
*/
#include <debug.h>
#include <sys/types.h>
#include <sys.h>
#include <sys/task.h>
static void arch_task_trampoline(void) __noreturn;
static void
arch_task_trampoline(void)
{
/* exit the critical section we implicitly started with */
exit_critical_section();
dprintf(DEBUG_SPEW, "arch_task_trampoline: task %p (%s) routine %p, arg %p\n",
current_task, current_task->name, current_task->routine, current_task->arg);
/* call the start routine */
int ret = current_task->routine(current_task->arg);
dprintf(DEBUG_SPEW, "arch_task_trampoline: task %p (%s) exits with %d\n",
current_task, current_task->name, ret);
task_exit(ret);
/* never return */
}
int
arch_task_create(struct task *t)
{
int i;
for (i=0; i < 29; i++)
t->arch.regs[i] = 0;
/* set the default starting point to be arch_task_trampoline */
t->arch.lr = (uintptr_t)&arch_task_trampoline;
t->arch.sp = (uintptr_t)t->stack_base + t->stack_len;
t->arch.fp = 0;
return 0;
}
arch/arm64/test.mk
+56
View File
@@ -0,0 +1,56 @@
# Copyright (C) 2014 Apple Inc. All rights reserved.
#
# This document is the property of Apple Computer, Inc.
# It is considered confidential and proprietary.
#
# This document may not be reproduced or transmitted in any form,
# in whole or in part, without the express written permission of
# Apple Computer, Inc.
#
CC := $(shell xcodebuild -sdk macosx.internal -find clang)
GCOV := $(shell xcodebuild -sdk macosx.internal -find gcov)
TEST_DIR := host_tests
INCDIRS := . \
include \
../arm/include \
../../include
CFLAGS := -arch x86_64 \
-std=c11 \
-g -Wall -Werror -O \
$(addprefix -I,$(INCDIRS)) \
-DARCH_ARMv8 -DTEST
TESTS := mmu_test
TEST_PASSES := $(patsubst %,$(TEST_DIR)/%.pass,$(TESTS))
.PHONY: all
all: host-tests
.PHONY: host-tests
host-tests: $(TEST_PASSES)
.PHONY: clean
clean:
rm -rf $(TEST_DIR)
rm -f *.gcno *.gcda
$(TEST_DIR)/mmu_test.pass: $(TEST_DIR)/mmu_test
@echo "Test running $(notdir $<)"
@rm -f mmu_test.gcda mmu.gcda
@$< 2>&1 1>$(TEST_DIR)/mmu_test.out || \
{ cat $(TEST_DIR)/mmu_test.out; false; }
@echo "Test passed $(notdir $<)"
@$(GCOV) -a mmu.c
@mv -f mmu.c.gcov $(TEST_DIR)
@grep '#####' $(TEST_DIR)/mmu.c.gcov || true
@touch $@
$(TEST_DIR)/mmu_test: mmu_test.c mmu.c
@echo "Build $(notdir $@)"
@mkdir -p $(dir $@)
@$(CC) -o $@ $^ $(CFLAGS) --coverage
arch/arm64/tests-mmu.mk
+30
View File
@@ -0,0 +1,30 @@
# Copyright (C) 2014 Apple Inc. All rights reserved.
#
# This document is the property of Apple Inc.
# It is considered confidential and proprietary.
#
# This document may not be reproduced or transmitted in any form,
# in whole or in part, without the express written permission of
# Apple Inc.
#
LOCAL_DIR := $(GET_LOCAL_DIR)
TEST_NAME := arm64-mmu
TEST_OBJS := \
$(LOCAL_DIR)/mmu.o \
TEST_SUPPORT_OBJS := \
$(LOCAL_DIR)/mmu_test.o
TEST_CFLAGS := \
-DCPU_CACHELINE_SIZE=64 \
-DARCH_ARMv8=1 \
-DTEST=1 \
-Iarch/arm64/include/ \
-Iarch/arm/include
TEST_INCLUDES := \
arch/arm64/include/ \
arch/arm/include
arch/arm64/timer.c
+147
View File
@@ -0,0 +1,147 @@
/*
* Copyright (C) 2011 Apple Inc. All rights reserved.
*
* This document is the property of Apple Inc.
* It is considered confidential and proprietary.
*
* This document may not be reproduced or transmitted in any form,
* in whole or in part, without the express written permission of
* Apple Inc.
*/
#include <arch.h>
#include <debug.h>
#include <platform.h>
#include <platform/soc/hwclocks.h>
#include <platform/timer.h>
extern void arm_write_cntp_ctl(uint64_t);
extern uint64_t arm_read_cntp_ctl();
extern void arm_write_cntp_tval(uint64_t);
extern uint64_t arm_read_cntpct();
static void timer_deadline(void *arg);
static void (* timer_deadline_func)(void);
#define DECR_MAX_COUNT (INT32_MAX)
int timer_init(uint32_t timer)
{
uint64_t val;
val = (0<<0) | (1<<1);
arm_write_cntp_ctl(val);
return 0;
}
void timer_stop_all(void)
{
uint64_t val;
val = (0<<0) | (1<<1);
arm_write_cntp_ctl(val);
}
void timer_deadline_enter(uint64_t deadline, void (* func)(void))
{
uint64_t ticks;
uint64_t decr;
uint64_t val;
timer_deadline_func = func;
if (func) {
/* convert absolute deadline to relative time */
ticks = timer_get_ticks();
if (deadline < ticks) {
// printf(" %llu too soon (it is now %llu)\n", deadline, ticks);
deadline = 0;
} else {
deadline -= ticks;
}
/* clamp the deadline to our maximum */
decr = (deadline >= DECR_MAX_COUNT) ? DECR_MAX_COUNT : deadline;
// printf(" using decrementer count %u\n", decr);
/*
* Reset the decrementer by programming the max count, clearing
* any pending interrupt status, then programming the desired count.
* Note the timer's value is a signed 32 bit number
*/
arm_write_cntp_tval(DECR_MAX_COUNT);
val = (1<<0) | (0<<1);
arm_write_cntp_ctl(val);
arm_write_cntp_tval(decr);
} else {
/* no deadline, disable the decrementer */
val = (0<<0) | (1<<1);
arm_write_cntp_ctl(val);
}
}
void arm_fiq()
{
uint64_t val;
_irq_enter_critical_section();
val = arm_read_cntp_ctl();
if (val & (1 << 2)) {
/* acknowledge the interrupt and disable the timer */
val = (0<<0) | (1<<1);
arm_write_cntp_ctl(val);
/* if we have a callback, invoke it now */
if (timer_deadline_func)
timer_deadline_func();
}
_irq_exit_critical_section();
}
#ifdef OSC_FREQ
utime_t timer_ticks_to_usecs(uint64_t ticks)
{
#if (OSC_FREQ % 1000000) == 0
return ticks / (OSC_FREQ / 1000000);
#else
#error "The code below has overflow issues. Make sure you really want to use it"
return (ticks * 1000 * 1000) / OSC_FREQ;
#endif
}
uint64_t timer_usecs_to_ticks(utime_t usecs)
{
#if (OSC_FREQ % 1000000) == 0
return usecs * (OSC_FREQ / 1000000);
#else
#error "The code below has overflow issues. Make sure you really want to use it"
uint64_t timer_scale = ((uint64_t)(OSC_FREQ) << 20) / (1000 * 1000);
return (ticks * timer_scale) >> 20;
#endif
}
uint32_t timer_get_tick_rate(void)
{
return OSC_FREQ;
}
#endif
uint64_t timer_get_ticks(void)
{
return (arm_read_cntpct());
}
void arm_no_wfe_spin(uint32_t usecs)
{
uint64_t delta_time = (uint64_t)usecs * (OSC_FREQ / 1000000);
uint64_t start_time;
start_time = timer_get_ticks();
while (true) {
if ((timer_get_ticks() - start_time) > delta_time)
break;
}
}