first and last commit

arch/arm/asm.S

@@ -0,0 +1,627 @@
+/*
+ * Copyright (C) 2007-2011 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <arch/arm/assembler.h>
+	
+	.text
+
+#define ALL_ROUTINES 1
+
+// Define isb as a nop for older architectures
+#if !ARCH_ARMv7
+#define isb	nop
+#define dsb	nop
+#endif /* ! ARCH_ARMv7 */
+
+ARM_FUNCTION _arm_read_cpsr
+
+	mrs		r0, cpsr
+	bx		lr
+
+#if ALL_ROUTINES
+ARM_FUNCTION _arm_read_main_id
+
+	mrc		p15, 0, r0, c0, c0, 0
+	bx		lr
+#endif
+
+#if ALL_ROUTINES
+ARM_FUNCTION _arm_read_cache_id
+
+	mrc		p15, 0, r0, c0, c0, 1
+	bx		lr
+
+ARM_FUNCTION _arm_read_cache_level_id
+
+	mrc		p15, 1, r0, c0, c0, 1
+	bx		lr
+
+ARM_FUNCTION _arm_read_cache_size_selection
+
+	mrc		p15, 2, r0, c0, c0, 0
+	bx		lr
+
+ARM_FUNCTION _arm_write_cache_size_selection
+
+	mcr		p15, 2, r0, c0, c0, 0
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_read_cache_size_id
+
+	mrc		p15, 1, r0, c0, c0, 0
+	bx		lr
+#endif
+
+#if ALL_ROUTINES
+ARM_FUNCTION _arm_read_extended_feature_regs
+
+	mov		r12, r0
+	mrc		p15, 0, r0, c0, c1, 0
+	mrc		p15, 0, r1, c0, c1, 1
+	mrc		p15, 0, r2, c0, c1, 2
+	mrc		p15, 0, r3, c0, c1, 3
+	stmia		r12!, { r0-r3 }
+	mrc		p15, 0, r0, c0, c1, 4
+	mrc		p15, 0, r1, c0, c1, 5
+	mrc		p15, 0, r2, c0, c1, 6
+	mrc		p15, 0, r3, c0, c1, 7
+	stmia		r12!, { r0-r3 }
+	mrc		p15, 0, r0, c0, c2, 0
+	mrc		p15, 0, r1, c0, c2, 1
+	mrc		p15, 0, r2, c0, c2, 2
+	mrc		p15, 0, r3, c0, c2, 3
+	stmia		r12!, { r0-r3 }
+	mrc		p15, 0, r0, c0, c2, 4
+	mrc		p15, 0, r1, c0, c2, 5
+	mrc		p15, 0, r2, c0, c2, 6
+	mrc		p15, 0, r3, c0, c2, 7
+	stmia		r12, { r0-r3 }
+	bx		lr
+#endif
+
+#if ALL_ROUTINES
+ARM_FUNCTION _arm_read_memory_model_feature_regs
+
+	mov		r12, r0
+	mrc		p15, 0, r0, c0, c1, 4
+	mrc		p15, 0, r1, c0, c1, 5
+	mrc		p15, 0, r2, c0, c1, 6
+	mrc		p15, 0, r3, c0, c1, 7
+	stmia		r12, { r0-r3 }
+	bx		lr
+#endif
+
+#if ALL_ROUTINES
+ARM_FUNCTION _arm_read_instruction_set_attribute_regs
+
+	mov		r12, r0
+	mrc		p15, 0, r0, c0, c2, 0
+	mrc		p15, 0, r1, c0, c2, 1
+	mrc		p15, 0, r2, c0, c2, 2
+	mrc		p15, 0, r3, c0, c2, 3
+	stmia		r12!, { r0-r3 }
+	mrc		p15, 0, r0, c0, c2, 4
+	mrc		p15, 0, r1, c0, c2, 5
+	stmia		r12!, { r0-r1 }
+	bx		lr
+#endif
+
+ARM_FUNCTION _arm_read_cr
+
+	mrc		p15, 0, r0, c1, c0, 0
+	bx		lr
+
+ARM_FUNCTION _arm_write_cr
+
+	mcr		p15, 0, r0, c1, c0, 0
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_read_aux_cr
+
+	mrc		p15, 0, r0, c1, c0, 1
+	bx		lr
+	
+ARM_FUNCTION _arm_write_aux_cr
+
+	mcr		p15, 0, r0, c1, c0, 1
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_write_dar
+
+	mcr		p15, 0, r0, c3, c0, 0
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_write_ttb
+
+	mcr		p15, 0, r0, c2, c0, 0
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_write_ttbcr
+
+	mcr		p15, 0, r0, c2, c0, 2
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_read_ifsr
+
+	mrc		p15, 0, r0, c5, c0, 1
+	bx		lr
+	
+ARM_FUNCTION _arm_read_dfsr
+
+	mrc		p15, 0, r0, c5, c0, 0
+	bx		lr
+
+ARM_FUNCTION _arm_read_ifar
+
+#if ARCH_ARMv7
+	mrc		p15, 0, r0, c6, c0, 2
+	bx		lr
+#endif
+ARM_FUNCTION _arm_read_dfar
+
+	mrc		p15, 0, r0, c6, c0, 0
+	bx		lr
+	
+ARM_FUNCTION _arm_read_cp_access_cr
+
+	mrc		p15, 0, r0, c1, c0, 2
+	bx		lr
+
+ARM_FUNCTION _arm_write_cp_access_cr
+
+	mcr		p15, 0, r0, c1, c0, 2
+	isb
+	bx		lr
+
+#if WITH_VFP
+ARM_FUNCTION _arm_read_fpexc
+
+	vmrs		r0, fpexc
+	bx		lr
+
+ARM_FUNCTION _arm_write_fpexc
+
+	vmsr		fpexc, r0
+	bx		lr
+
+ARM_FUNCTION _arm_read_fpscr
+
+	vmrs		r0, fpscr
+	bx		lr
+
+ARM_FUNCTION _arm_write_fpscr
+
+	vmsr		fpscr, r0
+	bx		lr
+
+// Reset core registers to a known value. Kingfisher doesnt
+// do this on reset, and simulation traces will show 'x' 
+// uninitialized values otherwise. Prefer deterministic
+ARM_FUNCTION _arm_init_fp_regs
+
+	vmov.i64	q0, #0
+	vmov		q1, q0
+	vmov		q2, q0
+	vmov		q3, q0
+	vmov		q4, q0
+	vmov		q5, q0
+	vmov		q6, q0
+	vmov		q7, q0
+#if FP_REGISTER_COUNT > 16
+	vmov		q8, q0
+	vmov		q9, q0
+	vmov		q10, q0
+	vmov		q11, q0
+	vmov		q12, q0
+	vmov		q13, q0
+	vmov		q14, q0
+	vmov		q15, q0
+#endif
+	bx		lr
+
+#endif
+	
+#if ARCH_ARMv7
+ARM_FUNCTION _arm_read_l2_aux_cr
+
+	mrc		p15, 1, r0, c9, c0, 2
+	bx		lr
+	
+ARM_FUNCTION _arm_write_l2_aux_cr
+
+	mcr		p15, 1, r0, c9, c0, 2
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_read_pmreg
+
+	cmp		r0, #0
+	bne		1f
+	mrc		p15, 0, r0, c9, c12, 0
+	b		2f
+1:	cmp		r0, #1
+	bne		1f
+	mrc		p15, 0, r0, c9, c12, 1
+	b		2f
+1:	cmp		r0, #2
+	bne		1f
+	mrc		p15, 0, r0, c9, c12, 2
+	b		2f
+1:	cmp		r0, #3
+	bne		1f
+	mrc		p15, 0, r0, c9, c12, 3
+	b		2f
+1:	cmp		r0, #4
+	bne		1f
+	mrc		p15, 0, r0, c9, c12, 4
+	b		2f
+1:	cmp		r0, #5
+	bne		1f
+	mrc		p15, 0, r0, c9, c12, 5
+	b		2f
+1:	cmp		r0, #6
+	bne		1f
+	mrc		p15, 0, r0, c9, c13, 0
+	b		2f
+1:	cmp		r0, #7
+	bne		1f
+	mrc		p15, 0, r0, c9, c13, 1
+	b		2f
+1:	cmp		r0, #8
+	bne		2f
+	mrc		p15, 0, r0, c9, c13, 2
+	b		2f
+1:	cmp		r0, #9
+	bne		2f
+	mcr		p15, 0, r0, c9, c14, 0
+	b		2f
+1:	cmp		r0, #10
+	bne		2f
+	mcr		p15, 0, r0, c9, c14, 1
+	b		2f
+1:	cmp		r0, #11
+	bne		2f
+	mcr		p15, 0, r0, c9, c14, 2
+2:	isb
+	bx		lr
+
+ARM_FUNCTION _arm_write_pmreg
+
+	cmp		r0, #0
+	bne		1f
+	mcr		p15, 0, r1, c9, c12, 0
+	b		2f
+1:	cmp		r0, #1
+	bne		1f
+	mcr		p15, 0, r1, c9, c12, 1
+	b		2f
+1:	cmp		r0, #2
+	bne		1f
+	mcr		p15, 0, r1, c9, c12, 2
+	b		2f
+1:	cmp		r0, #3
+	bne		1f
+	mcr		p15, 0, r1, c9, c12, 3
+	b		2f
+1:	cmp		r0, #4
+	bne		1f
+	mcr		p15, 0, r1, c9, c12, 4
+	b		2f
+1:	cmp		r0, #5
+	bne		1f
+	mcr		p15, 0, r1, c9, c12, 5
+	b		2f
+1:	cmp		r0, #6
+	bne		1f
+	mcr		p15, 0, r1, c9, c13, 0
+	b		2f
+1:	cmp		r0, #7
+	bne		1f
+	mcr		p15, 0, r1, c9, c13, 1
+	b		2f
+1:	cmp		r0, #8
+	bne		2f
+	mcr		p15, 0, r1, c9, c13, 2
+	b		2f
+1:	cmp		r0, #9
+	bne		2f
+	mcr		p15, 0, r1, c9, c14, 0
+	b		2f
+1:	cmp		r0, #10
+	bne		2f
+	mcr		p15, 0, r1, c9, c14, 1
+	b		2f
+1:	cmp		r0, #11
+	bne		2f
+	mcr		p15, 0, r1, c9, c14, 2
+2:	isb
+	bx		lr
+#endif /* ARCH_ARMv7 */
+
+#if ALL_ROUTINES
+ARM_FUNCTION _arm_read_user_rw_tid
+
+	mrc		p15, 0, r0, c13, c0, 2
+	bx		lr
+
+ARM_FUNCTION _arm_write_user_rw_tid
+
+	mcr		p15, 0, r0, c13, c0, 2
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_read_user_ro_tid
+
+	mrc		p15, 0, r0, c13, c0, 3
+	bx		lr
+
+ARM_FUNCTION _arm_write_user_ro_tid
+
+	mcr		p15, 0, r0, c13, c0, 3
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_read_sup_tid
+
+	mrc		p15, 0, r0, c13, c0, 4
+	bx		lr
+
+ARM_FUNCTION _arm_write_sup_tid
+
+	mcr		p15, 0, r0, c13, c0, 4
+	isb
+	bx		lr
+#endif
+
+ARM_FUNCTION _arm_read_perip_port_remap
+
+	mrc		p15, 0, r0, c15, c2, 4
+	bx		lr
+
+ARM_FUNCTION _arm_write_perip_port_remap
+
+	mcr		p15, 0, r0, c15, c2, 4
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_flush_tlbs
+
+	mov		r0, #0
+	mcr		p15, 0, r0, c8, c7, 0
+	dsb
+	isb
+	bx		lr
+
+#define EXT(x) _ ## x
+
+#if ALL_ROUTINES
+/* mpu routines */
+#define WRITE_DATA_PROT_REGION(func, n) \
+	.global EXT(func); \
+EXT(func): \
+	mcr		p15, 0, r0, c6, c##n, 0; \
+	isb
+	bx		lr
+
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_0, 0)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_1, 1)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_2, 2)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_3, 3)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_4, 4)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_5, 5)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_6, 6)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_7, 7)
+WRITE_DATA_PROT_REGION(arm_write_dprot_region_8, 8)
+
+#define WRITE_INSTRUCTION_PROT_REGION(func, n) \
+	.global EXT(func); \
+EXT(func): \
+	mcr		p15, 0, r0, c6, c##n, 1; \
+	isb
+	bx		lr
+
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_0, 0)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_1, 1)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_2, 2)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_3, 3)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_4, 4)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_5, 5)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_6, 6)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_7, 7)
+WRITE_INSTRUCTION_PROT_REGION(arm_write_iprot_region_8, 8)
+
+	.global _arm_write_data_prot_register
+_arm_write_data_prot_register:
+	mcr		p15, 0, r0, c5, c0, 0
+	isb
+	bx		lr
+
+	.global _arm_write_ins_prot_register
+_arm_write_ins_prot_register:
+	mcr		p15, 0, r0, c5, c0, 1
+	isb
+	bx		lr
+
+	.global _arm_write_cacheable_registers
+_arm_write_cacheable_registers:
+	mcr		p15, 0, r0, c2, c0, 0
+	mcr		p15, 0, r1, c2, c0, 1
+	isb
+	bx		lr
+
+	.global _arm_write_bufferable_register
+_arm_write_bufferable_register:
+	mcr		p15, 0, r0, c3, c0, 0
+	isb
+	bx		lr
+#endif
+
+/* uint arch_enable_ints(void); */
+ARM_FUNCTION _arch_enable_ints
+
+	mrs		r0, cpsr
+	bic		r1, r0, #(1<<7)
+#if WITH_FIQ_TIMER
+	bic		r1, r1, #(1<<6)
+#endif		
+	msr		cpsr_c, r1
+	and		r0, r0, #(1<<7)
+	bx		lr
+
+/* uint arch_disable_ints(void); */
+ARM_FUNCTION _arch_disable_ints
+
+	mrs		r0, cpsr
+	orr		r1, r0, #(1<<7)
+#if WITH_FIQ_TIMER
+	orr		r1, r1, #(1<<6)
+#endif	
+	msr		cpsr_c, r1
+	and		r0, r0, #(1<<7)
+	bx		lr
+
+/* uint arch_restore_ints(uint state); */
+ARM_FUNCTION _arch_restore_ints
+
+	mrs		r1, cpsr
+	and		r0, r0, #(1<<7)
+	bic		r1, r1, #(1<<7)
+	orr		r2, r1, r0
+	msr		cpsr_c, r2
+	and		r0, r1, #(1<<7)
+	bx		lr
+
+/* void arm_enable_fiqs(void); */
+ARM_FUNCTION _arm_enable_fiqs
+
+	mrs		r0, cpsr
+	bic		r0, r0, #(1<<6)
+	msr		cpsr_c, r0
+	bx		lr
+
+/* void arm_disable_fiqs(void); */
+ARM_FUNCTION _arm_disable_fiqs
+
+	mrs		r0, cpsr
+	orr		r0, r0, #(1<<6)
+	msr		cpsr_c, r0
+	bx		lr
+
+#if ARCH_ARMv7
+ARM_FUNCTION _arm_read_vbar
+
+	mrc		p15, 0, r0, c12, c0, 0
+	bx		lr
+	
+ARM_FUNCTION _arm_write_vbar
+
+	mcr		p15,0, r0, c12, c0, 0
+	bx		lr
+
+#endif
+	
+#if ARCH_ARMv7
+ARM_FUNCTION _arm_flush_branch_predictor
+
+	mov		r0, #0
+	mcr		p15, 0, r0, c7, c5, 6				/* flush the branch predictor cache */
+	isb
+	mov		pc, lr
+#endif
+
+/* void arm_memory_barrier(void); */
+ARM_FUNCTION _arm_memory_barrier
+
+#if ARCH_ARMv7 
+	dmb
+#endif
+	bx		lr
+
+#ifdef PRODUCT_EMBEDDEDIOP
+/*  <rdar://problem/16290859> WFI errata workaround for Cortex-A5 */
+	dummy_store:
+	.long 		0xaaaabbbb
+
+	.balign		32
+#endif //PRODUCT_EMBEDDEDIOP
+
+ARM_FUNCTION _arch_halt
+
+#if !NO_ARM_HALT
+# if ARCH_ARMv7
+	dsb
+#ifdef PRODUCT_EMBEDDEDIOP
+	str		r0, dummy_store
+#endif //PRODUCT_EMBEDDEDIOP
+	wfi
+# else
+	mov		r0, #0
+	mcr		p15, 0, r0, c7, c0, 4				/* wait for interrupt */
+# endif
+#endif
+	bx		lr
+
+ARM_FUNCTION _arch_spin
+
+#if ARCH_ARMv7 || ARCH_ARMv8
+	wfe
+#endif
+	b		_arch_spin
+
+#if WITH_ARM_DCC
+ARM_FUNCTION _read_dtr
+
+	mrc		p14, 0, r0, c0, c5, 0
+	bx		lr
+
+ARM_FUNCTION _write_dtr
+
+	mcr		p14, 0, r0, c0, c5, 0
+	isb
+	bx		lr
+
+ARM_FUNCTION _read_dscr
+
+	mrc		p14, 0, r0, c0, c1, 0
+	bx		lr
+#endif /* WITH_ARM_DCC */
+
+
+#if CPU_APPLE_SWIFT
+ARM_FUNCTION _swift_read_l2cerrsts
+
+	mrc		p15, 1, r0, c15, c2, 5
+	bx		lr
+
+ARM_FUNCTION _swift_write_l2cerrsts
+
+	mcr		p15, 1, r0, c15, c2, 5
+	isb
+	bx		lr
+
+ARM_FUNCTION _swift_read_l2cerradr
+
+	mrc		p15, 1, r0, c15, c2, 7
+	bx		lr
+
+ARM_FUNCTION _swift_write_l2cerradr
+
+	mcr		p15, 1, r0, c15, c2, 7
+	isb
+	bx		lr
+#endif /* CPU_APPLE_SWIFT */

arch/arm/cache_v7.S

@@ -0,0 +1,320 @@
+/*
+ * Copyright (C) 2007-2010 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <arch/arm/assembler.h>
+
+	.text
+
+	/*
+	 * Cache functions for ARM v7
+	 */
+
+	/* perform loop unrolling to optimize for multi-line operations rdar://problem/6722268 */
+ARM_FUNCTION _arm_clean_dcache_line_8
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line_7
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line_6
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line_5
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line_4
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line_3
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line_2
+
+	mcr		p15, 0, r0, c7, c10, 1      // clean L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_dcache_line
+
+	mcr		p15, 0, r0, c7, c10, 1		// clean L1/L2 line by MVA
+	dmb
+	bx		lr
+
+	/* perform loop unrolling to optimize for multi-line operations rdar://problem/6722268 */
+ARM_FUNCTION _arm_invalidate_dcache_line_8
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line_7
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line_6
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line_5
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line_4
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line_3
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line_2
+
+	mcr		p15, 0, r0, c7, c6, 1		  // invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_invalidate_dcache_line
+
+	mcr		p15, 0, r0, c7, c6, 1		// invalidate L1/L2 line by MVA
+	dmb
+	bx		lr
+
+	/* perform loop unrolling to optimize for multi-line operations rdar://problem/6722268 */
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_8
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_7
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_6
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_5
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_4
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_3
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line_2
+
+	mcr		p15, 0, r0, c7, c14, 1		 // clean/invalidate L1/L2 line by MVA
+	add		r0, r0, #(CPU_CACHELINE_SIZE)
+
+ARM_FUNCTION _arm_clean_invalidate_dcache_line
+
+	mcr		p15, 0, r0, c7, c14, 1		// clean/invalidate L1/L2 line by MVA
+	dmb
+	bx		lr
+	
+	
+ARM_FUNCTION _arm_clean_dcache
+
+	mov		r2, #((1 << CPU_CACHEINDEX_SHIFT) << CPU_CACHELINE_SHIFT)
+1:
+	sub		r2, r2, #(1 << CPU_CACHELINE_SHIFT)
+	mov		r1, #((1 << CPU_CACHESET_SHIFT) << (32-CPU_CACHESET_SHIFT))
+2:
+	subs		r1, r1, #(1 << (32-CPU_CACHESET_SHIFT))
+	orr		r0, r1, r2
+	mcr		p15, 0, r0, c7, c10, 2		// clean one line by way and set
+	bne		2b
+	cmp		r2, #0
+	bne		1b
+
+#if L2_CACHE_SIZE
+	mov		r2, #((1 << L2_CACHE_SETS) << 6)
+1:
+	sub		r2, r2, #(1 << 6)
+	mov		r1, #((1 << 3) << 29)
+2:
+	subs		r1, r1, #(1 << 29)
+	orr		r0, r1, r2
+	orr		r0, r0, #(1 << 1)
+	mcr		p15, 0, r0, c7, c10, 2		// clean one line by way and set
+	bne		2b
+	cmp		r2, #0
+	bne		1b
+#endif
+
+        mov		r0, #0
+	dmb
+	bx		lr
+
+ARM_FUNCTION _arm_clean_invalidate_dcache
+
+	mov		r2, #((1 << CPU_CACHEINDEX_SHIFT) << CPU_CACHELINE_SHIFT)
+1:
+	sub		r2, r2, #(1 << CPU_CACHELINE_SHIFT)
+	mov		r1, #((1 << CPU_CACHESET_SHIFT) << (32-CPU_CACHESET_SHIFT))
+2:
+	subs		r1, r1, #(1 << (32-CPU_CACHESET_SHIFT))
+	orr		r0, r1, r2
+	mcr		p15, 0, r0, c7, c14, 2		// clean and invalidate one line by way and set
+	bne		2b
+	cmp		r2, #0
+	bne		1b
+
+#if L2_CACHE_SIZE
+	mov		r2, #((1 << L2_CACHE_SETS) << 6)
+1:
+	sub		r2, r2, #(1 << 6)
+	mov		r1, #((1 << 3) << 29)
+2:
+	subs		r1, r1, #(1 << 29)
+	orr		r0, r1, r2
+	orr		r0, r0, #(1 << 1)
+	mcr		p15, 0, r0, c7, c14, 2		// clean and invalidate one line by way and set
+	bne		2b
+	cmp		r2, #0
+	bne		1b
+#endif
+
+        mov		r0, #0
+	dmb
+	bx		lr
+
+ARM_FUNCTION _arm_invalidate_dcache
+
+	// This is a fancy way of writing "0". Thr first sub will make this (1<<CPU_CACHESET_SHIFT)-1
+	mov		r1, #((1 << CPU_CACHESET_SHIFT) << (32-CPU_CACHESET_SHIFT))
+	mov		r2, #((1 << CPU_CACHEINDEX_SHIFT) << CPU_CACHELINE_SHIFT)
+1:
+	sub		r1, r1, #(1 << (32-CPU_CACHESET_SHIFT))
+	orr		r0, r1, r2
+2:
+#if (1 << CPU_CACHEINDEX_SHIFT) < 16
+#error "Not enough cache sets for unroll in _arm_invalidate_dcache"
+#endif
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	sub		r0, r0, #(1 << CPU_CACHELINE_SHIFT)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	cmp		r0, r1
+	bne		2b
+	cmp		r1, #0
+	bne		1b
+
+#if L2_CACHE_SIZE
+	mov		r2, #((1 << L2_CACHE_SETS) << 6)
+1:
+	sub		r2, r2, #(1 << 6)
+	mov		r1, #((1 << 3) << 29)
+2:
+	subs		r1, r1, #(1 << 29)
+	orr		r0, r1, r2
+	orr		r0, r0, #(1 << 1)
+	mcr		p15, 0, r0, c7, c6, 2		// invalidate one line by way and set
+	bne		2b
+	cmp		r2, #0
+	bne		1b
+#endif
+
+	dmb
+	bx		lr
+
+ARM_FUNCTION _arm_invalidate_icache
+
+	mov		r0, #0
+	mcr		p15, 0, r0, c7, c5, 0		/* invalidate icache + btb (if supported) */
+	dsb
+	isb
+	bx		lr
+
+ARM_FUNCTION _arm_clean_caches
+
+	stmfd		sp!, {lr}
+
+	bl		_arm_clean_dcache
+	bl		_arm_invalidate_icache
+
+	ldmfd		sp!, {lr}
+	bx		lr
+
+#if WITH_L1_PARITY
+ARM_FUNCTION _arm_enable_l1parity
+
+	mrc		p15, 0, r0, c1, c0, 1
+#if CPU_ARM_CORTEX_A8
+	orr		r0, r0, #(1 << 3)		// set the L1PE bit
+#elif	CPU_ARM_CORTEX_A5
+	orr		r0, r0, #(1 << 9)		// set the "parity on" bit
+#endif
+	mcr		p15, 0, r0, c1, c0, 1
+	isb
+	bx		lr
+#endif
+
+ARM_FUNCTION _arm_drain_write_buffer
+
+	mov		r0, #0
+	dsb
+	isb
+	bx		lr
+
+

arch/arm/cachedebug_a8.c

@@ -0,0 +1,83 @@
+
+#include <debug.h>
+#include <sys/menu.h>
+#include <arch/arm/arm.h>
+
+void arm_l1cache_dump(int dcache, unsigned int mva)
+{
+	register unsigned index = (mva >> 6) & 0x7f;
+	unsigned way, addr, word, data0, data1;
+	unsigned tag, i, parity, tmp;
+	unsigned lookup;
+	unsigned cr = arm_read_cr();
+
+	arm_write_cr(cr & ~((1 << 12) | (1 << 2)));
+	lookup = (mva != -1UL);
+	for (way=0; way<4; way++) {
+		if (!lookup) index = 0;
+		do {
+			addr = (way << 30) | (index << 6);
+			if (dcache) {
+				__asm__ volatile("mcr p15,0,%0,c15,c2,6" ::"r"(addr));
+				__asm__ volatile("mrc p15,0,%0,c15,c0,0" :"=r"(tag));
+			} else {
+				__asm__ volatile("mcr p15,0,%0,c15,c3,6" ::"r"(addr));
+				__asm__ volatile("mrc p15,0,%0,c15,c1,0" :"=r"(tag));
+			}
+			tag <<= 12;
+			for (word = 0; word < 16; word++) {
+				addr = (way << 30) | (index << 6) | (word << 2);
+				if (dcache) {
+					__asm__ volatile("mcr p15,0,%0,c15,c2,7" ::"r"(addr));
+					__asm__ volatile("mrc p15,0,%0,c15,c0,0" :"=r"(data0));
+					__asm__ volatile("mrc p15,0,%0,c15,c0,1" :"=r"(data1));
+				} else {
+					__asm__ volatile("mcr p15,0,%0,c15,c3,7" ::"r"(addr));
+					__asm__ volatile("mrc p15,0,%0,c15,c1,0" :"=r"(data0));
+					__asm__ volatile("mrc p15,0,%0,c15,c1,1" :"=r"(data1));
+				}
+				if (word == 0)
+					printf("PA %08x: %c ", tag | ((index & 0x3f) << 6),
+					       dcache && (data1&1) ? 'D':' ');
+				parity = 0;
+				tmp = data0;
+				for (i=0; i<8; i++) {
+					parity ^= tmp & 0x01010101;
+					tmp >>= 1;
+				}
+				parity &= 0x01010101;
+				parity = (parity >> 21) | (parity >> 14) | (parity >> 7) | parity;
+				if ((parity ^ (data1 >> 2)) & 0xf)
+					printf(" >%08x-%x<", data0, (data1>>2)&0xf);
+				else
+					printf(" %08x", data0);
+			}
+			printf("\n");
+		} while (!lookup && (++index < 128));
+	}
+	arm_clean_dcache();
+	arm_write_cr(cr);
+}
+
+void dump_l1caches(int disable)
+{
+	unsigned cr;
+
+	cr = arm_read_cr();
+
+	if (disable) {
+		arm_write_cr(cr & ~((1<<12)|(1<<2)));
+		arm_clean_dcache();
+	}
+
+	arm_l1cache_dump(0, -1);
+	arm_l1cache_dump(1, -1);
+
+	if (disable)
+		arm_write_cr(cr);
+}
+
+#if defined(WITH_MENU) && WITH_MENU
+static int do_l1dump(int argc, struct cmd_arg *args) { dump_l1caches(1); return 0; }
+MENU_COMMAND_DEVELOPMENT(l1dump, do_l1dump, "dump L1 i/d contents", NULL);
+#endif

arch/arm/context.S

@@ -0,0 +1,164 @@
+/*
+ * Copyright (c) 2009 Apple Inc.  All rights reserved.	
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <arch/arm/arm.h>
+#include <arch/arch_task.h>
+#include <arch/arm/assembler.h>
+
+/*
+ * Task context switch support.
+ * 
+ * Basic VFP/Neon floating point context switch only.
+ */
+	
+/*
+ * Code here must remain synchronised with the structure and offsets defined in arch_task.h:
+ *
+ * struct arch_task {
+ * 	uint32_t regs[10]; // r4-r11, r13, r14
+ * #if WITH_VFP
+ * 	uint32_t	fpscr;
+ * 	uint32_t	fpexc;
+ * 	uint32_t	fpinst;
+ * 	uint32_t	fpinst2;
+ * 	uint64_t	fpregs[32];	// assume VFP-D32
+ * #endif
+ * };
+*/
+
+/*
+ * void arch_task_context_switch(struct arch_task *old, struct arch_task *new_task);
+ */
+
+ARM_FUNCTION _arch_task_context_switch
+	
+	/* save non-volatile core registers */
+	stmia	r0, { r4-r11, r13, r14 }
+
+#if WITH_VFP
+#if !WITH_VFP_ALWAYS_ON
+	/* save fpexc */
+	vmrs	r2, fpexc
+	str	r2, [r0, #ARM_ARCH_TASK_FPEXC] 
+
+	/* check FP enable state */
+	tst	r2, #FPEXC_EN
+	beq	L_fp_save_done
+
+	/* save fpscr */
+	vmrs	r4, fpscr
+	str	r4, [r0, #ARM_ARCH_TASK_FPSCR]
+#endif
+	
+	/* save the FP registers */
+	add	r3, r0, #ARM_ARCH_TASK_FPREGS
+	vstm.64	r3!, { d0-d15 }
+#if FP_REGISTER_COUNT > 16
+	vstm.64	r3, { d16-d31 }
+#endif
+L_fp_save_done:	
+#endif	/* WITH_VFP */
+	
+	/* select the context we're going to switch to and invoke the context restore */
+	mov	r0, r1
+	b	_arch_task_context_restore
+
+	
+/*
+ * void arch_task_context_restore(struct arch_task *restore_context)
+ */
+
+ARM_FUNCTION _arch_task_context_restore
+
+#if WITH_VFP
+#if !WITH_VFP_ALWAYS_ON
+	/* restore fpexc */
+	ldr	r2, [r0, #ARM_ARCH_TASK_FPEXC]
+	vmsr	fpexc, r2
+
+	/* check FP enable state */
+	tst	r2, #FPEXC_EN
+	beq	L_fp_restore_done
+
+	/* restore fpscr */
+	ldr	r4, [r0, #ARM_ARCH_TASK_FPSCR]
+	vmsr	fpscr, r4
+#endif
+	
+	/* restore the FP registers */
+	add	r3, r0, #ARM_ARCH_TASK_FPREGS
+	vldm.64	r3!, { d0-d15 }
+#if FP_REGISTER_COUNT > 16
+	vldm.64	r3, { d16-d31 }
+#endif
+L_fp_restore_done:
+#endif	/* WITH_VFP */
+
+	/* restore non-volatile core registers */
+	ldmia	r0, { r4-r11, r13, r14 }
+	bx	lr
+
+
+#if WITH_VFP && !WITH_VFP_ALWAYS_ON
+/*
+ * void arm_call_fpsaved(void *arg, void (*func)(void *arg))
+ */
+	.globl _arm_call_fpsaved
+_arm_call_fpsaved:
+	stmfd           sp!, {lr}
+
+	/* fetch fpexc and check FP enable state */
+	vmrs	r2, fpexc
+	tst	r2, #FPEXC_EN
+	beq	1f		/* bypass save if VFP is off */
+
+	/* save fpscr */
+	vmrs	r3, fpscr
+	push	{r3}
+	
+	/* save the FP registers */
+#if FP_REGISTER_COUNT > 16
+	vpush	{ d16-d31 }
+#endif
+	vpush	{ d0-d15 }
+1:
+	/* stack fpexc last, always guranteed to be on the stack */
+	push	{r2}
+	
+	/* unconditionally enable VFP for this call */
+	orr	r2, r2, #FPEXC_EN
+	vmsr	fpexc, r2
+	
+	/* branch to the routine */
+	blx	r1
+
+	/* pop fpexc and test whether FP was saved */
+	pop	{r2}
+	tst	r2, #FPEXC_EN
+	beq	2f
+
+	/* restore the FP registers */
+	vpop	{ d0-d15 }
+#if FP_REGISTER_COUNT > 16
+	vpop	{ d16-d31 }
+#endif
+
+	/* restore fpcsr */
+	pop	{r3}
+	vmsr	fpscr, r2
+2:
+	/* restore fpexc last, may disable VFP */
+	vmsr	fpexc, r2
+
+	ldmfd		sp!, {lr}
+	bx		lr
+#endif /* WITH_VFP */

arch/arm/cortex_debug.c

@@ -0,0 +1,60 @@
+/*
+ * Copyright (C) 2009 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch/arm/arm.h>
+#include <lib/libc.h>
+#include <sys.h>
+#include <sys/menu.h>
+
+#if defined(WITH_MENU) && WITH_MENU
+
+extern char cortex_hang_start, cortex_hang_ffe, cortex_hang_end;
+typedef uint32_t (*hang_func_t)(uint32_t);
+static uint8_t *hang_buffer = NULL;
+static hang_func_t cortex_hang;
+
+static int do_hang(int argc, struct cmd_arg *args)
+{
+	u_int32_t count = 10000;
+	utime_t before, after;
+	int i, ret, always = 0;
+	char *entry_point;
+
+	if (argc > 1)
+		count = args[1].u;
+	if (!strcmp(args[0].str, "hangloop"))
+		always = 1;
+
+	if (hang_buffer == NULL) {
+		hang_buffer = malloc(8192);
+		entry_point = (char *)((((uint32_t)hang_buffer + 0xfff) & ~0xfff) -
+				       (&cortex_hang_ffe + 2 - &cortex_hang_start));
+		memcpy(entry_point, &cortex_hang_start, &cortex_hang_end - &cortex_hang_start);
+		arm_clean_dcache();
+		arm_invalidate_icache();
+		cortex_hang = (hang_func_t)(entry_point+1); /* Thumb */
+		printf("Created hang code buffer; base = %p, entry = %p\n", hang_buffer, cortex_hang);
+	}
+
+	for (i=0; (i<100) || always; i++) {
+		printf("Try %3d: ", i+1);
+		before = system_time();
+		ret = cortex_hang(count);
+		after = system_time();
+		printf("complete; time = %6d\n", after-before);
+		if (ret != 0) printf("   EARLY EXIT!\n");
+	}
+	return 0;
+}
+
+MENU_COMMAND_DEBUG(hang, do_hang, "try to hang cortex", NULL);
+MENU_COMMAND_DEBUG(hangloop, do_hang, "try to hang cortex - run forever", NULL);
+
+#endif

arch/arm/cortex_hang.S

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2009 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+	.text
+	.global _cortex_hang_start, _cortex_hang_ffe, _cortex_hang_end
+	.thumb
+	.syntax unified
+
+// Believe it or not, without this, it does not fail
+#define NOP	mov r8,r8
+
+	// R0 holds the number of iterations we should run
+	// start this routine at a page boundary
+_cortex_hang_start:
+	// save some registers and set aside a word
+	push	{r4,r5,r6,r7,lr}
+	sub	sp, #4
+1:
+	cmp	r0, #0
+	beq	1f
+	NOP			// 0xff0
+	NOP			// 0xff2
+	NOP			// 0xff4
+	NOP			// 0xff6
+	subs	r0, #1		// 0xff8
+	str.w	ip, [sp]	// 0xffa
+_cortex_hang_ffe:
+	bl	1b		// 0xffe
+1:
+	add	sp, #4
+	pop	{r4,r5,r6,r7,pc}
+	bx	lr
+_cortex_hang_end:

arch/arm/cpu.c

@@ -0,0 +1,276 @@
+/*
+ * Copyright (C) 2007-2013 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch.h>
+#include <arch/arm/arm.h>
+#include <debug.h>
+#include <platform.h>
+#include <platform/memmap.h>
+
+#if PERIPH_PORT_REMAP
+static void config_periph_port_remap(void *base, size_t len);
+#endif
+
+#if WITH_EARLY_ICACHE && WITH_L1_PARITY
+#error Early i-cache init and L1 parity enable are mutually exclusive
+#endif
+
+int arch_cpu_init(bool resume)
+{
+	u_int32_t cr, aux_cr;
+
+	aux_cr = 0;
+
+#if WITH_CACHE
+	/* make sure the cpu caches are invalidated */
+#if !WITH_EARLY_ICACHE
+    /* d-cache init is deferred if we're turning i-cache on early */
+	arm_invalidate_dcache();
+#endif
+	arm_invalidate_icache();
+#endif
+
+#if ARCH_ARMv7
+	aux_cr = arm_read_aux_cr();
+
+#if CPU_ARM_CORTEX_A8
+	// Ask platform if WFI should be enabled or not
+	if (platform_get_core_idle_enable()) {
+		aux_cr &= ~(1 << 8);
+	} else {
+		aux_cr |= (1 << 8);
+	}
+#endif
+
+#if WITH_ARCHITECTED_L2
+	/* Manage the v7 L2 cache */
+# if WITH_AUX_L2_ENABLE_BIT && L2_CACHE_SIZE
+	aux_cr |= (1 << 1);
+# else
+	aux_cr &= ~(1 << 1);
+# endif
+#endif
+
+#if SET_AUX_SMP_BIT
+	/* Set SMP bit; enable snooping within cluster and sharing of L2 */
+	aux_cr |= (1 << 6);
+#endif
+
+	arm_write_aux_cr(aux_cr);
+#endif
+
+	/* read control register default value */
+	cr = arm_read_cr();
+
+#if WITH_BRANCH_PREDICTION
+	/* turn on branch predictor */
+	cr |= (1 << 11);
+#endif
+
+#if WITH_CACHE && WITH_EARLY_ICACHE
+	/* early i-cache enable */
+	cr |= (1 << 12);
+	
+	arm_write_cr(cr);
+#endif
+
+#if PERIPH_PORT_REMAP
+	/* do any peripheral port remapping */
+	config_periph_port_remap((void *)PERIPH_BASE, PERIPH_LEN);
+#endif
+
+#if WITH_CACHE
+
+#if WITH_EARLY_ICACHE
+    /* d-cache invalidation delayed until this point */
+	arm_invalidate_dcache();
+#else
+	/* turn on i-cache */
+	cr |= (1 << 12);
+#endif
+
+	/* turn on d-cache */
+	cr |= (1 << 2);
+	
+#if WITH_L1_PARITY
+	arm_enable_l1parity();
+#endif
+#endif
+
+#if WITH_UNALIGNED_MEM
+	/* enable unaligned access handler */
+	cr &= ~(1 << 1);
+#if !ARCH_ARMv7
+	cr |= (1 << 22);
+#endif
+#endif
+
+	/* enable the caches and etc. before setting up the translation table */
+	arm_write_cr(cr);
+
+#if WITH_MMU
+	/* set up the translation table and then turn on the MMU */
+	arm_mmu_init(resume);
+	cr |= (1 << 0);
+
+	arm_write_cr(cr);
+#endif
+
+#if WITH_VFP
+	/* initialize VFP */
+	arm_fp_init();
+#if WITH_VFP_ALWAYS_ON	
+	arm_fp_enable(true);
+#endif	
+#endif
+
+	return 0;
+}
+
+int arch_cpu_quiesce(void)
+{
+	u_int32_t cr, aux_cr;
+
+	aux_cr = 0;
+
+#if ARCH_ARMv7
+	aux_cr = arm_read_aux_cr();
+
+#if CPU_ARM_CORTEX_A8
+	/* turn WFI on */
+	aux_cr &= ~(1 << 8);
+#endif
+
+	arm_write_aux_cr(aux_cr);
+#endif
+
+#if WITH_CACHE && !CLEAN_INV_CACHE_ON_QUIESCE
+	/* clean the d-cache */
+	arm_clean_dcache();
+#endif
+	
+	/* read the control regiser base value */
+	cr = arm_read_cr();
+
+#if WITH_MMU
+	/* turn off the mmu */
+	cr &= ~(1 << 0);
+#endif
+#if WITH_CACHE	
+	/* turn off d-cache */
+	cr &= ~(1 << 2);
+
+	/* turn off i-cache */
+	cr &= ~(1 << 12);
+#endif
+
+#if WITH_BRANCH_PREDICTION
+        /* turn off branch predictor */
+        cr &= ~(1 << 11);
+#endif
+
+#if WITH_UNALIGNED_MEM
+        /* disable unaligned access handler */
+        cr |= (1 << 1);
+        cr &= ~(1 << 22);
+#endif
+
+	/* disable the MMU, caches and etc. */
+	arm_write_cr(cr);
+	
+#if WITH_CACHE && CLEAN_INV_CACHE_ON_QUIESCE
+	/* clean and/or invalidate the caches */
+	arm_clean_invalidate_dcache();
+	arm_invalidate_icache();
+#endif		
+
+#if CPU_APPLE_SWIFT
+
+#if !CLEAN_INV_CACHE_ON_QUIESCE
+	/* Invalidate all lines in the L1-D cache */
+	arm_invalidate_dcache();
+#endif
+
+	/* Clear any pending L2C errors */
+	swift_write_l2cerrsts(0);
+	swift_write_l2cerradr(0);
+#endif /* CPU_APPLE_SWIFT */
+
+	return 0;
+}
+
+int arch_cpu_init_posttasks(void)
+{
+#if WITH_ARM_DCC
+	/* initialize the DCC mechanism */
+	arm_dcc_init();
+#endif
+
+	return 0;
+}
+
+u_int32_t arch_get_noncached_address(u_int32_t address)
+{
+#if WITH_MMU
+	return arm_get_noncached_address(address);
+#else
+	return address; /* not really any other choice... */
+#endif
+}
+
+#if PERIPH_PORT_REMAP
+static void config_periph_port_remap(void *base, size_t len)
+{
+	uint32_t val = (uint32_t)base;
+
+	switch (len) {
+		case 0: val |= 0x0; break;
+		case 0x1000: val |= 0x3; break;
+		case 0x2000: val |= 0x4; break;
+		case 0x4000: val |= 0x5; break;
+		case 0x8000: val |= 0x6; break;
+		case 0x10000: val |= 0x7; break;
+		case 0x20000: val |= 0x8; break;
+		case 0x40000: val |= 0x9; break;
+		case 0x80000: val |= 0xa; break;
+		case 0x100000: val |= 0xb; break;
+		case 0x200000: val |= 0xc; break;
+		case 0x400000: val |= 0xd; break;
+		case 0x800000: val |= 0xe; break;
+		case 0x1000000: val |= 0xf; break;
+		case 0x2000000: val |= 0x10; break;
+		case 0x4000000: val |= 0x11; break;
+		case 0x8000000: val |= 0x12; break;
+		case 0x10000000: val |= 0x13; break;
+		case 0x20000000: val |= 0x14; break;
+		case 0x40000000: val |= 0x15; break;
+		case 0x80000000: val |= 0x16; break;
+	}
+	
+	arm_write_perip_port_remap(val);
+}
+#endif
+
+static bool cpu_setup_mmu(bool resume)
+{
+#if WITH_MMU
+	arm_mmu_init(resume);
+	return true;
+#endif
+
+#if WITH_MPU
+	arm_mpu_init();
+	return true;
+#endif
+
+	return false;
+}
+

arch/arm/dcc/dcc.c

@@ -0,0 +1,108 @@
+/*
+ * Copyright (C) 2007 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch.h>
+#include <debug.h>
+#include <sys.h>
+#include <sys/task.h>
+#include <lib/cbuf.h>
+
+#define DSCR_WDTR_FULL  (1L<<29)
+#define DSCR_RDTR_FULL  (1L<<30)
+
+#if ARM_DCC_SYNCHRONOUS
+static bool dcc_wait_tx = true;
+#else
+static bool dcc_wait_tx;
+#endif
+
+#ifndef DCC_TX_BUFFER_SIZE
+# define DCC_TX_BUFFER_SIZE 256
+#endif
+
+static struct cbuf *dcc_output_cbuf;
+
+extern u_int32_t read_dtr(void);
+extern void write_dtr(u_int32_t c);
+extern u_int32_t read_dscr(void);
+
+void arm_write_dcc_char(int c)
+{
+	if (dcc_wait_tx) {
+		while (read_dscr() & DSCR_WDTR_FULL);
+		write_dtr(c);
+	} else {
+		if (dcc_output_cbuf != 0)
+		   	cbuf_write_char(dcc_output_cbuf, c);
+	}
+}
+
+int arm_read_dcc_char(void)
+{
+	while (!(read_dscr() & DSCR_RDTR_FULL)); 
+
+	return read_dtr(); 
+}
+
+int arm_read_dcc_char_nowait(void)
+{
+	if (!(read_dscr() & DSCR_RDTR_FULL))
+		return -1;
+
+	return read_dtr();
+}
+
+static int dcc_read_task(void *arg)
+{
+	for(;;) {
+		int c;
+
+		c = arm_read_dcc_char_nowait();
+		if (c >= 0) {
+			if (c != '\r')
+				debug_pushchar(c);
+		}
+		task_sleep(1 * 1000);
+	}
+	return 0;
+}
+
+static int dcc_write_task(void *arg)
+{
+	for(;;) {
+		ssize_t ret;
+		char c;
+
+		ret = cbuf_read_char(dcc_output_cbuf, &c);
+		if (ret == 1) {
+   			while(read_dscr() & DSCR_WDTR_FULL)
+				 task_yield();
+		    write_dtr(c);
+		}
+		task_sleep(1 * 1000);
+	}
+	return 0;
+}
+
+int arm_dcc_init(void)
+{
+	if (arm_read_dcc_char_nowait() >= 0)
+	   	dcc_wait_tx = true;
+
+	if (!dcc_wait_tx) {
+		dcc_output_cbuf = cbuf_create(DCC_TX_BUFFER_SIZE, NULL);
+		task_start(task_create("dcc write", &dcc_write_task, NULL, 256));
+	}
+
+	task_start(task_create("dcc read", &dcc_read_task, NULL, 512));
+
+	return 0;
+}

arch/arm/dcc/rules.mk

@@ -0,0 +1,19 @@
+# Copyright (C) 2007 Apple Inc. All rights reserved.
+#
+# This document is the property of Apple Inc.
+# It is considered confidential and proprietary.
+#
+# This document may not be reproduced or transmitted in any form,
+# in whole or in part, without the express written permission of
+# Apple Inc.
+#
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+OPTIONS += \
+	WITH_ARM_DCC=1
+
+MODULES	+= \
+	lib/cbuf
+
+ALL_OBJS += \
+	$(LOCAL_DIR)/dcc.o

arch/arm/entropy.S

@@ -0,0 +1,39 @@
+/*
+ * Copyright (C) 2010 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch/arm/assembler.h>
+
+	.text
+
+//	u_int32_t arch_get_entropy(volatile u_int32_t *counter_address)
+//
+//	arch_get_entropy exploits the uncertainty in the asynchronous boundary crossing
+//	between the CPU and a counter.  To work correctly, the CPU and counter must be
+//	derived from clocks with relative jitter (PLL vs. OSC).  Ideally the inner loop,
+//	up to 32 iterations, should consume a large part of the effective read period of
+//	the counter.
+//
+
+ARM_FUNCTION _arch_get_entropy
+
+	mov	r1, r0
+	mov	r0, #0
+	mov	r2, #32
+1:
+	mov	r3, r2
+2:
+	subs	r3, r3, #1
+	bne	2b
+	ldr	r3, [r1]
+	and	r3, r3, #1
+	orr	r0, r3, r0, lsl #1
+	subs	r2, r2, #1
+	bne	1b
+	bx	lr

arch/arm/exceptions.S

@@ -0,0 +1,195 @@
+/*
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ * Copyright (C) 2007-2013 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+
+#include <arch/arm/arm.h>
+#include <arch/arm/assembler.h>
+	
+/*
+ * struct arm_exception_frame {
+ *	uint32_t sp;
+ *	uint32_t lr;
+ * 	uint32_t spsr;
+ * 	uint32_t r[13];
+ * 	uint32_t pc;
+ *	uint64_t fpr[32];
+ *	uint32_t fpscr;
+ *	uint32_t fpexc;
+ * };
+ *
+ * Note that when returning to the mode in which the exception was
+ * taken that set the I and F bits to mask interrupts and clear the
+ * T bit so that we stay in ARM mode.
+ */
+
+#if VFP_REV >= 3
+
+#define VFP_SAVE			\
+	vstmdb.64   sp!, {d0-d3}    ;	\
+	vstmdb.64   sp!, {d4-d7}    ;	\
+	vstmdb.64   sp!, {d8-d11}   ;	\
+	vstmdb.64   sp!, {d12-d15}  ;	\
+	vstmdb.64   sp!, {d16-d19}  ;	\
+	vstmdb.64   sp!, {d20-d23}  ;	\
+	vstmdb.64   sp!, {d24-d27}  ;	\
+	vstmdb.64   sp!, {d28-d31}  ;	\
+	fmrx        r1, fpscr	    ;	\
+	stmfd	    sp!, { r1 }     ;	\
+	fmrx        r1, fpexc       ;	\
+	stmfd	    sp!, { r1 }
+	
+
+#define VFP_LOAD			\
+	ldmfd       sp!, { r1 }     ;	\
+	fmxr        fpexc, r1       ;	\
+	ldmfd       sp!, { r1 }     ;	\
+	vldmia.64   sp!, {d28-d31}  ;	\
+	vldmia.64   sp!, {d24-d27}  ;	\
+	vldmia.64   sp!, {d20-d23}  ;	\
+	vldmia.64   sp!, {d16-d19}  ;	\
+	vldmia.64   sp!, {d12-d15}  ;	\
+	vldmia.64   sp!, {d8-d11}   ;	\
+	vldmia.64   sp!, {d4-d7}    ;	\
+	vldmia.64   sp!, {d0-d3}    ;	\
+	fmxr        fpscr, r1
+
+#else
+
+#define VFP_SAVE
+#define VFP_LOAD
+
+#endif	
+
+#define PUSH_EXCEPTION_FRAME		\
+	stmfd	sp!, { r0-r12, lr } ;	\
+	mrs	r0, spsr	    ;	\
+	stmfd	sp!, { r0 }	    ;	\
+	orr	r0, r0, #0xc0	    ;	\
+	bic	r0, r0, #(1 << 5)   ;	\
+	mrs	r1, cpsr	    ;	\
+	msr	cpsr_c, r0	    ;	\
+	mov	r2, sp		    ;	\
+	mov	r3, lr		    ;	\
+	msr	cpsr_c, r1	    ;	\
+	stmfd	sp!, { r2-r3 }
+
+#define POP_EXCEPTION_FRAME		\
+	add	sp, sp, #12         ;	\
+	ldmfd	sp!, { r0-r12, pc }^
+
+
+	.text
+/*
+ * Interrupt handling.
+ *
+ * Note that FIQ is not intended to be "fast", but rather an NMI.
+ */
+
+ARM_FUNCTION ___arm_irq
+
+	sub	lr, lr, #4		/* fix up return address */
+
+	PUSH_EXCEPTION_FRAME
+	VFP_SAVE
+
+#if ARCH_ARMv7
+	mrs	r1, cpsr
+	bic	r1, #0x100		/* re-enable imprecise data aborts */
+	msr	cpsr_x, r1
+#endif
+
+	bl	_arm_irq		/* call handler */
+
+	VFP_LOAD
+	POP_EXCEPTION_FRAME
+
+
+ARM_FUNCTION	___arm_fiq
+
+	sub	lr, lr, #4		/* fix up return address */
+
+	/*
+	 * We can't use PUSH_EXCEPTION_FRAME here as we need to save banked registers,
+	 * but the only difference should be the re-saving of r8-r12 in the saved mode.
+	 */
+	stmfd	sp!, { r0-r12, lr }	/* save unbanked registers */
+	mrs	r0, spsr		/* save spsr */
+	stmfd	sp!, { r0 }
+	add	r4, sp, #36		/* address of r8 in save frame */
+	orr	r0, r0, #0xc0		/* mask interrupts and swap to previous mode */
+	mrs	r1, cpsr
+	msr	cpsr_c, r0
+	stmia	r4, { r8-r12 }		/* save banked registers */
+	mov	r2, sp			/* grab sp & lr,  swap back and save them */
+	mov	r3, lr
+	msr	cpsr_c, r1
+#if ARCH_ARMv7
+	bic	r1, #0x100		/* re-enable imprecise data aborts */
+	msr	cpsr_x, r1
+#endif
+	stmfd	sp!, { r2-r3 }
+
+	VFP_SAVE
+	
+	bl	_arm_fiq		/* call handler */
+
+	VFP_LOAD
+	POP_EXCEPTION_FRAME
+
+
+	
+/*
+ * Exception handling.
+ *
+ * In each case, the handler is expected to populate the
+ * exception frame structure and call the high-level dispatch
+ * routine.  Note that we don't support resuming from any
+ * exceptions other than IRQ/FIQ.
+ *
+ */
+
+ARM_FUNCTION ___arm_undefined
+
+	PUSH_EXCEPTION_FRAME
+
+	mov	r0, sp
+	bl	_arm_undefined
+	b	.
+
+ARM_FUNCTION ___arm_syscall
+
+	PUSH_EXCEPTION_FRAME
+
+	mov	r0, sp
+	bl	_arm_syscall
+	b	.
+
+ARM_FUNCTION ___arm_prefetch_abort
+
+	sub	lr, lr, #4
+
+	PUSH_EXCEPTION_FRAME
+
+	mov	r0, sp
+	bl	_arm_prefetch_abort
+	b	.
+
+ARM_FUNCTION ___arm_data_abort
+
+	PUSH_EXCEPTION_FRAME
+
+	mov	r0, sp
+	bl	_arm_data_abort
+	b	.
+
+ARM_FUNCTION ___arm_reserved
+
+	b	.

arch/arm/fp.c

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2009-2015 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+#include <arch.h>
+#include <arch/arm/arm.h>
+#include <debug.h>
+#include <sys/task.h>
+
+/*
+ * VFP/Neon support
+ */
+
+#if WITH_VFP
+
+void
+arm_fp_init(void)
+{
+	u_int32_t	cr;
+
+	cr = arm_read_cp_access_cr();
+	cr |= CPACR_CP_BITS(10, CPACR_CP_FULL) | CPACR_CP_BITS(11, CPACR_CP_FULL);
+	arm_write_cp_access_cr(cr);
+}
+
+#if WITH_VFP_ALWAYS_ON
+void
+arm_fp_enable(bool enable)
+{
+	arm_write_fpexc(FPEXC_EN);
+
+	/* reset VFP/NEON state */
+	arm_write_fpscr(FPSCR_DEFAULT);
+
+	/* zero init VFP/NEON registers */
+	arm_init_fp_regs();
+}
+#endif /* WITH_VFP_ALWAYS_ON */
+
+bool
+arch_task_fp_enable(bool enable)
+{
+	bool			current;
+	u_int32_t		cr;
+	struct task		*t;
+	struct arch_task	*at;
+
+	/* get the current FP enable state */
+	t = task_get_current_task();
+	at = &t->arch;
+	current = (at->fpexc & FPEXC_EN);
+
+	if (enable != current) {
+		
+		/* fetch current hardware state */
+		cr = arm_read_fpexc();
+		if (enable) {
+			/* enable VFP/NEON */
+			at->fpexc = FPEXC_EN;
+			arm_write_fpexc(at->fpexc);
+
+			/* reset VFP/NEON state */
+			at->fpscr = 0;
+			arm_write_fpscr(FPSCR_DEFAULT);
+
+		} else {
+			/* disable VFP/NEON */
+			at->fpexc = cr & ~FPEXC_EN;
+			arm_write_fpexc(at->fpexc);
+		}
+
+		return(!enable);
+	}
+	return(enable);
+}
+
+#endif /* WITH_VFP */

arch/arm/handlers.c

@@ -0,0 +1,560 @@
+/*
+ * Copyright (C) 2007-2010 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <debug.h>
+#include <arch.h>
+#include <arch/arch_task.h>
+#include <arch/arm/arm.h>
+#include <sys.h>
+#include <lib/libc.h>
+
+extern void platform_irq(void);
+extern void platform_fiq(void);
+
+static const char *arm_abort_decode_status(unsigned int dfsr);
+static const char *arm_decode_mode(unsigned int status_reg);
+static void arm_exception_abort(struct arm_exception_frame *frame, 
+				const char *kind, 
+				unsigned int fsr, 
+				unsigned int far) __noreturn;
+
+static struct arm_exception_frame *arm_get_exception_frame(void) __pure;
+static struct arm_exception_frame *arm_get_interrupt_frame(void) __pure;
+static struct arm_exception_frame *arm_get_fiq_frame(void) __pure;
+static void arm_print_frame_registers(struct arm_exception_frame *frame, const char *prefix);
+static void arm_backtrace(const char *prefix, u_int32_t pc, u_int32_t fp, u_int32_t stack_base, u_int32_t stack_len);
+static void arm_stack_for_status_reg(u_int32_t status_reg, u_int32_t *stack_base, u_int32_t *stack_len);
+
+static bool exception_dump_near_pc;
+
+#if WITH_MMU
+static char status_str[32];
+
+static const char *
+arm_abort_decode_status(unsigned int fsr)
+{
+	/* mask bit 10 & low 4 bits for cause decoding */
+	fsr &= FSR_CAUSE_MASK;
+
+	/* ARMv4v5 architecture reference B4-20 */
+	switch (fsr) {
+	case 0x01:
+	case 0x03:
+		return("alignment error");
+	case 0x04:
+		return("icache operation error");
+	case 0x0c:
+	case 0x0e:
+		return("external abort on translation");
+	case 0x05:
+	case 0x07:
+		return("translation error");
+	case 0x09:
+	case 0x0b:
+		return("domain violation");
+	case 0x0d:
+	case 0x0f:
+		return("permission violation");
+	case 0x08:
+	case 0x0a:		/* this code is deprecated in ARMv7*/
+		return("precise external abort");
+	case 0x406:
+		return("imprecise external abort");		/* ARMv7 */
+	case 0x40a:
+		return("coprocessor data abort");
+	case 0x408:
+		return("data cache parity error");		/* ARMv7 */
+	case 0x409:
+		return("instruction cache parity error");	/* ARMv7 */
+	case 0x02:
+		return("debug event");
+	}
+
+	snprintf(status_str, sizeof(status_str), "<unknown cause 0x%03x>", fsr);
+
+	return(status_str);
+}
+#endif
+
+static const char *
+arm_decode_mode(unsigned int status_reg)
+{
+	switch (CPSR_MODE(status_reg)) {
+	case CPSR_MODE_USER:
+		return("user");
+	case CPSR_MODE_FIQ:
+		return("FIQ");
+	case CPSR_MODE_IRQ:
+		return("IRQ");
+	case CPSR_MODE_SUPERVISOR:
+		return("supervisor");
+	case CPSR_MODE_ABORT:
+		return("abort");
+	case CPSR_MODE_UNDEFINED:
+		return("undefined");
+	case CPSR_MODE_SYSTEM:
+		return("system");
+	}
+	return("<unknown>");
+}
+
+void
+arm_data_abort(struct arm_exception_frame *frame)
+{
+	unsigned int fsr = arm_read_dfsr();
+	unsigned int far = arm_read_dfar();
+
+	frame->pc -= 8;
+	if (fsr == 0x408) {
+		arm_l1cache_dump(1, far);
+	}
+	arm_exception_abort(frame, "data abort", fsr, far);
+}
+
+void
+arm_prefetch_abort(struct arm_exception_frame *frame)
+{
+	unsigned int fsr = arm_read_ifsr();
+	unsigned int far = arm_read_ifar();
+
+	frame->pc -= 4;
+	if (fsr == 0x409) {
+		arm_l1cache_dump(0, far);
+	}
+	arm_exception_abort(frame, "prefetch abort", fsr, far);
+}
+
+void
+arm_undefined(struct arm_exception_frame *frame)
+{
+	exception_dump_near_pc = true;
+	frame->pc -= (frame->spsr & CPSR_STATE_THUMB) ? 2: 4;
+	arm_exception_abort(frame, "undefined instruction", ~0, frame->pc);
+}
+
+void
+arm_syscall(struct arm_exception_frame *frame)
+{
+	exception_dump_near_pc = true;
+	frame->pc -= (frame->spsr & CPSR_STATE_THUMB) ? 2: 4;
+	arm_exception_abort(frame, "syscall", arm_read_ifsr(), frame->pc);
+}
+
+static void
+arm_exception_abort(struct arm_exception_frame *frame, const char *kind, unsigned int fsr, unsigned int far)
+{
+#if WITH_MMU
+	panic("ARM %s abort in %s mode at 0x%08x due to %s:\n"
+	      " far 0x%08x fsr 0x%08x\n"
+	      " r0 0x%08x 0x%08x 0x%08x 0x%08x\n"
+	      " r4 0x%08x 0x%08x 0x%08x 0x%08x\n"
+	      " r8 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n"
+	      " sp 0x%08x lr 0x%08x spsr 0x%08x\n",
+	      kind, arm_decode_mode(frame->spsr), frame->pc, arm_abort_decode_status(fsr), far, fsr,
+	      frame->r[0], frame->r[1], frame->r[2], frame->r[3],
+	      frame->r[4], frame->r[5], frame->r[6], frame->r[7],
+	      frame->r[8], frame->r[9], frame->r[10], frame->r[11], frame->r[12], 
+	      frame->sp, frame->lr, frame->spsr);
+#else
+	panic("ARM %s abort in %s mode at 0x%08x\n"
+	      " r0 0x%08x 0x%08x 0x%08x 0x%08x\n"
+	      " r4 0x%08x 0x%08x 0x%08x 0x%08x\n"
+	      " r8 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n"
+	      " sp 0x%08x lr 0x%08x spsr 0x%08x\n",
+	      kind, arm_decode_mode(frame->spsr), frame->pc,
+	      frame->r[0], frame->r[1], frame->r[2], frame->r[3],
+	      frame->r[4], frame->r[5], frame->r[6], frame->r[7],
+	      frame->r[8], frame->r[9], frame->r[10], frame->r[11], frame->r[12], 
+	      frame->sp, frame->lr, frame->spsr);
+#endif
+}
+
+void
+arm_irq() /* __attribute__ ((naked)) */
+{
+#if WITH_FIQ_TIMER
+	// Currently only defined on M7, M7 uses a FIQ for thetimer, and we need to hold off timer interrupts
+	// until we are done running ISR code to prevent unwanted preemption (e.g. <rdar://problem/16757466>)
+	arm_disable_fiqs();
+#endif	
+	_irq_enter_critical_section();
+	platform_irq();
+	_irq_exit_critical_section();
+#if WITH_FIQ_TIMER	
+	arm_enable_fiqs();
+#endif
+}
+
+void 
+arm_fiq() /* __attribute__ ((naked)) */
+{
+	_irq_enter_critical_section();
+	platform_fiq();
+	_irq_exit_critical_section();
+}
+
+/*
+ * Accessors for exception frame information.
+ *
+ * These may return NULL if we don't believe the frame requested is currently valid.
+ *
+ * Note that not all possible cases of nesting are necessarily handled here; only those
+ * that are likely to be interesting in the case of a fatality.
+ */
+static struct arm_exception_frame *
+arm_get_exception_frame(void)
+{
+	/*
+	 * As exceptions are considered fatal, we don't expect to be asked for
+	 * the exception frame in any other context.
+	 */
+	switch (CPSR_MODE(arm_read_cpsr())) {
+	case CPSR_MODE_ABORT:
+	case CPSR_MODE_UNDEFINED:
+		return((struct arm_exception_frame *)exc_stack_top - 1);
+	default:
+		return(NULL);
+	}
+}
+
+static struct arm_exception_frame *
+arm_get_interrupt_frame(void)
+{
+	switch (CPSR_MODE(arm_read_cpsr())) {
+	case CPSR_MODE_ABORT:
+	case CPSR_MODE_UNDEFINED:
+		/*
+		 * In some configurations the IRQ and exception stacks share a top,
+		 * in which case we can't return the IRQ frame because the exception
+		 * frame has overwritten it.
+		 */
+		if (&irq_stack_top == &exc_stack_top)
+			return(NULL);
+
+		/*
+		 * If we took an exception while servicing an interrupt, the IRQ frame 
+		 * will be valid.
+		 */
+		if (CPSR_MODE_IRQ != arm_get_exception_frame()->spsr)
+			return(NULL);
+		break;
+
+	case CPSR_MODE_IRQ:
+		/* we're in interrupt mode, so the frame is valid */
+		break;
+
+	case CPSR_MODE_FIQ:
+		/* 
+		 * If we took a FIQ while in interrupt mode, the IRQ frame will be
+		 * valid.
+		 */
+		if (CPSR_MODE_IRQ != CPSR_MODE(arm_get_fiq_frame()->spsr))
+			return(NULL);
+		break;
+
+	default:
+		/* the interrupt frame is invalid in other modes */
+		return(NULL);
+	}
+	return ((struct arm_exception_frame *)irq_stack_top - 1);
+}
+
+static struct arm_exception_frame *
+arm_get_fiq_frame(void)
+{
+	switch (CPSR_MODE(arm_read_cpsr())) {
+	case CPSR_MODE_ABORT:
+	case CPSR_MODE_UNDEFINED:
+		/*
+		 * If we took an exception while servicing a FIQ, the frame 
+		 * will be valid.
+		 */
+		if (CPSR_MODE_FIQ != CPSR_MODE(arm_get_exception_frame()->spsr))
+			return(NULL);
+		break;
+
+	case CPSR_MODE_FIQ:
+		/* we're in FIQ mode, so the frame is valid */
+		break;
+		
+	default:
+		/* the FIQ frame is invalid in other modes */
+		return(NULL);
+	}
+	return((struct arm_exception_frame *)fiq_stack_top - 1);
+}
+
+/*
+ * arm_print_frame_registers
+ *
+ * Print the registers from an exception frame.
+ */
+static void
+arm_print_frame_registers(struct arm_exception_frame *frame, const char *prefix)
+{
+	dprintf(DEBUG_CRITICAL,
+		"%sr0 0x%08x 0x%08x 0x%08x 0x%08x\n"
+		"%sr4 0x%08x 0x%08x 0x%08x 0x%08x\n"
+		"%sr8 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n"
+		"%spc 0x%08x sp 0x%08x lr 0x%08x spsr 0x%08x\n",
+		prefix, frame->r[0], frame->r[1], frame->r[2], frame->r[3],
+		prefix, frame->r[4], frame->r[5], frame->r[6], frame->r[7],
+		prefix, frame->r[8], frame->r[9], frame->r[10], frame->r[11], frame->r[12], 
+		prefix, frame->pc, frame->sp, frame->lr, frame->spsr);
+}
+
+/*
+ * arm_backtrace
+ *
+ * Print a backtrace until we run out of frames on the stack.
+ */
+static void
+arm_backtrace(const char *prefix, u_int32_t pc, u_int32_t fp, u_int32_t stack_base, u_int32_t stack_len)
+{
+	u_int32_t	prev_fp;
+	u_int32_t	stack_top;
+	u_int32_t	*fpp;
+
+	/* print the mode prefix and current pc for starters */
+	dprintf(DEBUG_CRITICAL, "    %10s 0x%08x 0x%08x\n", prefix, pc, fp);
+
+	/* sanity check our arguments as much as we can */
+	if ((stack_base % 4) || 	/* bump these to CPU_CACHELINE_SIZE if/when stack allocation gets smarter */
+	    (stack_len % 4) || 
+	    (stack_len < 16) ||
+	    (stack_len > 100 * 1024)) {
+		dprintf(DEBUG_CRITICAL, "improbable stack, base %p size 0x%08x (task smashed?)\n", (void *)stack_base, stack_len);
+		return;
+	}
+
+	/* loop unwinding frames until we leave the stack region */
+	prev_fp = stack_base;
+	stack_top = stack_base + stack_len;
+	for (;;) {
+		/*
+		 * Frame pointer must be aligned, must be above the previous frame, must not point outside the stack.
+		 *
+		 * It would be nice to have a sentinel to allow differentiating clean termination from a smashed
+		 * stack, but that would require support in the interrupt and exception handlers.
+		 */
+		if ((fp % 4) || (fp <= prev_fp) || (fp > stack_top))
+			return;
+
+		/*
+		 * Looks like it's inside the stack, so go ahead and deref it.
+		 */
+		fpp = (u_int32_t *)fp;
+		prev_fp = fp;
+		fp = fpp[0];
+
+		/*
+		 * Print the return address for this frame.
+		 */
+		dprintf(DEBUG_CRITICAL, "               0x%08x 0x%08x\n", fpp[1], fp);
+	}	
+}
+
+/*
+ * arm_stack_for_status_reg
+ *
+ * Returns the base and size of the stack for the given status register value if it's "special".
+ */
+static void
+arm_stack_for_status_reg(u_int32_t status_reg, u_int32_t *stack_base, u_int32_t *stack_len)
+{
+	*stack_base = 0;
+	*stack_len = 0;
+
+#ifdef STACK_BASE
+	*stack_base = (void *)STACK_BASE;
+	switch (CPSR_MODE(status_reg)) {
+	case CPSR_MODE_FIQ:
+		*stack_len = FIQ_STACK_SIZE;
+		break;
+	case CPSR_MODE_IRQ:
+		*stack_len = IRQ_STACK_SIZE;
+		break;
+	case CPSR_MODE_ABORT:
+	case CPSR_MODE_UNDEFINED:
+		*stack_len = EXCEPTION_STACK_SIZE;
+		break;
+
+	}
+#else
+	switch (CPSR_MODE(status_reg)) {
+	case CPSR_MODE_FIQ:
+		*stack_base = (u_int32_t)fiq_stack_top - FIQ_STACK_SIZE;
+		*stack_len = FIQ_STACK_SIZE;
+		break;
+	case CPSR_MODE_IRQ:
+		*stack_base = (u_int32_t)irq_stack_top - IRQ_STACK_SIZE;
+		*stack_len = IRQ_STACK_SIZE;
+		break;
+	case CPSR_MODE_ABORT:
+	case CPSR_MODE_UNDEFINED:
+		*stack_base = (u_int32_t)exc_stack_top - EXCEPTION_STACK_SIZE;
+		*stack_len = EXCEPTION_STACK_SIZE;
+		break;
+	}
+#endif
+}
+
+/*
+ * arch_backtrace_current_task
+ *
+ * Print a backtrace for the current task.
+ *
+ * We support being called with the following context nesting:
+ *
+ * supervisor
+ * supervisor interrupt
+ * supervisor FIQ
+ * supervisor exception
+ * supervisor interrupt FIQ
+ * supervisor interrupt exception
+ * supervisor FIQ exception
+ * supervisor interrupt FIQ exception
+ *
+ */
+
+void
+arch_backtrace_current_task(void *stack_base, size_t stack_len)
+{
+	struct arm_exception_frame	*exception_frame, *fiq_frame, *interrupt_frame, *outer_frame;
+	u_int32_t			esb, esl;
+	u_int32_t			base, count, *insn, offending_pc;
+
+	uintptr_t			btext = (uintptr_t)__segment_start(__TEXT);
+	uintptr_t			etext = (uintptr_t)__segment_end(__TEXT);
+
+	/*
+	 * Find the stack limits for the current context and then backtrace it.
+	 */
+	arm_stack_for_status_reg(arm_read_cpsr(), &esb, &esl);
+	if (0 == esl) {
+		esb = (u_int32_t)stack_base;
+		esl = stack_len;
+	}
+	arm_backtrace(arm_decode_mode(arm_read_cpsr()), 
+		      (u_int32_t)&arch_backtrace_current_task, (u_int32_t)__builtin_frame_address(0), 
+		      esb, esl);
+
+	/*
+	 * Find valid saved contexts, for the case where we may be in an exception state.
+	 */
+	exception_frame = arm_get_exception_frame();
+	fiq_frame = arm_get_fiq_frame();
+	interrupt_frame = arm_get_interrupt_frame();
+	outer_frame = NULL;
+
+	/*
+	 * If we have a valid exception context, backtrace the context it was called from first.
+	 * That context may be supervisor, FIQ or interrupt mode.  If it was FIQ mode, we will
+	 * dump the context the FIQ was called from next.
+	 */
+	if (NULL != exception_frame) {
+
+		/* dump registers at entry to exception mode */
+		arm_print_frame_registers(exception_frame, "               ");
+
+		/*
+		 * If the exception handler thought that the data around the PC was worth
+		 * displaying, and we think the PC was inside the text section, display it here.
+		 */
+		if (true == exception_dump_near_pc) {
+			offending_pc = exception_frame->pc;
+			if ((offending_pc >= btext) && (offending_pc <= etext)) {
+				if ((offending_pc - btext) < 8) {
+					base = btext;
+				} else {
+					base = offending_pc - 8;
+				}
+				dprintf(DEBUG_CRITICAL, "               [0x%08x:", base);
+				for (count = 0; count < 4; count++) {
+					insn = (u_int32_t *)base + count;
+					dprintf(DEBUG_CRITICAL, " 0x%08x", *insn);
+				}
+				dprintf(DEBUG_CRITICAL, "]\n");
+			} else {
+				dprintf(DEBUG_CRITICAL, "               [pc 0x%08x not in text %p-%p]\n",
+					offending_pc, (void *)btext, (void *)etext);
+			}
+		}
+
+		/*
+		 * If the exception occurred in a context with a special stack (FIQ or interrupt)
+		 * backtrace that stack here.
+		 */
+		arm_stack_for_status_reg(exception_frame->spsr, &esb, &esl);
+		if (0 != esl) {
+			arm_backtrace(arm_decode_mode(exception_frame->spsr),
+				      exception_frame->pc, exception_frame->r[7],
+				      esb, esl);
+		} else {
+			/* the outer context is in the exception frame */
+			outer_frame = exception_frame;
+		}
+	}
+
+	/*
+	 * If we have a valid FIQ context, backtrace the context it was invoked from now.
+	 */
+	if (NULL != fiq_frame) {
+		
+		/* dump registers at entry to FIQ mode */
+		arm_print_frame_registers(fiq_frame, "               ");
+
+		/*
+		 * If the FIQ occurred from interrupt mode, backtrace the interrupt mode stack here.
+		 */
+		arm_stack_for_status_reg(fiq_frame->spsr, &esb, &esl);
+		if (0 != esl) {
+			arm_backtrace(arm_decode_mode(fiq_frame->spsr),
+				      fiq_frame->pc, fiq_frame->r[7],
+				      esb, esl);
+		} else {
+			/* the outer context is in the FIQ frame */
+			outer_frame = fiq_frame;
+		}
+	}
+
+	/*
+	 * If we have an interrupt-saved context, fetch the task state from it.
+	 */
+	if (NULL != interrupt_frame) {
+
+		/* dump registers at entry to interrupt mode */
+		arm_print_frame_registers(interrupt_frame, "               ");
+
+		outer_frame = interrupt_frame;
+	}
+
+	/*
+	 * Print the task context backtrace from the outer frame.
+	 */
+	if (NULL != outer_frame) {
+		arm_backtrace(arm_decode_mode(outer_frame->spsr), 
+			      outer_frame->pc, outer_frame->r[7],
+			      (u_int32_t)stack_base, (u_int32_t)stack_len);
+	}
+}
+
+/*
+ * arch_backtrace_task
+ *
+ * Print a backtrace from a saved task context.
+ */
+void
+arch_backtrace_task(struct arch_task *arch_task, void *stack_base, size_t stack_len)
+{
+	arm_backtrace("supervisor", arch_task->regs[ARM_ARCH_TASK_LR], arch_task->regs[ARM_ARCH_TASK_FP], 
+		      (u_int32_t)stack_base, (u_int32_t)stack_len);
+}

arch/arm/include/arch/arch_task.h

@@ -0,0 +1,82 @@
+/*
+ * Copyright (C) 2006-2011 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#ifndef __ARCH_TASK_H
+#define __ARCH_TASK_H
+
+/*
+ * Note that this structure is known implicitly by code in context.S, and
+ * must be kept in sync.
+ */
+#if defined(__arm__)
+#ifndef __ASSEMBLY__
+struct arch_task {
+	/*
+	 * Core registers.
+	 */
+	uint32_t	regs[10]; 	// r4-r11, r13, r14
+
+#if WITH_VFP
+	/*
+	 * VFP/Neon registers.
+	 *
+	 * Note that layout must be kept in sync with offset defines below.
+	 */
+	uint32_t	fpexc;
+	uint32_t	fpscr;
+	uint64_t	fpregs[32];
+#endif
+};
+#endif /* __ASSEMBLY__ */
+
+/*
+ * Indices for useful core registers.
+ */
+#define ARM_ARCH_TASK_FP	3
+#define ARM_ARCH_TASK_SP	8
+#define ARM_ARCH_TASK_LR	9
+
+/*
+ * VFP/Neon field offsets - it would be nice to generate these automatically.
+ */
+#define ARM_ARCH_TASK_FP_OFFSET		40
+#define ARM_ARCH_TASK_FPEXC		(ARM_ARCH_TASK_FP_OFFSET + 0)
+#define ARM_ARCH_TASK_FPSCR		(ARM_ARCH_TASK_FP_OFFSET + 4)
+#define ARM_ARCH_TASK_FPREGS		(ARM_ARCH_TASK_FP_OFFSET + 8)
+
+
+#elif defined(__arm64__)
+
+#ifndef __ASSEMBLY__
+
+typedef __uint128_t uint128_t;
+
+struct arch_task {
+	/* Core registers */
+	uint64_t		regs[29];	// x0-x28
+	uint64_t		fp;
+	uint64_t		lr;
+	uint64_t		sp;	
+#if WITH_VFP		
+	/* Neon registers */
+	union {
+		uint128_t	q;
+		uint64_t	d;
+		uint32_t	s;
+	} vregs[32];				// v0-v31
+	uint32_t		fpsr;
+	uint32_t		fpcr;
+#endif	
+};
+#endif /* __ASSEMBLY__ */
+
+#endif /* __arm64__ */
+
+#endif /* ! __ARCH_TASK_H */

arch/arm/include/arch/arm/arm.h

@@ -0,0 +1,320 @@
+/*
+ * Copyright (C) 2007-2011 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#ifndef __ARM_H
+#define __ARM_H
+
+#ifndef __ASSEMBLER__
+
+#include <compiler.h>
+#include <non_posix_types.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+__BEGIN_DECLS
+
+unsigned int arm_read_cpsr(void) __pure;
+
+unsigned int arm_read_main_id(void) __pure;
+unsigned int arm_read_cache_id(void) __pure;
+unsigned int arm_read_cache_level_id(void) __pure;
+unsigned int arm_read_cache_size_selection(void) __pure;
+void arm_write_cache_size_selection(unsigned int);
+unsigned int arm_read_cache_size_id(void) __pure;
+
+void arm_read_memory_model_feature_regs(unsigned int regs[4]) __pure;
+void arm_read_instruction_set_attribute_regs(unsigned int regs[6]) __pure;
+void arm_read_extended_feature_regs(unsigned int regs[16]) __pure;
+
+unsigned int arm_read_cr(void) __pure;
+void arm_write_cr(unsigned int cr);
+unsigned int arm_read_aux_cr(void) __pure;
+void arm_write_aux_cr(unsigned int cr);
+unsigned int arm_read_perip_port_remap(void) __pure;
+void arm_write_perip_port_remap(unsigned int val);
+
+extern uint32_t arm_read_cp_access_cr(void) __pure;
+extern void arm_write_cp_access_cr(uint32_t val);
+
+extern void arm_fp_init(void);
+extern void arm_fp_enable(bool);
+
+extern uint32_t arm_read_fpexc(void) __pure;
+extern void arm_write_fpexc(uint32_t val);
+
+extern uint32_t arm_read_fpscr(void) __pure;
+extern void arm_write_fpscr(uint32_t val);
+
+extern void arm_init_fp_regs();
+
+#if ARCH_ARMv7
+extern uint32_t arm_read_l2_aux_cr(void) __pure;
+extern void arm_write_l2_aux_cr(uint32_t val);
+
+typedef enum {
+  ARM_PMCR = 0,			/* The order here matches a jump table in asm.S */
+  ARM_PMCNTENSET,
+  ARM_PMCNTENCLR,
+  ARM_PMOVSR,
+  ARM_PMSWINC,
+  ARM_PMSELR,
+
+  ARM_PMCCNTR,
+  ARM_PMXEVTYPER,
+  ARM_PMXEVCNTR,
+
+  ARM_PMUSERNR,
+  ARM_PMINTENSET,
+  ARM_PMINTENCLR,
+} arm_pmreg_t;
+
+extern uint32_t arm_read_pmreg(arm_pmreg_t reg) __pure;
+extern void arm_write_pmreg(arm_pmreg_t reg, uint32_t val);
+#endif /* ARCH_ARMv7 */
+
+#if ARCH_ARMv8
+
+extern void arm_write_mair(uint64_t mair);
+extern void arm_write_tcr(uint64_t tcr);
+extern void arm_write_ttbr0(void *ttbr0);
+extern void arm_store_pte(uint64_t *ent_p, uint64_t ent);
+
+#if WITH_L2_AS_RAM
+extern uint64_t arm_read_l2_cramconfig(void);
+#endif
+
+#endif /* ARCH_ARMv8 */
+
+void arm_mmu_init(bool resume);
+void arm_mpu_init(void);
+
+uint32_t arm_get_noncached_address(uint32_t address);
+
+void arm_write_dar(unsigned int dar);
+void arm_write_ttb(void *tt);
+void arm_write_ttbcr(unsigned int ttbc);
+
+#if WITH_MMU
+#define FSR_CAUSE_MASK	((1 << 10) | (0xF << 0))
+unsigned int arm_read_ifsr(void) __pure;
+unsigned int arm_read_dfsr(void) __pure;
+unsigned int arm_read_dfar(void) __pure;
+unsigned int arm_read_ifar(void) __pure;
+#else
+# define arm_read_ifsr()	~0
+# define arm_read_dfsr()	~0
+# define arm_read_dfar()	~0
+# define arm_read_ifar()	~0
+#endif
+
+#if WITH_CACHE_DEBUG
+void arm_l1cache_dump(int dcache, unsigned int addr);
+#else
+#define arm_l1cache_dump(dcache, addr)
+#endif
+
+/*
+ * Exception frame saved at the top of the exception/interrupt stacks.
+ */
+struct arm_exception_frame {
+	uint32_t	sp;		/* saved stack pointer (r13) */
+	uint32_t	lr;		/* saved link register (r14) */
+	uint32_t	spsr;		/* saved cpsr */
+	uint32_t	r[13];		/* r0 - r12 */
+	uint32_t	pc;		/* saved pc (r15) */
+};
+
+struct arm_exception_frame64 {
+	uint64_t	regs[29];	// x0-x28
+	uint64_t	fp;
+	uint64_t	lr;
+	uint64_t	sp;		
+	uint32_t	spsr;
+	uint32_t	reserved0;		
+	uint64_t	pc;
+	uint64_t	far;
+	uint32_t	esr;
+	uint32_t	reserved1;		
+};
+
+extern void *irq_stack_top;		/* stacks defined in start.S */
+extern void *fiq_stack_top;
+extern void *exc_stack_top;
+extern void *svc_stack_top;
+
+void arm_flush_tlbs(void);
+
+void arm_invalidate_dcache(void);
+void arm_invalidate_icache(void);
+void arm_clean_caches(void);
+void arm_clean_dcache(void);
+void arm_clean_dcache_line(addr_t addr);
+void arm_clean_dcache_line_2(addr_t addr);
+void arm_clean_dcache_line_3(addr_t addr);
+void arm_clean_dcache_line_4(addr_t addr);
+void arm_clean_dcache_line_5(addr_t addr);
+void arm_clean_dcache_line_6(addr_t addr);
+void arm_clean_dcache_line_7(addr_t addr);
+void arm_clean_dcache_line_8(addr_t addr);
+void arm_invalidate_dcache_line(addr_t addr);
+void arm_invalidate_dcache_line_2(addr_t addr);
+void arm_invalidate_dcache_line_3(addr_t addr);
+void arm_invalidate_dcache_line_4(addr_t addr);
+void arm_invalidate_dcache_line_5(addr_t addr);
+void arm_invalidate_dcache_line_6(addr_t addr);
+void arm_invalidate_dcache_line_7(addr_t addr);
+void arm_invalidate_dcache_line_8(addr_t addr);
+void arm_clean_invalidate_dcache_line(addr_t addr);
+void arm_clean_invalidate_dcache_line_2(addr_t addr);
+void arm_clean_invalidate_dcache_line_3(addr_t addr);
+void arm_clean_invalidate_dcache_line_4(addr_t addr);
+void arm_clean_invalidate_dcache_line_5(addr_t addr);
+void arm_clean_invalidate_dcache_line_6(addr_t addr);
+void arm_clean_invalidate_dcache_line_7(addr_t addr);
+void arm_clean_invalidate_dcache_line_8(addr_t addr);
+void arm_clean_invalidate_dcache(void);
+
+void arm_enable_l1parity(void);
+
+void arm_drain_write_buffer(void);
+
+void arm_flush_branch_predictor(void);
+
+void arm_init_vfp(void);
+void arm_enable_vfp(void);
+void arm_disable_vfp(void);
+
+void arm_enable_fiqs(void);
+void arm_disable_fiqs(void);
+
+unsigned int arm_read_vbar(void);
+void arm_write_vbar(unsigned int addr);
+
+void arm_memory_barrier(void);
+
+/* mmu stuff */
+#ifdef ARCH_ARMv8
+
+typedef enum {
+	kARMMMUDeviceRX,
+	kARMMMUDeviceR,
+	kARMMMUDeviceRW,
+	kARMMMUNormalRX,
+	kARMMMUNormalR,
+	kARMMMUNormalRW,
+	kARMMMUNumExtendedAttr,
+} arm_mmu_extended_attr_t;
+
+void arm_mmu_map_range(addr_t vaddr, addr_t paddr, size_t size, arm_mmu_extended_attr_t attr);
+
+// map  in place using the specified attributes
+void arm_mmu_map_rw(addr_t addr, size_t size);
+void arm_mmu_map_rx(addr_t addr, size_t size);
+void arm_mmu_map_ro(addr_t addr, size_t size);
+void arm_mmu_map_device_rw(addr_t addr, size_t size);
+void arm_mmu_map_device_rx(addr_t addr, size_t size);
+void arm_mmu_map_device_ro(addr_t addr, size_t size);
+
+#else
+
+typedef enum {
+	kARMMMUStronglyOrdered = 0,
+	kARMMMUDevice,
+	kARMMMUNormal,
+	kARMMMUInnerNormalOuterNoncached,
+	kARMMMUWriteCombined,
+	kARMMMUNormalRX,
+	kARMMMUNumAttr,
+} arm_mmu_attr_t;
+
+// Section and page mapping is explicit as there is always a fixed
+// granule at each level (without LPAE) in ARMv4-ARMv7.
+void arm_mmu_map_section(addr_t vaddr, addr_t paddr, arm_mmu_attr_t attr, bool shared);
+void arm_mmu_map_section_range(addr_t vaddr, addr_t paddr, size_t sections, arm_mmu_attr_t attr, bool shared, bool flush);
+
+#endif
+
+/* mpu stuff */
+void mpu_init(void);
+void arm_write_dprot_region_0(uint32_t base_size);
+void arm_write_dprot_region_1(uint32_t base_size);
+void arm_write_dprot_region_2(uint32_t base_size);
+void arm_write_dprot_region_3(uint32_t base_size);
+void arm_write_dprot_region_4(uint32_t base_size);
+void arm_write_dprot_region_5(uint32_t base_size);
+void arm_write_dprot_region_6(uint32_t base_size);
+void arm_write_dprot_region_7(uint32_t base_size);
+void arm_write_iprot_region_0(uint32_t base_size);
+void arm_write_iprot_region_1(uint32_t base_size);
+void arm_write_iprot_region_2(uint32_t base_size);
+void arm_write_iprot_region_3(uint32_t base_size);
+void arm_write_iprot_region_4(uint32_t base_size);
+void arm_write_iprot_region_5(uint32_t base_size);
+void arm_write_iprot_region_6(uint32_t base_size);
+void arm_write_iprot_region_7(uint32_t base_size);
+void arm_write_data_prot_register(uint32_t prot);
+void arm_write_ins_prot_register(uint32_t prot);
+void arm_write_cacheable_registers(uint32_t dcache, uint32_t icache);
+void arm_write_bufferable_register(uint32_t buff);
+
+/* thread id */
+uint32_t arm_read_user_ro_tid(void) __pure;
+uint32_t arm_read_user_rw_tid(void) __pure;
+uint32_t arm_read_sup_tid(void) __pure;
+void arm_write_user_ro_tid(uint32_t tid);
+void arm_write_user_rw_tid(uint32_t tid);
+void arm_write_sup_tid(uint32_t tid);
+
+/* DCC based debug */
+void arm_write_dcc_char(int c);
+int arm_read_dcc_char(void);
+int arm_read_dcc_char_nowait(void);
+int arm_dcc_init(void);
+
+#if CPU_APPLE_SWIFT
+extern uint32_t swift_read_l2cerrsts(void) __pure;
+extern void swift_write_l2cerrsts(uint32_t l2cerrsts);
+extern uint32_t swift_read_l2cerradr(void) __pure;
+extern void swift_write_l2cerradr(uint32_t l2cerradr);
+#endif /* CPU_APPLE_SWIFT */
+
+/* FP support */
+#if WITH_VFP
+void arm_call_fpsaved(void *arg, void (*func)(void *arg));
+#endif
+
+__END_DECLS
+
+#endif /* __ASSEMBLER__ */
+
+#define CPSR_MODE(_x)	((_x) & 0x1f)
+#define CPSR_MODE_USER		0x10
+#define CPSR_MODE_FIQ		0x11
+#define CPSR_MODE_IRQ		0x12
+#define CPSR_MODE_SUPERVISOR	0x13
+#define CPSR_MODE_ABORT		0x17
+#define CPSR_MODE_UNDEFINED	0x1b
+#define CPSR_MODE_SYSTEM	0x1f
+
+#define CPSR_STATE_THUMB	0x20
+
+#define CPACR_CP_BITS(_cp, _bits)	((_bits) << ((_cp) * 2))
+#define CPACR_CP_DENIED		0x0
+#define CPACR_CP_PRIVILEGED	0x1
+#define CPACR_CP_FULL		0x3
+
+#define FPEXC_EX		(1<<31)
+#define FPEXC_EN		(1<<30)
+#define FPEXC_FP2V		(1<<28)
+
+#define FPSCR_DEFAULT 		(1<<25) | (1<<24)	// DN, FZ bits
+
+#endif

arch/arm/include/arch/arm/assembler.h

@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2011 Apple Inc.  All rights reserved.	
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+
+/*
+ * Declare a globally-visible ARM function
+ */
+.macro ARM_FUNCTION
+	.text
+	.arm
+	.balign 4
+	.global $0
+$0:
+.endmacro
+
+/*
+ * Declare a globally-visible THUMB function
+ */
+.macro THUMB_FUNCTION
+	.text	
+	.thumb_func $0
+	.balign 2
+	.global $0
+$0:
+.endmacro
+	

arch/arm/link.exp

@@ -0,0 +1 @@
+_start

arch/arm/mmu.c

@@ -0,0 +1,176 @@
+/*
+ * Copyright (C) 2006-2011 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+#include <debug.h>
+#include <compiler.h>
+#include <sys.h>
+#include <arch.h>
+#include <arch/arm/arm.h>
+#include <platform.h>
+#include <platform/memmap.h>
+
+#define SECTION_SIZE MB
+#define LARGE_PAGE_SIZE 65536
+#define PAGE_SIZE 4096
+
+#define TTBCR_PD1 (1 << 5)
+
+typedef u_int32_t tt_ent;
+
+/* needs to be in non-static, initialized data for successful patching
+ * during IOP firmware loading */
+u_int32_t tt_size __attribute__((used)) = MMU_TT_SIZE;
+tt_ent *tt = (void *)0xffffffff;
+
+/* ARM MMU first-level section descriptor definitions */
+#define ARM_DESC_SECT		((0 << 18) | (1 << 1))
+#define ARM_SECT_B		(1 << 2)
+#define ARM_SECT_C		(1 << 3)
+#define ARM_SECT_XN		(1 << 4)
+#define ARM_SECT_DOMAIN(_d)	((_d) << 5)
+#define ARM_SECT_TEX(_t)	((_t) << 12)
+#define ARM_SECT_SHARED		(1 << 16)
+#define ARM_SECT_FULLACC	((0 << 15) | (3 << 10))
+#define ARM_SECT_EXEC		(ARM_SECT_DOMAIN(0))
+#define ARM_SECT_NOEXEC		(ARM_SECT_DOMAIN(1) | ARM_SECT_XN)
+#define ARM_SECT_READ_ONLY	(1 << 15)
+
+/* Non-normal memory needs XN set (and consequently a client domain assignment) to avoid
+ * instruction prefetch to those regions */
+static u_int32_t sect_flags[] = {
+	[kARMMMUStronglyOrdered]           = ARM_SECT_NOEXEC | ARM_SECT_FULLACC,
+	[kARMMMUDevice]                    = ARM_SECT_NOEXEC | ARM_SECT_B | ARM_SECT_FULLACC,
+	[kARMMMUNormal]                    = ARM_SECT_EXEC | ARM_SECT_C | ARM_SECT_B | ARM_SECT_FULLACC,
+	[kARMMMUInnerNormalOuterNoncached] = ARM_SECT_EXEC | ARM_SECT_TEX(4) | ARM_SECT_C | ARM_SECT_B | ARM_SECT_FULLACC,
+	[kARMMMUWriteCombined]             = ARM_SECT_NOEXEC | ARM_SECT_TEX(1) | ARM_SECT_FULLACC,	
+	[kARMMMUNormalRX]                  = ARM_SECT_DOMAIN(1) | ARM_SECT_C | ARM_SECT_B | ARM_SECT_READ_ONLY | ARM_SECT_FULLACC,
+};
+
+static tt_ent attr_to_sect_flags(arm_mmu_attr_t attr)
+{
+	if (attr >= kARMMMUNumAttr)
+		attr = kARMMMUStronglyOrdered;
+	return sect_flags[attr];
+}
+
+void arm_mmu_map_section(addr_t vaddr, addr_t paddr, arm_mmu_attr_t attr, bool shared)
+{
+	if ((vaddr & (SECTION_SIZE - 1)) != 0)
+		panic("section virtual address base is nonzero");
+	if ((paddr & (SECTION_SIZE - 1)) != 0)
+		panic("section physical address base is nonzero");
+
+	tt[vaddr / MB] = paddr | ARM_DESC_SECT | attr_to_sect_flags(attr) | (shared ?  ARM_SECT_SHARED : 0);
+
+	arm_clean_dcache_line((unsigned long)&tt[vaddr / MB]);
+	arm_flush_tlbs();
+}
+
+void arm_mmu_map_section_range(addr_t vaddr, addr_t paddr, size_t sections, arm_mmu_attr_t attr, bool shared, bool flush)
+{
+	tt_ent		entry, *tt_ptr;
+	addr_t		flush_start, flush_stop;
+	u_int32_t	cnt;
+
+	if ((vaddr & (SECTION_SIZE - 1)) != 0)
+		panic("section range virtual address base is nonzero");
+	if ((paddr & (SECTION_SIZE - 1)) != 0)
+		panic("section range physical address base is nonzero");
+
+	entry  = paddr | ARM_DESC_SECT | attr_to_sect_flags(attr) | (shared ?  ARM_SECT_SHARED : 0);
+	tt_ptr = &tt[vaddr / MB];
+
+	while (sections > 0) {
+		cnt = (sections < 16) ? sections : 16;
+		switch (cnt) {
+			case 16 : tt_ptr[15] = entry + MB * 15;
+			case 15 : tt_ptr[14] = entry + MB * 14;
+			case 14 : tt_ptr[13] = entry + MB * 13;
+			case 13 : tt_ptr[12] = entry + MB * 12;
+			case 12 : tt_ptr[11] = entry + MB * 11;
+			case 11 : tt_ptr[10] = entry + MB * 10;
+			case 10 : tt_ptr[ 9] = entry + MB *  9;
+			case  9 : tt_ptr[ 8] = entry + MB *  8;
+			case  8 : tt_ptr[ 7] = entry + MB *  7;
+			case  7 : tt_ptr[ 6] = entry + MB *  6;
+			case  6 : tt_ptr[ 5] = entry + MB *  5;
+			case  5 : tt_ptr[ 4] = entry + MB *  4;
+			case  4 : tt_ptr[ 3] = entry + MB *  3;
+			case  3 : tt_ptr[ 2] = entry + MB *  2;
+			case  2 : tt_ptr[ 1] = entry + MB *  1;
+			case  1 : tt_ptr[ 0] = entry + MB *  0;
+		}
+
+		entry	 += MB * cnt;
+		tt_ptr	 += cnt;
+		sections -= cnt;
+	}
+
+	if (flush) {
+		flush_start = ((addr_t)&tt[vaddr / MB]) & ~CPU_CACHELINE_SIZE;
+		flush_stop  = ((addr_t)tt_ptr + (CPU_CACHELINE_SIZE - 1)) & ~CPU_CACHELINE_SIZE;
+
+		while (flush_start < flush_stop) {
+			arm_clean_dcache_line(flush_start);
+
+			flush_start += CPU_CACHELINE_SIZE;
+		}
+
+		arm_flush_tlbs();
+	}
+}
+
+void arm_mmu_init(bool resume)
+{
+#ifdef MMU_TT_BASE
+	/* TT usually at the end of memory */
+	tt = (tt_ent *)MMU_TT_BASE;
+#endif
+
+	if (false == resume) {
+		/* identity map everything using the translation table as noncached */
+		arm_mmu_map_section_range(0, 0, MMU_TT_SIZE / sizeof(tt_ent), kARMMMUStronglyOrdered, false, false);
+	}
+
+	/* let the platform code get a shot at mapping some ranges */
+	platform_mmu_setup(resume);
+
+	/* set up the domain register */
+	arm_write_dar(0x7); /* domain 0 manager, domain 1 client */
+
+	/* write the new translation table */
+#if MMU_TT_SIZE == 0x4000
+        arm_write_ttbcr(0);
+#elif (MMU_TT_SIZE == 0x1000) && (WITH_MMU_SECURITY_EXTENSIONS)
+        arm_write_ttbcr(2 | TTBCR_PD1);
+#else
+#error "Must set up TTBCR appropriately for translation table smaller than 16 kB"
+#endif
+	arm_write_ttb(tt);
+
+	arm_flush_tlbs();
+}
+
+u_int32_t arm_get_noncached_address(u_int32_t address)
+{
+#ifdef MMU_NONCACHE0_SIZE
+	if (((int32_t)(address - MMU_NONCACHE0_PBASE) >= 0) && ((address - MMU_NONCACHE0_PBASE) < MMU_NONCACHE0_SIZE)) {
+	   	return address - MMU_NONCACHE0_PBASE + MMU_NONCACHE0_VBASE;
+       }
+#endif
+
+#ifdef MMU_NONCACHE1_SIZE
+	if (((int32_t)(address - MMU_NONCACHE1_PBASE) >= 0) && ((address - MMU_NONCACHE1_PBASE) < MMU_NONCACHE1_SIZE)) {
+	   	return address - MMU_NONCACHE1_PBASE + MMU_NONCACHE1_VBASE;
+       }
+#endif
+
+	return 0xFFFFFFFF;
+}

arch/arm/mpu.c

@@ -0,0 +1,28 @@
+/*
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+#include <arch.h>
+#include <platform/memmap.h>
+#include <sys.h>
+
+void arm_mpu_init(void)
+{
+	/* set everything as fully cached, except for the hardware register region */
+	arm_write_iprot_region_0(0x0000003f);
+	arm_write_dprot_region_0(0x0000003f);
+#ifdef PERIPH_BASE
+	arm_write_dprot_region_1((PERIPH_BASE | (0x1F - ((PERIPH_LEN == 0) ? 0 : (__builtin_clz(PERIPH_LEN) + 1))) << 1) | 1);
+#endif
+	arm_write_ins_prot_register(0x3); // region 0 full access
+	arm_write_data_prot_register(0xf); // region 0 & 1 full access
+	arm_write_cacheable_registers(0x1, 0x1); // region 0 data and instruction is cacheable
+	arm_write_bufferable_register(0x1); // data region 0 is write bufferable
+}
+

arch/arm/rules.mk

@@ -0,0 +1,248 @@
+# Copyright (C) 2007-2010 Apple Inc. All rights reserved.
+# Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+#
+# This document is the property of Apple Computer, Inc.
+# It is considered confidential and proprietary.
+#
+# This document may not be reproduced or transmitted in any form,
+# in whole or in part, without the express written permission of
+# Apple Computer, Inc.
+#
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+#
+# ARM CPU config 
+#
+
+# sanity check
+ifeq ($(ARM_CPU),)
+$(error ARM_CPU is not set)
+endif
+
+ifneq ($(ARM_ARCH),)
+$(info ARM_ARCH does not need to be set at the platform level)
+endif
+
+################################################################################
+# Per-CPU configuration 
+################################################################################
+
+#
+# Apple Swift
+#
+ifeq ($(ARM_CPU),apple-swift)
+ARM_ARCH ?= armv7
+ARM_ARCH_OPTIONS := ARCH_ARMv7=1 WITH_VFP=1
+OPTIONS += \
+	CPU_APPLE_SWIFT=1 \
+	WITH_MMU=1 \
+	WITH_MMU_SECURITY_EXTENSIONS=1 \
+	WITH_CACHE=1 \
+	CPU_CACHELINE_SIZE=64 \
+	CPU_CACHELINE_SHIFT=6 \
+	CPU_CACHEINDEX_SHIFT=8 \
+	CPU_CACHESET_SHIFT=3 \
+	WITH_BRANCH_PREDICTION=1 \
+	WITH_UNALIGNED_MEM=1 \
+	WITH_L1_PARITY=1 \
+	PAGE_SIZE=4096
+ALL_OBJS += \
+	$(LOCAL_DIR)/cache_v7.o \
+	$(LOCAL_DIR)/mmu.o \
+	$(LOCAL_DIR)/fp.o
+endif
+
+#
+# ARM Cortex A8
+#
+ifeq ($(ARM_CPU),cortex-a8)
+ARM_ARCH ?= armv7
+ARM_ARCH_OPTIONS := ARCH_ARMv7=1 WITH_VFP=1
+OPTIONS += \
+	CPU_ARM_CORTEX_A8=1 \
+	WITH_MMU=1 \
+	WITH_MMU_SECURITY_EXTENSIONS=1 \
+	WITH_CACHE=1 \
+	CPU_CACHELINE_SIZE=64 \
+	CPU_CACHELINE_SHIFT=6 \
+	CPU_CACHEINDEX_SHIFT=7 \
+	CPU_CACHESET_SHIFT=2 \
+	WITH_BRANCH_PREDICTION=1 \
+	WITH_UNALIGNED_MEM=1 \
+	WITH_L1_PARITY=1 \
+	WITH_CACHE_DEBUG=1 \
+	WITH_ARCHITECTED_L2=1 \
+	WITH_AUX_L2_ENABLE_BIT=1 \
+	PAGE_SIZE=4096
+ALL_OBJS += \
+	$(LOCAL_DIR)/cache_v7.o \
+	$(LOCAL_DIR)/mmu.o \
+	$(LOCAL_DIR)/fp.o \
+	$(LOCAL_DIR)/cachedebug_a8.o
+
+# Turned off to save space, enable as needed
+ifeq ($(APPLICATION),XXX_iBoot)
+ALL_OBJS += \
+	$(LOCAL_DIR)/cortex_hang.o \
+	$(LOCAL_DIR)/cortex_debug.o
+endif
+endif
+
+#
+# ARM Cortex A9
+#
+ifeq ($(ARM_CPU),cortex-a9)
+ARM_ARCH ?= armv7
+ARM_ARCH_OPTIONS := ARCH_ARMv7=1 WITH_VFP=1
+OPTIONS += \
+	CPU_ARM_CORTEX_A9=1 \
+	WITH_MMU=1 \
+	WITH_MMU_SECURITY_EXTENSIONS=1 \
+	WITH_CACHE=1 \
+	CPU_CACHELINE_SIZE=32 \
+	CPU_CACHELINE_SHIFT=5 \
+	CPU_CACHEINDEX_SHIFT=8 \
+	CPU_CACHESET_SHIFT=2 \
+	WITH_BRANCH_PREDICTION=1 \
+	WITH_UNALIGNED_MEM=1 \
+	WITH_L1_PARITY=1 \
+	SET_AUX_SMP_BIT=1 \
+	PAGE_SIZE=4096
+ALL_OBJS += \
+	$(LOCAL_DIR)/cache_v7.o \
+	$(LOCAL_DIR)/mmu.o \
+	$(LOCAL_DIR)/fp.o
+endif
+
+#
+# ARM Cortex A5 "Sparrow"
+#
+ifeq ($(ARM_CPU),cortex-a5)
+ARM_ARCH ?= armv7
+ARM_ARCH_OPTIONS := ARCH_ARMv7=1 WITH_VFP=1
+OPTIONS += \
+	CPU_ARM_CORTEX_A5=1 \
+	FP_REGISTER_COUNT=32 \
+	WITH_MMU=1 \
+	WITH_MMU_SECURITY_EXTENSIONS=1 \
+	WITH_CACHE=1 \
+	CPU_CACHELINE_SIZE=32 \
+	CPU_CACHELINE_SHIFT=5 \
+	CPU_CACHEINDEX_SHIFT=8 \
+	CPU_CACHESET_SHIFT=2 \
+	WITH_BRANCH_PREDICTION=1 \
+	WITH_UNALIGNED_MEM=1 \
+	SET_AUX_SMP_BIT=1 \
+	PAGE_SIZE=4096
+ALL_OBJS += \
+	$(LOCAL_DIR)/cache_v7.o \
+	$(LOCAL_DIR)/mmu.o \
+	$(LOCAL_DIR)/fp.o
+endif
+
+#
+# ARM Cortex A7 "Kingfisher"
+#
+ifeq ($(ARM_CPU),cortex-a7)
+ARM_ARCH ?= armv7k
+ARM_ARCH_OPTIONS := ARCH_ARMv7=1 WITH_VFP=1 WITH_VFP_ALWAYS_ON=1
+OPTIONS += \
+	CPU_ARM_CORTEX_A7=1 \
+	VFP_REV=4 \
+	FP_REGISTER_COUNT=32 \
+	WITH_MMU=1 \
+	WITH_MMU_SECURITY_EXTENSIONS=1 \
+	WITH_CACHE=1 \
+	WITH_EARLY_ICACHE=1 \
+	CPU_CACHELINE_SIZE=64 \
+	CPU_CACHELINE_SHIFT=6 \
+	CPU_CACHEINDEX_SHIFT=7 \
+	CPU_CACHESET_SHIFT=2 \
+	WITH_BRANCH_PREDICTION=1 \
+	WITH_UNALIGNED_MEM=1 \
+	WITH_ARCHITECTED_L2=1 \
+	SET_AUX_SMP_BIT=1 \
+	PAGE_SIZE=4096
+ALL_OBJS += \
+	$(LOCAL_DIR)/cache_v7.o \
+	$(LOCAL_DIR)/mmu.o \
+	$(LOCAL_DIR)/fp.o
+endif
+
+################################################################################
+# General and conditional configuration
+################################################################################
+
+# sanity check
+ifeq ($(ARM_ARCH),)
+$(error ARM_ARCH not set, no valid CPU selected)
+endif
+
+# build for the proper cpu
+OPTIONS			+= ARCH_ARM=1
+GLOBAL_ALLFLAGS		+= -arch $(ARM_ARCH) $(addprefix -D,$(ARM_ARCH_OPTIONS))
+GLOBAL_LDFLAGS		+= -arch $(ARM_ARCH)
+
+# Tell the build system how libraries should be tagged
+LIBRARY_TAG		+= $(ARM_ARCH)
+LIBRARY_OPTIONS		+= ARCH=arm ARM_ARCH=$(ARM_ARCH)
+
+# Always build libbuiltin, to partially override the gcc static library.
+LIBRARY_MODULES		+= lib/libbuiltin
+
+# set up to build with thumb
+WITH_THUMB ?= true
+ifeq ($(WITH_THUMB),true)
+THUMBFLAGS		:= -mthumb
+LIBRARY_TAG		+= thumb
+else
+THUMBFLAGS		:= -mno-thumb
+LIBRARY_TAG		+= arm
+endif
+
+GLOBAL_ALLFLAGS		+= $(THUMBFLAGS)
+GLOBAL_INCLUDES		+= $(LOCAL_DIR)/include
+
+# Defaults for stack sizes
+# These are override in platform/.../rules.mk
+ifeq ($(IRQ_STACK_SIZE),)
+  IRQ_STACK_SIZE	?= 2048
+endif
+ifeq ($(FIQ_STACK_SIZE),)
+  FIQ_STACK_SIZE	?= 1024
+endif
+ifeq ($(EXCEPTION_STACK_SIZE),)
+  EXCEPTION_STACK_SIZE	?= 2048
+endif
+ifeq ($(BOOTSTRAP_STACK_SIZE),)
+  BOOTSTRAP_STACK_SIZE	?= 2048
+endif
+OPTIONS += \
+	IRQ_STACK_SIZE=$(IRQ_STACK_SIZE) \
+	FIQ_STACK_SIZE=$(FIQ_STACK_SIZE) \
+	EXCEPTION_STACK_SIZE=$(EXCEPTION_STACK_SIZE) \
+	BOOTSTRAP_STACK_SIZE=$(BOOTSTRAP_STACK_SIZE)
+
+# Force start.o and asm.o to link first
+ALL_OBJS := \
+	$(LOCAL_DIR)/start.o \
+	$(LOCAL_DIR)/asm.o \
+	$(ALL_OBJS)
+
+# Code common to all cores
+ALL_OBJS += \
+	$(LOCAL_DIR)/cpu.o \
+	$(LOCAL_DIR)/context.o \
+	$(LOCAL_DIR)/entropy.o \
+	$(LOCAL_DIR)/exceptions.o \
+	$(LOCAL_DIR)/handlers.o \
+	$(LOCAL_DIR)/task.o
+
+LINKER_EXPORTS := $(LOCAL_DIR)/link.exp
+
+# if someone requests DCC, add it in
+ifeq ($(WITH_DCC),true)
+OPTIONS		+=	ARM_DCC_SYNCHRONOUS=1
+include $(LOCAL_DIR)/dcc/rules.mk
+endif
+

arch/arm/start.S

@@ -0,0 +1,343 @@
+/*
+ * Copyright (C) 2007-2014 Apple Inc. All rights reserved.
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <platform/memmap.h>
+
+	.text
+	.balign	4
+	.no_dead_strip _start
+	.global _start
+
+_start:
+	b	__reset
+
+	ldr	pc, vectoraddr + 4
+	ldr	pc, vectoraddr + 8
+	ldr	pc, vectoraddr + 12
+	ldr	pc, vectoraddr + 16
+	ldr	pc, vectoraddr + 20
+	ldr	pc, vectoraddr + 24
+	ldr	pc, vectoraddr + 28
+
+vectoraddr:
+	.word	__reset
+	.word	___arm_undefined
+	.word	___arm_syscall
+	.word	___arm_prefetch_abort
+	.word	___arm_data_abort
+	.word	___arm_reserved
+	.word	___arm_irq
+	.word	___arm_fiq
+
+__reset:
+	/*
+	 * Test the running versus desired the start of the text
+	 * to determine if relocaiton is required
+	 */
+	adr	r0, _start
+	ldr	r1, L_TEXT_START
+	cmp	r0, r1
+
+#ifdef PLATFORM_START_FUNCTION
+	/*
+	 * r0 and r1 must be preserved
+	 * 'eq' will be true if no text relocation is required
+	 */
+	bl	PLATFORM_START_FUNCTION
+#endif
+
+	/* re-test the running start versus desired start of the text */
+	cmp	r0, r1
+	beq	__do_relocate_data
+
+	/* relocate to the desired text location and branch to it */
+	ldr	r2, L_TEXT_END
+	sub	r2, r2, r1
+__relocate_loop:
+	subs	r2, r2, #4
+	ldr	r3, [r0, r2]
+	str	r3, [r1, r2]
+	bne	__relocate_loop
+
+#if ARCH_ARMv7
+	/* prevent speculation to the desired text location */
+	dsb
+	isb
+#endif
+
+	/* branch into the desired text location */
+	bx	r1
+
+	/* test if the data if needs to be relocated */
+__do_relocate_data:
+	ldr	r0, L_DATA_RO_START
+	mov	r1, #0x00001000		// Round up to a 4K page 
+	sub	r1, r1, #1		//
+	add	r0, r0, r1		//
+	bic	r0, r0, r1		//
+	ldr	r1, L_DATA_START
+	cmp	r0, r1
+	beq	__stacksetup
+
+	/* relocate the data */
+	ldr	r2, L_DATA_END
+	sub	r2, r2, r1
+__relocate_data_loop:
+	subs	r2, r2, #4
+	ldr	r3, [r0, r2]
+	str	r3, [r1, r2]
+	bne	__relocate_data_loop
+
+__stacksetup:
+	/* set up the stack for irq, fiq, abort, undefined, and lastly supervisor mode */
+	mrs	r0, cpsr
+	bic	r0, r0, #0x1f
+
+	orr	r1, r0, #0x12 // irq
+	msr	cpsr_c, r1
+	mov	lr, #0
+	ldr	sp, _irq_stack_top
+
+	orr	r1, r0, #0x11 // fiq
+	msr	cpsr_c, r1
+	mov	lr, #0
+	ldr	sp, _fiq_stack_top
+
+	orr	r1, r0, #0x17 // abort
+	msr	cpsr_c, r1
+	mov	lr, #0
+	ldr	sp, _exc_stack_top
+	    
+	orr	r1, r0, #0x1b // undefined
+	msr	cpsr_c, r1
+	mov	lr, #0
+	ldr	sp, _exc_stack_top
+
+	orr	r1, r0, #0x13 // supervisor
+	msr	cpsr_c, r1
+	ldr	sp, _svc_stack_top
+
+#if ARCH_ARMv7
+	mrs	r1, cpsr
+	bic	r1, #0x100 // enable imprecise data aborts
+	msr	cpsr_x, r1
+#endif
+
+#if SUPPORT_SLEEP
+	/* look at the global wakeup flag, avoid clearing the BSS and go straight to wake up */
+.global	_platform_wakeup
+	ldr	r1, _arch_sleep_magic	/* will be zero on the wake path, nonzero otherwise */
+	eor	r0, r0, r0
+	cmp	r0, r1
+	bne	__do_bss
+	ldr	r0, L__platform_wakeup
+	mov	lr, pc
+	bx	r0
+#endif
+
+__do_bss:
+	/* clear the bss section */
+	ldr	r0, L_BSS_START
+	ldr	r1, L_BSS_END
+	mov	r2, #0
+	mov	r3, #0
+	mov	r4, #0
+	mov	r5, #0
+	mov	r6, #0
+	mov	r7, #0
+	mov	r8, #0
+	mov	r10, #0
+	sub	r12, r1, r0
+	/* If there are less than 256 bytes, skip this loop */
+	subs	r12, r12, #256
+	blo	__bss_lessthan256
+
+__bss_loop256:
+	/* Clear 256 bytes at a time */
+	subs	r12, #256
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	stmia	r0!, {r2-r8, r10}
+	bge	__bss_loop256
+
+__bss_lessthan256:
+	/* If are less than 32 bytes left to handle, skip this loop */
+	adds	r12, r12, #(256 - 32)
+	bcc	__bss_lessthan32
+
+__bss_loop32:
+	/* Clear 32 bytes at a time */
+	subs	r12, r12, #32
+	stmia	r0!, {r2-r8, r10}
+	bge	__bss_loop32
+
+__bss_lessthan32:
+	/* If there's nothing left to go, skip this loop */
+	adds	r12, r12, #32
+	beq	__bss_done
+
+__bss_loop4:
+	/* Clear 4 bytes at a time */
+	subs	r12, r12, #4
+	str	r2, [r0],#4
+	bgt	__bss_loop4
+
+__bss_done:
+	/* branch to main */
+	ldr	r0, L__main
+	mov	lr, pc
+	bx	r0
+	b	.
+
+
+	.org 0x200
+L__build_banner_string:
+	.ascii CONFIG_PROGNAME_STRING
+	.ascii " for "
+	.ascii CONFIG_BOARD_STRING
+	.ascii ", Copyright 2007-2016, Apple Inc."
+
+	.org 0x240
+L__build_style_string:
+	.ascii BUILD_STYLE
+
+/* This will get replaced by macho_post_process.py so that the build tag
+   is accurate even for iterative desktop builds that don't rebuild start.o */
+	.org 0x280
+_build_tag_string_contents:
+
+/* Placeholder for macho_post_process.py. Placing the UUID here allows
+   debuggers to locate the UUID based on the address of the vectors */
+	.org 0x300
+_UUID:
+	.space 0x20
+
+	.global _build_banner_string
+_build_banner_string:
+	.long L__build_banner_string
+
+	.global _build_style_string
+_build_style_string:
+	.long L__build_style_string
+
+	.global _build_tag_string
+_build_tag_string:
+	.long _build_tag_string_contents
+
+L__main:
+	.long __main
+
+	/* linked start and end addresses of the text */
+L_TEXT_START:
+	.long (TEXT_BASE)
+L_TEXT_END:
+	.long section$start$__DATA$__zerofill
+
+	/* start address of the data in the read only text */
+L_DATA_RO_START:
+	.long segment$end$__TEXT
+
+	/* linked start and end addresses of the data */
+L_DATA_START:
+	.long segment$start$__DATA
+L_DATA_END:
+	.long section$start$__DATA$__zerofill
+
+	/* linked start and end addresses of the zero-filled range */
+L_BSS_START:
+	.long section$start$__DATA$__zerofill
+L_BSS_END:
+	.long segment$end$__DATA
+
+	/*
+	 * Stacks.
+	 *
+	 * The following stacks must be defined:
+	 * 
+	 * bootstrap stack
+	 *	Used as the stack for the initial thread.
+	 *
+	 * exception stack
+	 *	Used when handling exceptions other than interrupts.
+	 *
+	 * interrupt stack
+	 *	Used when handling regular interrupts.
+	 *
+	 * fiq stack
+	 *	Used when handling FIQ interrupts.
+	 */
+	
+	.global _irq_stack_top
+	.global _fiq_stack_top
+	.global _exc_stack_top
+	.global _svc_stack_top
+
+#ifndef STACK_SIZE
+	/*
+	 * This is a bit hokey, as the platform doesn't really know
+	 * what the bootstrap stack size should be.
+	 */
+# define STACK_SIZE	BOOTSTRAP_STACK_SIZE
+#endif
+#if APPLICATION_EMBEDDEDIOP
+	/*
+	 * Locally-allocated stacks.
+	*/
+_irq_stack_top:
+	.long irq_stack + IRQ_STACK_SIZE
+_fiq_stack_top:
+	.long fiq_stack + FIQ_STACK_SIZE
+_exc_stack_top:
+	.long exc_stack + EXCEPTION_STACK_SIZE
+_svc_stack_top:
+	.long svc_stack + STACK_SIZE
+
+	/* allocate stacks here */
+	.lcomm irq_stack, IRQ_STACK_SIZE, 4
+	.lcomm fiq_stack, FIQ_STACK_SIZE, 4
+	.lcomm exc_stack, EXCEPTION_STACK_SIZE, 4
+	.lcomm svc_stack, STACK_SIZE, 4
+
+#elif defined(STACK_BASE)
+	/*
+	 * Simple stack layout with just one allocation in STACK_BASE/STACK_SIZE
+	 * supplied by the platform definition.
+	 *
+	 * Bootstrap stack starts at the top of the allocation,
+	 * exception stack and interrupt stack share the same
+	 * chunk at the bottom.
+	 */
+_fiq_stack_top:	
+	.long STACK_BASE + FIQ_STACK_SIZE
+_exc_stack_top:
+_irq_stack_top:
+	.long STACK_BASE + IRQ_STACK_SIZE
+_svc_stack_top:
+	.long STACK_BASE + STACK_SIZE
+#else
+# error No stack defined
+#endif
+
+#if SUPPORT_SLEEP
+.global	_arch_sleep_magic
+_arch_sleep_magic:
+	.long	0xffffffff
+
+L__platform_wakeup:
+	.long _platform_wakeup
+	
+#endif

arch/arm/task.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+#include <debug.h>
+#include <sys/types.h>
+#include <sys.h>
+#include <sys/task.h>
+
+static void arch_task_trampoline(void) __noreturn;
+
+static void 
+arch_task_trampoline(void)
+{
+	/* exit the critical section we implicitly started with */
+	exit_critical_section();
+
+	dprintf(DEBUG_SPEW, "arch_task_trampoline: task %p (%s) routine %p, arg %p\n", 
+			current_task, current_task->name, current_task->routine, current_task->arg);
+
+	/* call the start routine */
+	int ret = current_task->routine(current_task->arg);
+
+	dprintf(DEBUG_SPEW, "arch_task_trampoline: task %p (%s) exits with %d\n",
+			current_task, current_task->name, ret);
+
+	task_exit(ret);
+
+	/* never return */
+}
+
+int 
+arch_task_create(struct task *t)
+{
+	int i;
+
+	for (i=0; i < 10; i++)
+		t->arch.regs[i] = 0;
+
+	/* set the default starting point to be arch_task_trampoline */
+	t->arch.regs[ARM_ARCH_TASK_LR] = (uint32_t)&arch_task_trampoline;
+	/* the ARM describes some encodings of SP ops (pre/post increment and/or decrement) as
+	   potentially reaching above the SP by a word. Our toolchain might not emit such encodings,
+	   but we'd rather be safe than sorry. Originally, we took off 4 bytes, but
+	   to better align the stacks for some of the newer ABIs (e.g. armv7k), we'll consistently
+	   take off 16 bytes from the top of the stack */
+	t->arch.regs[ARM_ARCH_TASK_SP] = (uint32_t)t->stack_base + t->stack_len - 16;
+	t->arch.regs[ARM_ARCH_TASK_FP] = 0;
+
+#if WITH_VFP
+	t->arch.fpexc = 0;	/* default state is FP disabled */
+#endif
+	
+	return 0;
+}

arch/arm64/asm.S

@@ -0,0 +1,276 @@
+/*
+ * Copyright (c) 2011-2012, 2014 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <arch/arm64/proc_reg.h>
+
+	.text
+	.align	2
+	.globl	_arm_read_sctlr
+_arm_read_sctlr:
+	mrs	x0, SCTLR_ELx
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_write_sctlr
+_arm_write_sctlr:
+	msr	SCTLR_ELx, x0
+	dsb	sy
+	isb	sy
+	// The ifdef should match the ifdef in arch_cpu_quiesce()
+#if CPU_APPLE_CYCLONE
+	// Don't allow MMU or WXN to be disabled. Placing the check after the msr
+	// means that a ROP attack will fail unless an attacker can trigger an
+	// interrupt between the msr and check
+	mov	x1, x0
+	orr	x0, x0, #SCTLR_M_ENABLED
+	orr	x0, x0, #SCTLR_WXN_ENABLED
+	cmp	x0, x1
+	b.ne	_arm_write_sctlr
+#endif
+	ret
+
+#if WITH_EL3
+	.text
+	.align	2
+	.globl	_arm_write_scr
+_arm_write_scr:
+	msr	SCR_EL3, x0
+	isb	sy
+	ret
+#endif
+
+#if WITH_L2_AS_RAM
+	.text
+	.align	2
+	.globl	_arm_read_l2_cramconfig
+_arm_read_l2_cramconfig:
+	mrs	x0, s3_3_c15_c7_0
+	ret
+#endif
+
+	.text
+	.align	2
+	.globl	_arm_write_mair
+_arm_write_mair:
+	msr	MAIR_ELx, x0
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_write_tcr
+_arm_write_tcr:
+	msr	TCR_ELx, x0
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_write_ttbr0
+_arm_write_ttbr0:
+	msr	TTBR0_ELx, x0
+	isb	sy
+	ret
+
+
+	.text
+	.align	2
+	.globl	_arm_read_cpacr
+_arm_read_cpacr:
+	mrs	x0, CPACR_EL1
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_write_cpacr
+_arm_write_cpacr:
+	msr	CPACR_EL1, x0
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_read_cntp_ctl
+_arm_read_cntp_ctl:
+	isb	sy
+	mrs	x0, CNTP_CTL_EL0
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_write_cntp_ctl
+_arm_write_cntp_ctl:
+	msr	CNTP_CTL_EL0, x0
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_read_cntpct
+_arm_read_cntpct:
+	isb	sy
+	mrs	x0, CNTPCT_EL0
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_write_cntp_tval
+_arm_write_cntp_tval:
+	msr	CNTP_TVAL_EL0, x0
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_flush_tlbs
+_arm_flush_tlbs:
+	dsb	sy
+#if WITH_EL3
+	tlbi	alle3
+#else
+	tlbi	vmalle1
+#endif
+	dsb	sy
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_invalidate_icache
+_arm_invalidate_icache:
+	ic	iallu
+	dsb	sy
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_invalidate_dcache_line
+_arm_invalidate_dcache_line:
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_clean_dcache_line
+_arm_clean_dcache_line:
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_clean_invalidate_dcache
+_arm_clean_invalidate_dcache:
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_clean_dcache
+_arm_clean_dcache:
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_invalidate_dcache
+_arm_invalidate_dcache:
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_clean_invalidate_dcache_line
+_arm_clean_invalidate_dcache_line:
+	dc	civac, x0
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_drain_write_buffer
+_arm_drain_write_buffer:
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_memory_barrier
+_arm_memory_barrier:
+	dmb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_vtop
+_arm_vtop:
+	mrs	x2, DAIF				// Load current DAIF
+	msr	DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)	// Disable IRQ
+	isb	sy
+	mov	x9, x0					// Saved address
+	at	s1e3r, x0				// Translation Stage 1 EL1
+	isb	sy
+	mrs	x1, PAR_EL1				// Read result
+	tst	x1, #1					// Test Translation not valid
+	cset	x0, eq
+	b.ne	L_arm_vtop_ret
+	movk	x3, #0xf000
+	movk	x3, #0xffff, lsl #16
+	movk	x3, #0xffff, lsl #32
+	and	x1, x1, x3				// Get physical address base
+	and	x0, x9, #0xfff
+	orr	x0, x1, x0				// Add page offset
+L_arm_vtop_ret:
+	msr	DAIF, x2				// Restore interrupt state, return
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_disable_async_aborts
+_arm_disable_async_aborts:
+	msr	DAIFSet, #(DAIFSC_ASYNCF)
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arm_enable_async_aborts
+_arm_enable_async_aborts:
+	msr	DAIFClr, #(DAIFSC_ASYNCF)
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arch_disable_ints
+_arch_disable_ints:
+	msr	DAIFSet, #(DAIFSC_IRQF | DAIFSC_FIQF)
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arch_enable_ints
+_arch_enable_ints:
+	msr	DAIFClr, #(DAIFSC_IRQF | DAIFSC_FIQF)
+	isb	sy
+	ret
+
+	.text
+	.align	2
+	.globl	_arch_halt
+_arch_halt:
+	dsb	sy
+	isb	sy
+	wfi
+	ret
+
+	.text
+	.align	2
+	.globl	_arch_spin
+_arch_spin:
+	hint	0x45	// make fastsim drop to debugger (nop on hardware)
+	b	.

arch/arm64/context.S

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2011-2012 Apple Inc.  All rights reserved.	
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+/*
+ * Task context switch support.
+ * 
+ */
+	
+/*
+ * Code here must remain synchronized with the structure and offsets defined in arch_task.h:
+ *
+ *	struct arch_task {
+ *		uint64_t		regs[29];	// x0-x28
+ *		uint64_t		fp;
+ *		uint64_t		lr;
+ *		uint64_t		sp;	
+ *
+ *		union {
+ *			uint128_t	q;
+ *			uint64_t	d;
+ *			uint32_t	s;
+ *		} vregs[32];			// v0-v31
+ *		uint32_t		fpsr;
+ *		uint32_t		fpcr;
+ *	};
+ */
+
+/* AAPCS-64 5.1.1 Page 13:
+ * Preserve r16-17, r19-r28, FP, and SP.
+ *
+ * AAPCS-64 5.1.2 Page 14:
+ * Registers d8-d15 (s8-s15) must be preserved by a callee across subroutine
+ * calls; the remaining registers (v0-v7, v16-v31) do not need to be preserved
+ * (or should be preserved by the caller).
+ */
+
+/*
+ * void arch_task_context_switch(struct arch_task *old, struct arch_task *new_task);
+ */
+
+	.text
+	.align 2
+	.globl _arch_task_context_switch
+_arch_task_context_switch:
+	stp	x16, x17, [x0, #120]
+	stp	x19, x20, [x0, #152]
+	stp	x21, x22, [x0, #168]
+	stp	x23, x24, [x0, #184]
+	stp	x25, x26, [x0, #200]
+	stp	x27, x28, [x0, #216]
+	stp	fp, lr, [x0, #232]
+	mov	x2, sp
+	str	x2, [x0, #248]
+
+#if WITH_VFP
+        str	d8,  [x0, #384]
+        str	d9,  [x0, #400]
+        str	d10, [x0, #416]
+        str	d11, [x0, #432]
+        str	d12, [x0, #448]
+        str	d13, [x0, #464]
+        str	d14, [x0, #480]
+        str	d15, [x0, #496]
+#endif
+	
+	mov	x0, x1
+	b	_arch_task_context_restore
+	
+/*
+ * void arch_task_context_restore(struct arch_task *restore_context)
+ */
+
+	.text
+	.align 2
+	.globl _arch_task_context_restore
+_arch_task_context_restore:
+	ldp	x16, x17, [x0, #120]
+	ldp	x19, x20, [x0, #152]
+	ldp	x21, x22, [x0, #168]
+	ldp	x23, x24, [x0, #184]
+	ldp	x25, x26, [x0, #200]
+	ldp	x27, x28, [x0, #216]
+	ldp	fp, lr, [x0, #232]
+	ldr	x1, [x0, #248]
+	mov	sp, x1
+
+#if WITH_VFP
+        ldr	d8,  [x0, #384]
+        ldr	d9,  [x0, #400]
+        ldr	d10, [x0, #416]
+        ldr	d11, [x0, #432]
+        ldr	d12, [x0, #448]
+        ldr	d13, [x0, #464]
+        ldr	d14, [x0, #480]
+        ldr	d15, [x0, #496]
+#endif
+
+	ret
+	

arch/arm64/cpu.c

@@ -0,0 +1,100 @@
+/*
+ * Copyright (C) 2011-2012, 2014 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch.h>
+#include <arch/arm/arm.h>
+#include <arch/arm64/proc_reg.h>
+#include <debug.h>
+#include <platform.h>
+#include <platform/memmap.h>
+
+extern void arm_write_sctlr(uint64_t);
+extern uint64_t arm_read_sctlr();
+extern void arm_enable_async_aborts();
+#if	WITH_EL3
+extern void arm_write_scr(uint64_t);
+#endif
+
+int arch_cpu_init(bool resume)
+{
+	uint64_t sctlr;
+
+#if	WITH_EL3
+	uint64_t scr;
+
+	/* enable interrupts forwarding to EL3 */
+	scr = (SCR_EA | SCR_FIQ | SCR_IRQ);
+	arm_write_scr(scr);
+#endif
+
+	/* enable async aborts */
+	arm_enable_async_aborts();
+
+	/* invalidate caches */
+	arm_invalidate_icache();
+	arm_invalidate_dcache();
+
+	sctlr = arm_read_sctlr();
+
+	/* turn on Stack Alignment check */
+	sctlr |= SCTLR_SA_ENABLED;
+
+	/* turn on the mmu */
+	arm_mmu_init(resume);
+	sctlr |= SCTLR_M_ENABLED;
+
+	/* disallow executing from writeable pages */
+	sctlr |= SCTLR_WXN_ENABLED;
+
+	/* turn on d-cache */
+	sctlr |= SCTLR_D_ENABLED;
+
+	/* turn on i-cache */
+	sctlr |= SCTLR_I_ENABLED;
+
+	/* enable the MMU, caches and etc. */
+	arm_write_sctlr(sctlr);
+
+#if WITH_VFP
+	/* initialize VFP */
+	arm_fp_init();
+#endif
+
+	return 0;
+}
+
+int arch_cpu_quiesce(void)
+{
+#if !CPU_APPLE_CYCLONE
+	uint64_t sctlr;
+
+	/* clean the d-cache */
+	arm_clean_dcache();
+
+	/* read the control regiser base value */
+	sctlr = arm_read_sctlr();
+
+	sctlr &= ~(SCTLR_D_ENABLED | SCTLR_SA_ENABLED | SCTLR_I_ENABLED | SCTLR_M_ENABLED | SCTLR_WXN_ENABLED);
+
+	/* disable the MMU, caches and etc. */
+	arm_write_sctlr(sctlr);
+
+	/* invalidate i-cache */
+	arm_invalidate_icache();
+#endif /* !CPU_APPLE_CYCLONE */
+
+	return 0;
+}
+
+int arch_cpu_init_posttasks(void)
+{
+	return 0;
+}
+

arch/arm64/entropy.S

@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2012 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch/arm/assembler.h>
+
+	.text
+	.balign 4
+	.globl _arch_get_entropy
+
+//	u_int32_t arch_get_entropy(volatile u_int32_t *counter_address)
+//
+//	arch_get_entropy exploits the uncertainty in the asynchronous boundary crossing
+//	between the CPU and a counter.  To work correctly, the CPU and counter must be
+//	derived from clocks with relative jitter (PLL vs. OSC).  Ideally the inner loop,
+//	up to 32 iterations, should consume a large part of the effective read period of
+//	the counter.
+//
+
+_arch_get_entropy:
+
+	mov	x1, x0
+	mov	w0, #0
+	mov	w2, #32
+1:
+	mov	w3, w2
+2:
+	subs	w3, w3, #1
+	b.ne	2b
+	ldr	w3, [x1]
+	and	w3, w3, #1
+	orr	w0, w3, w0, lsl #1
+	subs	w2, w2, #1
+	b.ne	1b
+	ret

arch/arm64/exceptions_el1.S

@@ -0,0 +1,306 @@
+/*
+ * Copyright (C) 2011-2012, 2014 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+
+#if	WITH_EL3
+#error	"This file should not be built for platforms WITH_EL3"
+#endif
+
+////////////////////////////////////////////////////////////
+// This is what exception frame looks like:
+//
+//	struct arm_exception_frame64 {
+//		uint64_t	regs[29];	// x0-x28
+//		uint64_t	fp;
+//		uint64_t	lr;
+//		uint64_t	sp;
+//		uint32_t	spsr;
+//		uint32_t	reserved0;
+//		uint64_t	pc;
+//		uint64_t	far;
+//		uint32_t	esr;
+//		uint32_t	reserved1;
+//		uint64_t	reserved2;	// stp requires multiple of 16 in imm field
+//		uint64_t	reserved3;	// stp requires multiple of 16 in imm field
+//		union {
+//			uint128_t	q;
+//			uint64_t	d;
+//			uint32_t	s;
+//		} vregs[32];			// v0-v31
+//		uint32_t	fpsr;
+//		uint32_t	reserved4;
+//		uint32_t	fpcr;
+//		uint32_t	reserved5;
+//	};
+////////////////////////////////////////////////////////////
+
+.macro EL1_SP0_VECTOR
+	msr	SPSel, #0			// Switch to SP0
+	sub	sp, sp, #832			// Create exception frame
+	stp	x0, x1, [sp, #0]		// Save x0, x1 to exception frame
+	add	x0, sp, #832			// Calculate the original stack pointer
+	str	x0, [sp, #248]			// Save stack pointer to exception frame
+	stp	fp, lr, [sp, #232]		// Save fp and lr to exception frame
+	mrs     lr, ELR_EL1			// Get exception link reg so _dispatch64 can make a frame
+	mov	x0, sp				// Copy saved state pointer to x0
+.endmacro
+
+.macro FASTSIM_DEBUG_HINT
+	hint	0x45	// make fastsim drop to debugger (nop on hardware)
+.endmacro
+
+	.text
+	.align 12
+	.globl _exception_vector_base
+_exception_vector_base:
+L__el1_sp0_synchronous_vector:
+	EL1_SP0_VECTOR
+	mrs	x1, ESR_EL1			// Load exception syndrome
+	str	x1, [x0, #280]
+	mrs	x1, FAR_EL1			// Load fault address
+	str	x1, [x0, #272]
+	adrp	x1, _arm_synchronous_exception@page
+	add	x1, x1, _arm_synchronous_exception@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el1_sp0_irq_vector:
+	EL1_SP0_VECTOR
+	adrp	x1, _interrupt_stack_top@page
+	add	x1, x1, _interrupt_stack_top@pageoff
+	ldr	x1, [x1]
+	mov	sp, x1
+	adrp	x1, _arm_irq@page
+	add	x1, x1, _arm_irq@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el1_sp0_fiq_vector:
+	EL1_SP0_VECTOR
+	adrp	x1, _interrupt_stack_top@page
+	add	x1, x1, _interrupt_stack_top@pageoff
+	ldr	x1, [x1]
+	mov	sp, x1
+	adrp	x1, _arm_fiq@page
+	add	x1, x1, _arm_fiq@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el1_sp0_serror_vector:
+	EL1_SP0_VECTOR
+	mrs	x1, ESR_EL1			// Load exception syndrome
+	str	x1, [x0, #280]
+	mrs	x1, FAR_EL1			// Load fault address
+	str	x1, [x0, #272]
+	adrp	x1, _arm_serror_exception@page
+	add	x1, x1, _arm_serror_exception@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el1_sp1_synchronous_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__el1_sp1_irq_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__el1_sp1_fiq_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__el1_sp1_serror_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_64_synchronous_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_64_irq_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_64_fiq_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_64_serror_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_32_synchronous_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_32_irq_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_32_fiq_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_32_serror_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+/* Placeholder for macho_post_process.py. Placing the UUID here
+   allows debuggers to locate the UUID based on the value of VBAR */
+	.text
+	.align 7
+_UUID:
+	.space 32
+
+/* 64-bit first level exception handler dispatcher.
+* Completes register context saving and branches to FLEH.
+* Expects:
+*  {x0, x1, fp, lr, sp} - saved
+*  x0 - arm_context_t
+*  x1 - address of FLEH
+*  fp - previous stack frame if EL1
+*  lr - unused
+*  sp - kernel stack
+*/
+	.text
+	.align 2
+__dispatch64:
+	stp	fp, lr, [sp, #-16]!		// Create a frame to help debuggers unwind
+	mov	fp, sp				// past the exception handler
+	stp	x2, x3, [x0, #16]		// Save remaining GPRs
+	stp	x4, x5, [x0, #32]
+	stp	x6, x7, [x0, #48]
+	stp	x8, x9, [x0, #64]
+	stp	x10, x11, [x0, #80]
+	stp	x12, x13, [x0, #96]
+	stp	x14, x15, [x0, #112]
+	stp	x16, x17, [x0, #128]
+	stp	x18, x19, [x0, #144]
+	stp	x20, x21, [x0, #160]
+	stp	x22, x23, [x0, #176]
+	stp	x24, x25, [x0, #192]
+	stp	x26, x27, [x0, #208]
+	str	x28, [x0, #224]
+	mrs	lr, ELR_EL1			// Get exception link register
+	str	lr, [x0, #264]			// Save ELR to PCB
+	mrs	x2, SPSR_EL1			// Get CPSR
+	str	w2, [x0, #256]			// Save CPSR to PCB
+
+#if WITH_VFP
+	stp	q0, q1, [x0, #304]
+	stp	q2, q3, [x0, #336]
+	stp	q4, q5, [x0, #368]
+	stp	q6, q7, [x0, #400]
+	stp	q8, q9, [x0, #432]
+	stp	q10, q11, [x0, #464]
+	stp	q12, q13, [x0, #496]
+	stp	q14, q15, [x0, #528]
+	stp	q16, q17, [x0, #560]
+	stp	q18, q19, [x0, #592]
+	stp	q20, q21, [x0, #624]
+	stp	q22, q23, [x0, #656]
+	stp	q24, q25, [x0, #688]
+	stp	q26, q27, [x0, #720]
+	stp	q28, q29, [x0, #752]
+	stp	q30, q31, [x0, #784]
+	mrs	x2, FPSR
+	str	w2, [x0, #816]
+	mrs	x2, FPCR
+	str	w2, [x0, #820]
+#endif
+
+	mov	x21, x0				// Copy arm_context_t pointer to x21
+	blr	x1
+	b	__exception_return
+	
+	.text
+	.align 2
+__exception_return:
+	mov     sp, x21				// Reload the arm_context_t pointer
+
+#if WITH_VFP
+	ldp	q0, q1, [sp, #304]
+	ldp	q2, q3, [sp, #336]
+	ldp	q4, q5, [sp, #368]
+	ldp	q6, q7, [sp, #400]
+	ldp	q8, q9, [sp, #432]
+	ldp	q10, q11, [sp, #464]
+	ldp	q12, q13, [sp, #496]
+	ldp	q14, q15, [sp, #528]
+	ldp	q16, q17, [sp, #560]
+	ldp	q18, q19, [sp, #592]
+	ldp	q20, q21, [sp, #624]
+	ldp	q22, q23, [sp, #656]
+	ldp	q24, q25, [sp, #688]
+	ldp	q26, q27, [sp, #720]
+	ldp	q28, q29, [sp, #752]
+	ldp	q30, q31, [sp, #784]
+	ldr	w3, [sp, #816]
+	msr	FPSR, x3
+	ldr	w4, [sp, #820]
+	msr	FPCR, x4
+#endif
+
+	ldr	x0, [sp, #264]			// Get the return address
+	msr	ELR_EL1, x0			// Load the return address into ELR
+	ldr	w1, [sp, #256]			// Get the return CPSR
+	msr	SPSR_EL1, x1			// Load the return CPSR into SPSR
+	ldp	x0, x1, [sp, #0]		// Restore the GPRs
+	ldp	x2, x3, [sp, #16]
+	ldp	x4, x5, [sp, #32]
+	ldp	x6, x7, [sp, #48]
+	ldp	x8, x9, [sp, #64]
+	ldp	x10, x11, [sp, #80]
+	ldp	x12, x13, [sp, #96]
+	ldp	x14, x15, [sp, #112]
+	ldp	x16, x17, [sp, #128]
+	ldp	x18, x19, [sp, #144]
+	ldp	x20, x21, [sp, #160]
+	ldp	x22, x23, [sp, #176]
+	ldp	x24, x25, [sp, #192]
+	ldp	x26, x27, [sp, #208]
+	ldr	x28, [sp, #224]
+	ldp	fp, lr, [sp, #232]
+
+	/* Use exception stack to restore SP0 */
+
+	msr	SPSel, #1			// Switch to SPx
+	stp	x0, x1, [sp, #-16]!		// Save x0 and x1
+	mrs	x0, SP_EL0			// Reload the pcb pointer from SP0
+	ldr	x1, [x0, #248]			// Get the saved SP from the pcb
+	msr	SP_EL0, x1			// Restore SP0
+	ldp	x0, x1, [sp], #16		// Restore x0 and x1
+	eret

arch/arm64/exceptions_el3.S

@@ -0,0 +1,315 @@
+/*
+ * Copyright (C) 2011-2012, 2014 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+
+#if	!WITH_EL3
+#error	"This file should only be built for platforms WITH_EL3"
+#endif
+
+////////////////////////////////////////////////////////////
+// This is what exception frame looks like:
+//
+//	struct arm_exception_frame64 {
+//		uint64_t	regs[29];	// x0-x28
+//		uint64_t	fp;
+//		uint64_t	lr;
+//		uint64_t	sp;		
+//		uint32_t	spsr;
+//		uint32_t	reserved0;		
+//		uint64_t	pc;
+//		uint64_t	far;
+//		uint32_t	esr;
+//		uint32_t	reserved1;
+//		uint64_t	reserved2;	// stp requires multiple of 16 in imm field
+//		uint64_t	reserved3;	// stp requires multiple of 16 in imm field
+//		union {
+//			uint128_t	q;
+//			uint64_t	d;
+//			uint32_t	s;
+//		} vregs[32];			// v0-v31
+//		uint32_t	fpsr;
+//		uint32_t	reserved4;
+//		uint32_t	fpcr;
+//		uint32_t	reserved5;
+//	};
+////////////////////////////////////////////////////////////
+
+.macro EL3_SP0_VECTOR
+	msr	SPSel, #0		// Switch to SP0
+	sub	sp, sp, #832		// Create exception frame
+	stp	x0, x1, [sp, #0]	// Save x0, x1 to exception frame
+	add	x0, sp, #832		// Calculate the original stack pointer
+	str	x0, [sp, #248]		// Save stack pointer to exception frame
+	stp	fp, lr, [sp, #232]	// Save fp and lr to exception frame
+	mrs     lr, ELR_EL3		// Get exception link reg so _dispatch64 can make a frame
+	mov	x0, sp			// Copy saved state pointer to x0
+.endmacro
+
+.macro FASTSIM_DEBUG_HINT
+	hint	0x45	// make fastsim drop to debugger (nop on hardware)
+.endmacro
+
+	.text
+	.align 12
+	.globl _exception_vector_base
+_exception_vector_base:
+L__el3_sp0_synchronous_vector:
+	EL3_SP0_VECTOR
+	mrs	x1, ESR_EL3					// Load exception syndrome
+	str	x1, [x0, #280]
+	mrs	x1, FAR_EL3					// Load fault address
+	str	x1, [x0, #272]
+	adrp	x1, _arm_synchronous_exception@page
+	add	x1, x1, _arm_synchronous_exception@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el3_sp0_irq_vector:
+	EL3_SP0_VECTOR
+	adrp	x1, _interrupt_stack_top@page
+	add	x1, x1, _interrupt_stack_top@pageoff
+	ldr	x1, [x1]
+	mov	sp, x1
+	adrp	x1, _arm_irq@page
+	add	x1, x1, _arm_irq@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el3_sp0_fiq_vector:
+	EL3_SP0_VECTOR
+	adrp	x1, _interrupt_stack_top@page
+	add	x1, x1, _interrupt_stack_top@pageoff
+	ldr	x1, [x1]
+	mov	sp, x1
+	adrp	x1, _arm_fiq@page
+	add	x1, x1, _arm_fiq@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el3_sp0_serror_vector:
+	EL3_SP0_VECTOR
+	mrs	x1, ESR_EL3					// Load exception syndrome
+	str	x1, [x0, #280]
+	mrs	x1, FAR_EL3					// Load fault address
+	str	x1, [x0, #272]
+	adrp	x1, _arm_serror_exception@page
+	add	x1, x1, _arm_serror_exception@pageoff
+	b	__dispatch64
+
+	.text
+	.align 7
+L__el3_sp3_synchronous_vector:
+	FASTSIM_DEBUG_HINT
+    	b       .
+
+	.text
+	.align 7
+L__el3_sp3_irq_vector:
+	FASTSIM_DEBUG_HINT
+	b       .
+
+	.text
+	.align 7
+L__el3_sp3_fiq_vector:
+	FASTSIM_DEBUG_HINT
+	b       .
+
+	.text
+	.align 7
+L__el3_sp3_serror_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_64_synchronous_vector:
+	// Monitor (EL3) call.
+	// Check "smc #0x5ec3" was executed and no other reason code.
+	movz	x1, #0x5ec3
+	movk	x1, #0x5e00, lsl #16
+	mrs	x2, ESR_EL3
+	cmp	x1, x2
+	b.eq	1f
+	FASTSIM_DEBUG_HINT
+	b	.
+1:	// Got here from an "smc". Branch to x0.
+	br	x0
+
+	.text
+	.align 7
+L__elx_64_irq_vector:
+	FASTSIM_DEBUG_HINT
+	b       .
+
+	.text
+	.align 7
+L__elx_64_fiq_vector:
+	FASTSIM_DEBUG_HINT
+	b       .
+
+	.text
+	.align 7
+L__elx_64_serror_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+	.text
+	.align 7
+L__elx_32_synchronous_vector:
+	FASTSIM_DEBUG_HINT
+    	b       .
+
+	.text
+	.align 7
+L__elx_32_irq_vector:
+	FASTSIM_DEBUG_HINT
+	b       .
+
+	.text
+	.align 7
+L__elx_32_fiq_vector:
+	FASTSIM_DEBUG_HINT
+	b       .
+
+	.text
+	.align 7
+L__elx_32_serror_vector:
+	FASTSIM_DEBUG_HINT
+	b	.
+
+/* Placeholder for macho_post_process.py. Placing the UUID here
+   allows debuggers to locate the UUID based on the value of VBAR */
+	.text
+	.align 7
+_UUID:
+	.space 32
+
+/* 64-bit first level exception handler dispatcher.
+* Completes register context saving and branches to FLEH.
+* Expects:
+*  {x0, x1, fp, lr, sp} - saved
+*  x0 - arm_context_t
+*  x1 - address of FLEH
+*  fp - previous stack frame if EL1
+*  lr - unused
+*  sp - kernel stack
+*/
+	.text
+	.align 2
+__dispatch64:
+	stp	fp, lr, [sp, #-16]!		// Create a frame to help debuggers unwind
+	mov	fp, sp				// past the exception handler
+	stp	x2, x3, [x0, #16]               // Save remaining GPRs
+	stp	x4, x5, [x0, #32]
+	stp	x6, x7, [x0, #48]
+	stp	x8, x9, [x0, #64]
+	stp	x10, x11, [x0, #80]
+	stp	x12, x13, [x0, #96]
+	stp	x14, x15, [x0, #112]
+	stp	x16, x17, [x0, #128]
+	stp	x18, x19, [x0, #144]
+	stp	x20, x21, [x0, #160]
+	stp	x22, x23, [x0, #176]
+	stp	x24, x25, [x0, #192]
+	stp	x26, x27, [x0, #208]
+	str	x28, [x0, #224]
+	mrs	lr, ELR_EL3                      // Get exception link register
+	str	lr, [x0, #264]                   // Save ELR to PCB
+	mrs	x2, SPSR_EL3                     // Get CPSR
+	str	w2, [x0, #256]                   // Save CPSR to PCB
+
+#if WITH_VFP
+	stp	q0, q1, [x0, #304]
+	stp	q2, q3, [x0, #336]
+	stp	q4, q5, [x0, #368]
+	stp	q6, q7, [x0, #400]
+	stp	q8, q9, [x0, #432]
+	stp	q10, q11, [x0, #464]
+	stp	q12, q13, [x0, #496]
+	stp	q14, q15, [x0, #528]
+	stp	q16, q17, [x0, #560]
+	stp	q18, q19, [x0, #592]
+	stp	q20, q21, [x0, #624]
+	stp	q22, q23, [x0, #656]
+	stp	q24, q25, [x0, #688]
+	stp	q26, q27, [x0, #720]
+	stp	q28, q29, [x0, #752]
+	stp	q30, q31, [x0, #784]
+	mrs	x2, FPSR
+	str	w2, [x0, #816]
+	mrs	x2, FPCR
+	str	w2, [x0, #820]
+#endif
+
+	mov	x21, x0                          // Copy arm_context_t pointer to x21
+	blr	x1
+	b	__exception_return
+	
+	.text
+	.align 2
+__exception_return:
+	mov     sp, x21                          // Reload the arm_context_t pointer
+
+#if WITH_VFP
+	ldp	q0, q1, [sp, #304]
+	ldp	q2, q3, [sp, #336]
+	ldp	q4, q5, [sp, #368]
+	ldp	q6, q7, [sp, #400]
+	ldp	q8, q9, [sp, #432]
+	ldp	q10, q11, [sp, #464]
+	ldp	q12, q13, [sp, #496]
+	ldp	q14, q15, [sp, #528]
+	ldp	q16, q17, [sp, #560]
+	ldp	q18, q19, [sp, #592]
+	ldp	q20, q21, [sp, #624]
+	ldp	q22, q23, [sp, #656]
+	ldp	q24, q25, [sp, #688]
+	ldp	q26, q27, [sp, #720]
+	ldp	q28, q29, [sp, #752]
+	ldp	q30, q31, [sp, #784]
+	ldr	w3, [sp, #816]
+	msr	FPSR, x3
+	ldr	w4, [sp, #820]
+	msr	FPCR, x4
+#endif
+
+	ldr     x0, [sp, #264]                   // Get the return address
+	msr     ELR_EL3, x0                      // Load the return address into ELR
+	ldr     w1, [sp, #256]                   // Get the return CPSR
+	msr     SPSR_EL3, x1                     // Load the return CPSR into SPSR
+	ldp     x0, x1, [sp, #0]                 // Restore the GPRs
+	ldp     x2, x3, [sp, #16]
+	ldp     x4, x5, [sp, #32]
+	ldp     x6, x7, [sp, #48]
+	ldp     x8, x9, [sp, #64]
+	ldp     x10, x11, [sp, #80]
+	ldp     x12, x13, [sp, #96]
+	ldp     x14, x15, [sp, #112]
+	ldp     x16, x17, [sp, #128]
+	ldp     x18, x19, [sp, #144]
+	ldp     x20, x21, [sp, #160]
+	ldp     x22, x23, [sp, #176]
+	ldp     x24, x25, [sp, #192]
+	ldp     x26, x27, [sp, #208]
+	ldr     x28, [sp, #224]
+	ldp     fp, lr, [sp, #232]
+
+	/* Use exception stack to restore SP0 */
+
+	msr     SPSel, #1			// Switch to SPx
+	stp     x0, x1, [sp, #-16]!		// Save x0 and x1
+	mrs     x0, SP_EL0			// Reload the pcb pointer from SP0
+	ldr     x1, [x0, #248]			// Get the saved SP from the pcb
+	msr     SP_EL0, x1			// Restore SP0
+	ldp     x0, x1, [sp], #16		// Restore x0 and x1
+	eret

arch/arm64/fp.c

@@ -0,0 +1,40 @@
+/*
+ * Copyright (C) 2012-2013 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+#include <arch.h>
+#include <arch/arm/arm.h>
+#include <debug.h>
+#include <sys/task.h>
+
+/*
+ * VFP/Neon support
+ */
+
+#if WITH_VFP
+
+extern uint32_t arm_read_cpacr();
+extern void arm_write_cpacr(uint32_t);
+
+void arm_fp_init(void)
+{
+	uint32_t cpacr;
+
+	/* v8: CPACR FPEN bits [21:20] needs to be programmed correctly to avoid a trap */
+	cpacr = arm_read_cpacr();
+	cpacr |= (3 << 20);
+	arm_write_cpacr(cpacr);
+}
+
+bool arch_task_fp_enable(bool enable)
+{
+	return true;
+}
+
+#endif /* WITH_VFP */

arch/arm64/handlers.c

@@ -0,0 +1,138 @@
+/*
+ * Copyright (C) 2011-2012 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <debug.h>
+#include <arch.h>
+#include <arch/arch_task.h>
+#include <arch/arm/arm.h>
+#include <arch/arm64/proc_reg.h>
+#include <sys.h>
+#include <lib/libc.h>
+#include <platform.h>
+
+extern void platform_irq(void);
+
+typedef enum {
+	ESR_EC_UNCATEGORIZED			= 0x00,
+	ESR_EC_TRAP_SIMD_FP			= 0x07,
+	ESR_EC_ILLEGAL_INSTR_SET		= 0x0c,
+	ESR_EC_SVC_32				= 0x11,
+	ESR_EC_SVC_64				= 0x15,
+	ESR_EC_MSR_TRAP				= 0x18,
+	ESR_EC_IABORT_EL0			= 0x20,
+	ESR_EC_IABORT_EL1			= 0x21,
+	ESR_EC_PC_ALIGN				= 0x22,
+	ESR_EC_DABORT_EL0			= 0x24,
+	ESR_EC_DABORT_EL1			= 0x25,
+	ESR_EC_SP_ALIGN				= 0x26
+} esr_exception_class_t;
+
+typedef enum {
+	FSC_TRANSLATION_FAULT_L0		= 0x04,
+	FSC_TRANSLATION_FAULT_L1		= 0x05,
+	FSC_TRANSLATION_FAULT_L2		= 0x06,
+	FSC_TRANSLATION_FAULT_L3		= 0x07,
+	FSC_ACCESS_FLAG_FAULT_L1		= 0x09,
+	FSC_ACCESS_FLAG_FAULT_L2		= 0x0A,
+	FSC_ACCESS_FLAG_FAULT_L3		= 0x0B,
+	FSC_PERMISSION_FAULT_L1			= 0x0D,
+	FSC_PERMISSION_FAULT_L2			= 0x0E,
+	FSC_PERMISSION_FAULT_L3			= 0x0F,
+	FSC_SYNC_EXT_ABORT			= 0x10,
+	FSC_ASYNC_EXT_ABORT			= 0x11,
+	FSC_SYNC_EXT_ABORT_TT_L1		= 0x15,
+	FSC_SYNC_EXT_ABORT_TT_L2		= 0x16,
+	FSC_SYNC_EXT_ABORT_TT_L3		= 0x17,
+	FSC_SYNC_PARITY				= 0x18,
+	FSC_ASYNC_PARITY			= 0x19,
+	FSC_SYNC_PARITY_TT_L1			= 0x1D,
+	FSC_SYNC_PARITY_TT_L2			= 0x1E,
+	FSC_SYNC_PARITY_TT_L3			= 0x1F,
+	FSC_ALIGNMENT_FAULT			= 0x21,
+	FSC_DEBUG_FAULT				= 0x22
+} fault_status_t;
+
+static const char *exception_codes[50] = {
+	[0x00] = "Unknown",
+	[0x0E] = "Illegal Instruction Set",
+	[0x21] = "Instruction Abort",
+	[0x22] = "PC Alignment",
+	[0x25] = "Data Abort",
+	[0x26] = "Stack Pointer Alignment",
+	[0x2F] = "SError",
+};
+
+void 
+arm_synchronous_exception(struct arm_exception_frame64 *frame)
+{
+	const char **kind;
+
+	kind = &exception_codes[ESR_EC(frame->esr)];
+
+	panic("ARM synchronous abort at 0x%016llx:\n"
+	      " esr 0x%08x (ec: %s, iss: 0x%08x) far 0x%016llx\n"
+	      " x0 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x4 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x8 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x12 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x16 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x20 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x24 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x28 0x%016llx sp 0x%016llx lr 0x%016llx spsr 0x%08x\n",
+	      frame->pc, 
+              frame->esr, *kind, ESR_ISS(frame->esr), frame->far,
+	      frame->regs[0], frame->regs[1], frame->regs[2], frame->regs[3],
+	      frame->regs[4], frame->regs[5], frame->regs[6], frame->regs[7],
+	      frame->regs[8], frame->regs[9], frame->regs[10], frame->regs[11], 
+	      frame->regs[12], frame->regs[13], frame->regs[14], frame->regs[15], 
+	      frame->regs[16], frame->regs[17], frame->regs[18], frame->regs[19], 
+	      frame->regs[20], frame->regs[21], frame->regs[22], frame->regs[23],
+	      frame->regs[24], frame->regs[25], frame->regs[26], frame->regs[27],
+	      frame->regs[28], frame->sp, frame->lr, frame->spsr);
+}
+
+void 
+arm_serror_exception(struct arm_exception_frame64 *frame)
+{
+	const char **kind;
+
+	kind = &exception_codes[ESR_EC(frame->esr)];
+
+	platform_asynchronous_exception();
+
+	panic("ARM SError abort at 0x%016llx:\n"
+	      " esr 0x%08x (ec: %s, iss: 0x%08x) far 0x%016llx\n"
+	      " x0 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x4 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x8 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x12 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x16 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x20 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x24 0x%016llx 0x%016llx 0x%016llx 0x%016llx\n"
+	      " x28 0x%016llx sp 0x%016llx lr 0x%016llx spsr 0x%08x\n",
+	      frame->pc, 
+              frame->esr, *kind, ESR_ISS(frame->esr), frame->far,
+	      frame->regs[0], frame->regs[1], frame->regs[2], frame->regs[3],
+	      frame->regs[4], frame->regs[5], frame->regs[6], frame->regs[7],
+	      frame->regs[8], frame->regs[9], frame->regs[10], frame->regs[11], 
+	      frame->regs[12], frame->regs[13], frame->regs[14], frame->regs[15], 
+	      frame->regs[16], frame->regs[17], frame->regs[18], frame->regs[19], 
+	      frame->regs[20], frame->regs[21], frame->regs[22], frame->regs[23],
+	      frame->regs[24], frame->regs[25], frame->regs[26], frame->regs[27],
+	      frame->regs[28], frame->sp, frame->lr, frame->spsr);
+}
+
+void
+arm_irq()
+{
+	_irq_enter_critical_section();
+	platform_irq();
+	_irq_exit_critical_section();
+}

arch/arm64/include/arch/arm64/proc_reg.h

@@ -0,0 +1,599 @@
+/*
+ * Copyright (c) 2007-2011, 2014 Apple Inc. All rights reserved.
+ */
+/*
+ * Processor registers for ARM64
+ */
+#ifndef _ARM64_PROC_REG_H_
+#define _ARM64_PROC_REG_H_
+
+// Pick the right registers depending on the exception level we're booting into.
+#if	WITH_EL3
+#define	SCTLR_ELx	SCTLR_EL3
+#define	MAIR_ELx	MAIR_EL3
+#define	TCR_ELx		TCR_EL3
+#define TTBR0_ELx	TTBR0_EL3
+#define	VBAR_ELx	VBAR_EL3
+#else
+#define	SCTLR_ELx	SCTLR_EL1
+#define	MAIR_ELx	MAIR_EL1
+#define	TCR_ELx		TCR_EL1
+#define TTBR0_ELx	TTBR0_EL1
+#define	VBAR_ELx	VBAR_EL1
+#endif
+
+/*
+ * 64-bit Program Status Register (PSR64)
+ *
+ *  31	    27	22 21 20 19	 10 9	    5 4	  0
+ * +-+-+-+-+------+--+--+----------+-+-+-+-+-+-----+
+ * |N|Z|C|V|000000|SS|IL|0000000000|D|A|I|F|0|	M  |
+ * +-+-+-+-+-+----+--+--+----------+-+-+-+-+-+-----+
+ *
+ * where:
+ *	NZCV	Comparison flags
+ *  SS		Single step
+ *	IL		Illegal state
+ *	DAIF	Interrupt masks
+ *	M		Mode field
+ */
+
+#define PSR64_NZCV_SHIFT		28
+#define PSR64_NZCV_MASK			(1 << PSR64_NZCV_SHIFT)
+
+#define PSR64_N_SHIFT			31
+#define PSR64_N				(1 << PSR64_N_SHIFT)
+
+#define PSR64_Z_SHIFT			30
+#define PSR64_Z				(1 << PSR64_Z_SHIFT)
+
+#define PSR64_C_SHIFT			29
+#define PSR64_C				(1 << PSR64_C_SHIFT)
+
+#define PSR64_V_SHIFT			28
+#define PSR64_V				(1 << PSR64_V_SHIFT)
+
+#define PSR64_SS_SHIFT			21
+#define PSR64_SS			(1 << PSR64_SS_SHIFT)
+
+#define PSR64_IL_SHIFT			20
+#define PSR64_IL			(1 << PSR64_IL_SHIFT)
+
+/*
+ * msr DAIF, Xn and mrs Xn, DAIF transfer into
+ * and out of bits 9:6
+ */
+#define DAIF_DEBUG_SHIFT		9
+#define DAIF_DEBUGF			(1 << DAIF_DEBUG_SHIFT)
+
+#define DAIF_ASYNC_SHIFT		8
+#define DAIF_ASYNCF			(1 << DAIF_ASYNC_SHIFT)
+
+#define DAIF_IRQF_SHIFT			7
+#define DAIF_IRQF			(1 << DAIF_IRQF_SHIFT)
+
+#define DAIF_FIQF_SHIFT			6
+#define DAIF_FIQF			(1 << DAIF_FIQF_SHIFT)
+
+#define DAIF_ALL			(DAIF_DEBUGF | DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
+#define DAIF_STANDARD_DISABLE		(DAIF_ASYNCF | DAIF_IRQF | DAIF_FIQF)
+
+#define SPSR_INTERRUPTS_ENABLED(x)	(!(x & DAIF_FIQF))
+
+/*
+ * msr DAIFSet, Xn, and msr DAIFClr, Xn transfer
+ * from bits 3:0.
+ */
+#define DAIFSC_DEBUGF			(1 << 3)
+#define DAIFSC_ASYNCF			(1 << 2)
+#define DAIFSC_IRQF			(1 << 1)
+#define DAIFSC_FIQF			(1 << 0)
+#define DAIFSC_ALL			(DAIFSC_DEBUGF | DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
+#define DAIFSC_STANDARD_DISABLE 	(DAIFSC_ASYNCF | DAIFSC_IRQF | DAIFSC_FIQF)
+
+/*
+ * ARM64_TODO: unify with ARM?
+ */
+#define PSR64_CF			0x20000000	/* Carry/Borrow/Extend */
+
+#define PSR64_MODE_MASK			0x1F
+
+#define PSR64_MODE_USER32_THUMB		0x20
+
+#define PSR64_MODE_RW_SHIFT		4
+#define PSR64_MODE_RW_64		0
+#define PSR64_MODE_RW_32		(0x1 << PSR64_MODE_RW_SHIFT)
+
+#define PSR64_MODE_EL_SHIFT		2
+#define PSR64_MODE_EL_MASK		(0x3 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL3			(0x3 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL1			(0x1 << PSR64_MODE_EL_SHIFT)
+#define PSR64_MODE_EL0			0
+
+#define PSR64_MODE_SPX			0x1
+#define PSR64_MODE_SP0			0
+
+#define PSR64_USER32_DEFAULT		(PSR64_MODE_RW_32 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
+#define PSR64_USER64_DEFAULT		(PSR64_MODE_RW_64 | PSR64_MODE_EL0 | PSR64_MODE_SP0)
+#define PSR64_KERNEL_DEFAULT		(DAIF_STANDARD_DISABLE | PSR64_MODE_RW_64 | PSR64_MODE_EL1 | PSR64_MODE_SP0)
+
+#define PSR64_IS_KERNEL(x)		((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL1)
+#define PSR64_IS_USER(x)		((x & PSR64_MODE_EL_MASK) == PSR64_MODE_EL0)
+
+
+/*
+ * Floating Point Control Register (FPCR)
+ */
+#define FPCR_DN				(1 << 25)
+
+#define FPCR_DEFAULT			(FPCR_DN)
+
+
+/*
+ * System Control Register (SCTLR)
+ *
+ *  31	  26  25 24  23	  19  18  15  14  13 12 11   7	 6    5	     4	 3  2 1 0
+ * +-----+---+--+---+----+---+---+---+---+--+--+----+---+----+------+---+--+-+-+-+
+ * |00110|UCI|EE|E0E|1101|WXN|101|UCT|DZE| 0|I |1000|ITD|THEE|C15BEN|SA0|SA|D|A|M|
+ * +-----+---+--+---+----+---+---+---+---+--+--+----+---+----+------+---+--+-+-+-+
+ *
+ *	UCI		User Cache Instructions
+ *	EE		Exception Endianness
+ *	E0E		EL0 Endianness
+ *	WXN		Writeable implies eXecute Never
+ *	UCT		User Cache Type register (CTR_EL0)
+ *	DZE	user Data Cache Zero (DC ZVA)
+ *	I		Instruction cache enable
+ *	ITD		IT Disable
+ *	THEE	THumb EE enable
+ *	CP15BEN CP15 Barrier ENable
+ *	SA0		Stack Alignment check for EL0
+ *	SA		Stack Alignment check enable
+ *	D		Data/unified cache enable
+ *	A		Alignment check enable
+ *	M		MMU enable
+ */
+#define SCTLR_RESERVED				0x30d50800
+
+#define SCTLR_UCI_ENABLED			(1 << 26)
+#define SCTLR_EE_ENABLED			(1 << 25)
+#define SCTLR_E0E_ENABLED			(1 << 24)
+#define SCTLR_WXN_ENABLED			(1 << 19)
+#define SCTLR_UCT_ENABLED			(1 << 15)
+#define SCTLR_DZE_ENABLED			(1 << 14)
+#define SCTLR_I_ENABLED				(1 << 12)
+#define SCTLR_ITD_DISABLED			(1 << 7)
+#define SCTLR_THEE_ENABLED			(1 << 6)
+#define SCTLR_CP15BEN_ENABLED			(1 << 5)
+#define SCTLR_SA0_ENABLED			(1 << 4)
+#define SCTLR_SA_ENABLED			(1 << 3)
+#define SCTLR_D_ENABLED				(1 << 2)
+#define SCTLR_A_ENABLED				(1 << 1)
+#define SCTLR_M_ENABLED				(1 << 0)
+
+#define SCTLR_EL1_DEFAULT			(SCTLR_RESERVED | SCTLR_I_ENABLED | SCTLR_SA_ENABLED | SCTLR_D_ENABLED | SCTLR_M_ENABLED)
+
+/*
+ * Secure Configuratio Register (SCR)
+ *
+ *  31			12		     0
+ * +---+---+------+------+--------------------+
+ * |00000000000000000000 ||
+ * +---+---+------+------+--------------------+
+ *
+ * where:
+ *	TTA		Trace trap
+ *	FPEN	Floating point enable
+ */
+#define SCR_EA_SHIFT		(3)
+#define SCR_EA			(1 << SCR_EA_SHIFT)
+
+#define SCR_FIQ_SHIFT		(2)
+#define SCR_FIQ			(1 << SCR_FIQ_SHIFT)
+
+#define SCR_IRQ_SHIFT		(1)
+#define SCR_IRQ			(1 << SCR_IRQ_SHIFT)
+
+#define SCR_DEFAULT		(SCR_EA | SCR_FIQ | SCR_IRQ)
+
+/*
+ * Coprocessor Access Control Register (CPACR)
+ *
+ *  31	28  27	22 21  20 19		     0
+ * +---+---+------+------+--------------------+
+ * |000|TTA|000000| FPEN |00000000000000000000|
+ * +---+---+------+------+--------------------+
+ *
+ * where:
+ *	TTA	Trace trap
+ *	FPEN	Floating point enable
+ */
+#define CPACR_TTA_SHIFT				28
+#define CPACR_TTA				(1 << CPACR_TTA_SHIFT)
+
+#define CPACR_FPEN_SHIFT			20
+#define CPACR_FPEN_EL0_TRAP			(0x1 << CPACR_FPEN_SHIFT)
+#define CPACR_FPEN_ENABLE			(0x3 << CPACR_FPEN_SHIFT)
+
+/*
+ * Translation Control Register (TCR) - EL3
+ *
+ *  31 30   24 23 22 21 20 19 18  16  14 13 12 11 10 9  8 7 6 5  0
+ * +-+-------+--+-----+---+--+-----+---+-----+-----+-----+-+-+----+
+ * |1|   0   | 1|  0  |TBI| 0|  PS |TG0| SH0 |ORGN0|IRGN0|0|0|T0SZ|
+ * +-+-------+--+-----+---+--+-----+---+-----+-----+-----+-+-+----+
+ *
+ *	TBI 	Top Byte Ignored for TTBR0 region
+ *	PS	Physical Address Size limit
+ *	TG0	Granule Size for TTBR0 region
+ *	SH0	Shareability for TTBR0 region
+ *	ORGN0	Outer Cacheability for TTBR0 region
+ *	IRGN0	Inner Cacheability for TTBR0 region
+ *	T0SZ	Virtual address size for TTBR0
+
+ * Translation Control Register (TCR) - EL1
+ *
+ *  63   39   38   37 36   34 32    30 29 28 27 26 25 24   23 22 21  16   14 13 12 11 10 9   8     7   5  0
+ * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
+ * |   0  |TBI1|TBI0|AS|0| IPS |0|TG1| SH1 |ORGN1|IRGN1|EPD1|A1| T1SZ |0|TG0| SH0 |ORGN0|IRGN0|EPD0|0|T0SZ|
+ * +------+----+----+--+-+-----+-+---+-----+-----+-----+----+--+------+-+---+-----+-----+-----+----+-+----+
+ *
+ *	TBI1	Top Byte Ignored for TTBR1 region
+ *	TBI0	Top Byte Ignored for TTBR0 region
+ *	AS	ASID Size
+ *	IPS	Physical Address Size limit
+ *	TG1	Granule Size for TTBR1 region
+ *	SH1	Shareability for TTBR1 region
+ *	ORGN1	Outer Cacheability for TTBR1 region
+ *	IRGN1	Inner Cacheability for TTBR1 region
+ *	EPD1	Translation table walk disable for TTBR1
+ *	A1	ASID selection from TTBR1 enable
+ *	T1SZ	Virtual address size for TTBR1
+ *	TG0	Granule Size for TTBR0 region
+ *	SH0	Shareability for TTBR0 region
+ *	ORGN0	Outer Cacheability for TTBR0 region
+ *	IRGN0	Inner Cacheability for TTBR0 region
+ *	T0SZ	Virtual address size for TTBR0
+ */
+
+// Common defintions for EL1 and EL3 versions of this register
+#define TCR_ELx_T0SZ_SHIFT		0
+
+#define TCR_ELx_IRGN0_SHIFT		8
+#define TCR_ELx_IRGN0_DISABLED		(0ULL << TCR_ELx_IRGN0_SHIFT)
+#define TCR_ELx_IRGN0_WRITEBACK		(1ULL << TCR_ELx_IRGN0_SHIFT)
+#define TCR_ELx_IRGN0_WRITETHRU		(2ULL << TCR_ELx_IRGN0_SHIFT)
+#define TCR_ELx_IRGN0_WRITEBACKNO	(3ULL << TCR_ELx_IRGN0_SHIFT)
+
+#define TCR_ELx_ORGN0_SHIFT		10
+#define TCR_ELx_ORGN0_DISABLED		(0ULL << TCR_ELx_ORGN0_SHIFT)
+#define TCR_ELx_ORGN0_WRITEBACK		(1ULL << TCR_ELx_ORGN0_SHIFT)
+#define TCR_ELx_ORGN0_WRITETHRU		(2ULL << TCR_ELx_ORGN0_SHIFT)
+#define TCR_ELx_ORGN0_WRITEBACKNO	(3ULL << TCR_ELx_ORGN0_SHIFT)
+
+#define TCR_ELx_SH0_SHIFT		12
+#define TCR_ELx_SH0_NONE		(0ULL << TCR_ELx_SH0_SHIFT)
+#define TCR_ELx_SH0_OUTER		(2ULL << TCR_ELx_SH0_SHIFT)
+#define TCR_ELx_SH0_INNER		(3ULL << TCR_ELx_SH0_SHIFT)
+
+#define TCR_ELx_TG0_GRANULE_SHIFT	14
+#define TCR_ELx_TG0_GRANULE_4KB		(0ULL << TCR_ELx_TG0_GRANULE_SHIFT)
+#define TCR_ELx_TG0_GRANULE_64KB	(1ULL << TCR_ELx_TG0_GRANULE_SHIFT)
+#define TCR_ELx_TG0_GRANULE_16KB	(2ULL << TCR_ELx_TG0_GRANULE_SHIFT)
+
+#if WITH_EL3
+// EL3-specific TCR register definitions
+
+#define TCR_EL3_PS_SHIFT		16
+#define TCR_EL3_PS_32BITS		(0ULL << TCR_EL3_PS_SHIFT)
+#define TCR_EL3_PS_36BITS		(1ULL << TCR_EL3_PS_SHIFT)
+#define TCR_EL3_PS_40BITS		(2ULL << TCR_EL3_PS_SHIFT)
+#define TCR_EL3_PS_42BITS		(3ULL << TCR_EL3_PS_SHIFT)
+#define TCR_EL3_PS_44BITS		(4ULL << TCR_EL3_PS_SHIFT)
+#define TCR_EL3_PS_48BITS		(5ULL << TCR_EL3_PS_SHIFT)
+
+#define TCR_EL3_TBI_TOPBYTE_IGNORED	(1ULL << 20)
+
+#else
+// EL1-specific TCR register definitions
+
+#define TCR_EL1_EPD0_TTBR0_DISABLED	(1ULL << 7)
+
+#define TCR_EL1_T1SZ_SHIFT		16
+
+#define TCR_EL1_A1_ASID1		(1ULL << 22)
+#define TCR_EL1_EPD1_TTBR1_DISABLED	(1ULL << 23)
+
+#define TCR_EL1_IRGN1_SHIFT		24
+#define TCR_EL1_IRGN1_DISABLED		(0ULL << TCR_EL1_IRGN1_SHIFT)
+#define TCR_EL1_IRGN1_WRITEBACK		(1ULL << TCR_EL1_IRGN1_SHIFT)
+#define TCR_EL1_IRGN1_WRITETHRU		(2ULL << TCR_EL1_IRGN1_SHIFT)
+#define TCR_EL1_IRGN1_WRITEBACKNO	(3ULL << TCR_EL1_IRGN1_SHIFT)
+
+#define TCR_EL1_ORGN1_SHIFT		26
+#define TCR_EL1_ORGN1_DISABLED		(0ULL << TCR_EL1_ORGN1_SHIFT)
+#define TCR_EL1_ORGN1_WRITEBACK		(1ULL << TCR_EL1_ORGN1_SHIFT)
+#define TCR_EL1_ORGN1_WRITETHRU		(2ULL << TCR_EL1_ORGN1_SHIFT)
+#define TCR_EL1_ORGN1_WRITEBACKNO	(3ULL << TCR_EL1_ORGN1_SHIFT)
+
+#define TCR_EL1_SH1_SHIFT		28
+#define TCR_EL1_SH1_NONE		(0ULL << TCR_EL1_SH1_SHIFT)
+#define TCR_EL1_SH1_OUTER		(2ULL << TCR_EL1_SH1_SHIFT)
+#define TCR_EL1_SH1_INNER		(3ULL << TCR_EL1_SH1_SHIFT)
+
+#define TCR_EL1_TG1_GRANULE_SHIFT	30
+#define TCR_EL1_TG1_GRANULE_16KB	(1ULL << TCR_EL1_TG1_GRANULE_SHIFT)
+#define TCR_EL1_TG1_GRANULE_4KB		(2ULL << TCR_EL1_TG1_GRANULE_SHIFT)
+#define TCR_EL1_TG1_GRANULE_64KB	(3ULL << TCR_EL1_TG1_GRANULE_SHIFT)
+
+#define TCR_EL1_IPS_SHIFT		32
+#define TCR_EL1_IPS_32BITS		(0ULL << TCR_EL1_IPS_SHIFT)
+#define TCR_EL1_IPS_36BITS		(1ULL << TCR_EL1_IPS_SHIFT)
+#define TCR_EL1_IPS_40BITS		(2ULL << TCR_EL1_IPS_SHIFT)
+#define TCR_EL1_IPS_42BITS		(3ULL << TCR_EL1_IPS_SHIFT)
+#define TCR_EL1_IPS_44BITS		(4ULL << TCR_EL1_IPS_SHIFT)
+#define TCR_EL1_IPS_48BITS		(5ULL << TCR_EL1_IPS_SHIFT)
+
+#define TCR_EL1_AS_16BIT_ASID		(1ULL << 36)
+#define TCR_EL1_TBI0_TOPBYTE_IGNORED	(1ULL << 37)
+#define TCR_EL1_TBI1_TOPBYTE_IGNORED	(1ULL << 38)
+
+#endif
+
+/*
+ * Translation Table Base Register (TTBR)
+ *
+ *  63	  48 47		      x x-1  0
+ * +--------+------------------+------+
+ * |  ASID  |	Base Address   | zero |
+ * +--------+------------------+------+
+ *
+ */
+#define TTBR_ASID_SHIFT			48
+#define TTBR_ASID_MASK			0xffff000000000000
+
+#define TTBR_BADDR_MASK			0x0000ffffffffffff
+
+/*
+ * Memory Attribute Indirection Register
+ *
+ *  63	 56 55	 48 47	 40 39	 32 31	 24 23	 16 15	  8 7	  0
+ * +-------+-------+-------+-------+-------+-------+-------+-------+
+ * | Attr7 | Attr6 | Attr5 | Attr4 | Attr3 | Attr2 | Attr1 | Attr0 |
+ * +-------+-------+-------+-------+-------+-------+-------+-------+
+ *
+ */
+
+#define MAIR_ATTR_SHIFT(x)			(8*(x))
+
+/* Strongly ordered or device memory attributes */
+#define MAIR_OUTER_STRONGLY_ORDERED		0x0
+#define MAIR_OUTER_DEVICE			0x0
+
+#define MAIR_INNER_STRONGLY_ORDERED		0x0
+#define MAIR_INNER_DEVICE			0x4
+
+/* Normal memory attributes */
+#define MAIR_OUTER_NON_CACHEABLE		0x40
+#define MAIR_OUTER_WRITE_THROUGH		0x80
+#define MAIR_OUTER_WRITE_BACK			0xc0
+
+#define MAIR_INNER_NON_CACHEABLE		0x4
+#define MAIR_INNER_WRITE_THROUGH		0x8
+#define MAIR_INNER_WRITE_BACK			0xc
+
+/* Allocate policy for cacheable memory */
+#define MAIR_OUTER_WRITE_ALLOCATE		0x10
+#define MAIR_OUTER_READ_ALLOCATE		0x20
+
+#define MAIR_INNER_WRITE_ALLOCATE		0x1
+#define MAIR_INNER_READ_ALLOCATE		0x2
+
+/* Memory Atribute Encoding */
+#define MAIR_DISABLE				0x04	/* Strongly Ordered */
+#define MAIR_WRITECOMB				0x44	/* Normal Memory, Outer Non-Cacheable, Inner Non-Cacheable */
+#define MAIR_WRITETHRU				0xBB	/* Normal Memory, Outer Write-through, Inner Write-through */
+#define MAIR_WRITEBACK				0xFF	/* Normal Memory, Outer Write-back, Inner Write-back */
+#define MAIR_INNERWRITEBACK			0x4F	/* Normal Memory, Outer Non-Cacheable, Inner Write-back */
+
+
+/*
+ *	ARM 4-level Page Table support - 2*1024TB (2^48) of address space
+ */
+
+
+/*
+ *  Memory Attribute Index
+ */
+#define CACHE_ATTRINDX_DISABLE			0x0	/* no cache, no buffer */
+#define CACHE_ATTRINDX_WRITEBACK		0x1	/* cache enabled, buffer enabled */
+#define CACHE_ATTRINDX_WRITECOMB		0x1	/* no cache, buffered writes */
+#define CACHE_ATTRINDX_WRITETHRU		0x2	/* cache enabled, buffer disabled */
+#define CACHE_ATTRINDX_INNERWRITEBACK		0x4	/* inner cache enabled, buffer enabled, write allocate */
+#define CACHE_ATTRINDX_DEFAULT			CACHE_ATTRINDX_WRITEBACK
+
+/*
+ *	Access protection bit values (TTEs and PTEs)
+ */
+#define AP_RWNA					0x0	/* priv=read-write, user=no-access */
+#define AP_RWRW					0x1	/* priv=read-write, user=read-write */
+#define AP_RONA					0x2	/* priv=read-only, user=no-access */
+#define AP_RORO					0x3	/* priv=read-only, user=read-only */
+#define AP_MASK					0x3	/* mask to find ap bits */
+
+/*
+ * Shareability attributes
+ */
+#define SH_NONE					0x0 	/* Non shareable  */
+#define SH_OUTER_MEMORY				0x2 	/* Normal memory Inner shareable - Outer shareable */
+#define SH_INNER_MEMORY				0x3 	/* Normal memory Inner shareable - Outer non shareable */
+
+/*
+ * LPAE/ARMv8 Table Entry
+ *
+ *  63 62 61 60	 59 58	 52 51	48 47		       12 11	2 1 0
+ * ,--+-----+--+---+-------+------+----------------------+-------+-+-.
+ * |NS|  AP |XN|PXN|ignored| zero | TableOutputAddress   |ignored|1|V|
+ * `--+-----+--+---+-------+------+----------------------+-------+-+-'
+ *
+ * LPAE/ARMv8 Block Entry
+ *
+ *  63 59 58 55 54  53	 52 51 48 47         12 11 10 9  8 7  6  5 4     2 1 0
+ * ,-----+-----+--+---+----+-----+-------------+--+--+----+----+--+-------+-+-.
+ * | ign |  SW |XN|PXN|HINT| zero|OutputAddress|nG|AF| SH | AP |NS|AttrIdx|W|V|
+ * `-----+-----+--+---+----+-----+-------------+--+--+----+----+--+-------+-+-'
+ *
+ * NS (NSTable)		Non-Secure: NS=1 for everything outside of TZ region
+ * AP (APTable)		Access Permission: AP[2:1]=0 for RW, 2 for RO
+ * XN (XNTable)		eXecute Never: XN=0 for code, 1 for data
+ * PXN (PXNTable)	Privileged eXecute Never: 1=dont execute when EL3
+ * TableOutputAddress	Next physical base: Lower bits 0 as alignment requires.
+ * SW			Reserved for software use (unused, can have metadata)
+ * HINT			Contiguous hint. Optimization only, leave as 0.
+ * OutputAddress	Block physical base: Lower bits 0 as alignment requires.
+ * nG			notGlobal: nG=1 restricts TLB hit to matching ASID
+ * AF			Access Flag: 0=never,1=hit. ARMv8 may fault if 0.
+ * SH			Shareability, SH[1:0]=2 for normal memory
+ * AttrIdx		Memory Attribute Index, 0 for device, 1 for normal
+ * W			"Walk" flag: L0-2 0=Block,1=Table; L3 0=Fault,1=Block
+ */
+
+#define ARM_LPAE_TABLE_MASK		(0x0000fffffffff000ULL) 	/* mask for extracting pointer to next table (works at any level) */
+
+#define ARM_LPAE_NSTABLE		0x8000000000000000ULL		/* value for a secure mapping */
+#define ARM_LPAE_NSTABLE_MASK		0x8000000000000000ULL		/* notSecure mapping mask */
+
+#define ARM_LPAE_APTABLE_SHIFT		61
+#define ARM_LPAE_APTABLE(x)		((x)<<ARM_LPAE_APTABLE_SHIFT) 	/* access protection */
+
+#define ARM_LPAE_XNTABLE		0x1000000000000000ULL		/* value for no execute */
+#define ARM_LPAE_XNTABLE_MASK		0x1000000000000000ULL		/* no execute mask */
+
+#define ARM_LPAE_PXNTABLE		0x0800000000000000ULL		/* value for privilege no execute bit */
+#define ARM_LPAE_PXNTABLE_MASK		0x0800000000000000ULL		/* privilege execute mask */
+
+#define ARM_LPAE_XN			0x0040000000000000ULL		/* value for no execute */
+#define ARM_LPAE_XN_MASK		0x0040000000000000ULL		/* no execute mask */
+
+#define ARM_LPAE_PXN			0x0020000000000000ULL		/* value for privilege no execute bit */
+#define ARM_LPAE_PXN_MASK		0x0020000000000000ULL		/* privilege execute mask */
+
+#define ARM_LPAE_NG			0x0000000000000800ULL		/* value for a global mapping */
+#define ARM_LPAE_NG_MASK		0x0000000000000800ULL		/* notGlobal mapping mask */
+
+#define ARM_LPAE_AF			0x0000000000000400ULL		/* value for access */
+#define ARM_LPAE_AF_MASK		0x0000000000000400ULL		/* access mask */
+
+#define ARM_LPAE_SH_SHIFT		8
+#define ARM_LPAE_SH(x)			((x) << ARM_LPAE_SH_SHIFT)	/* access shared */
+#define ARM_LPAE_SH_MASK		(0x3ULL << ARM_LPAE_SH_SHIFT)	/* mask access shared */
+
+#define ARM_LPAE_AP_SHIFT		6
+#define ARM_LPAE_AP(x)			((x)<<ARM_LPAE_AP_SHIFT) 	/* access protection */
+#define ARM_LPAE_AP_MASK		(0x3ULL<<ARM_LPAE_AP_SHIFT)
+
+#define ARM_LPAE_NS			0x0000000000000020ULL		/* value for a secure mapping */
+#define ARM_LPAE_MASK			0x0000000000000020ULL		/* notSecure mapping mask */
+
+#define ARM_LPAE_ATTRINDX_SHIFT		2				/* memory attributes index */
+#define ARM_LPAE_ATTRINDX(x)		((x) << ARM_LPAE_ATTRINDX_SHIFT)	/* memory attributes index */
+#define ARM_LPAE_ATTRINDX_MASK		(0x3ULL << ARM_LPAE_ATTRINDX_SHIFT)	/* mask memory attributes index */
+
+#define ARM_LPAE_TYPE_MASK		0x0000000000000002ULL		/* mask for extracting the type */
+#define ARM_LPAE_TYPE_TABLE		0x0000000000000002ULL		/* page table type */
+#define ARM_LPAE_TYPE_L0L1L2BLOCK	0x0000000000000000ULL		/* block entry type */
+#define ARM_LPAE_TYPE_L3BLOCK		0x0000000000000002ULL		/* L3 is different */
+
+#define ARM_LPAE_VALID			0x0000000000000001ULL		/* valid entry */
+
+/*
+ * Exception Syndrome Register
+ *
+ *  31	26 25 24	       0
+ * +------+--+------------------+
+ * |  EC  |IL|	     ISS	|
+ * +------+--+------------------+
+ *
+ *	EC - Exception Class
+ *	IL - Instruction Length
+ *  ISS- Instruction Specific Syndrome
+ *
+ * Note: The ISS can have many forms. These are defined separately below.
+ */
+
+#define ESR_EC_SHIFT			26
+#define ESR_EC_MASK			(0x3F << ESR_EC_SHIFT)
+#define ESR_EC(x)			((x & ESR_EC_MASK) >> ESR_EC_SHIFT)
+
+#define ESR_IL_SHIFT			25
+#define ESR_IL				(1 < ESR_IL_SHIFT)
+
+#define ESR_IS_THUMB(x)			(!(x & ESR_IL))
+
+#define ESR_ISS_MASK			0x01FFFFFF
+#define ESR_ISS(x)			(x & ESR_ISS_MASK)
+
+/*
+ * Instruction Abort ISS (EL1)
+ *  24		 10 9	   5	0
+ * +---------------+--+---+------+
+ * |000000000000000|EA|000| IFSC |
+ * +---------------+--+---+------+
+ *
+ * where:
+ *	EA		External Abort type
+ *	IFSC	Instruction Fault Status Code
+ */
+
+#define ISS_IA_EA_SHIFT				9
+#define ISS_IA_EA				(0x1 << ISS_IA_EA_SHIFT)
+
+#define ISS_IA_FSC_MASK				0x2F
+#define ISS_IA_FSC(x)				(x & ISS_DA_FSC_MASK)
+
+
+/*
+ * Data Abort ISS (EL1)
+ *
+ *  24		    9  8  7  6	5  0
+ * +---------------+--+--+-+---+----+
+ * |000000000000000|EA|CM|0|WnR|DFSC|
+ * +---------------+--+--+-+---+----+
+ *
+ * where:
+ *	EA		External Abort type
+ *	CM		Cache Maintenance operation
+ *	WnR		Write not Read
+ *	DFSC		Data Fault Status Code
+ */
+#define ISS_DA_EA_SHIFT				9
+#define ISS_DA_EA				(0x1 << ISS_DA_EA_SHIFT)
+
+#define ISS_DA_CM_SHIFT				8
+#define ISS_DA_CM				(0x1 << ISS_DA_CM_SHIFT)
+
+#define ISS_DA_WNR_SHIFT			6
+#define ISS_DA_WNR				(0x1 << ISS_DA_WNR_SHIFT)
+
+#define ISS_DA_FSC_MASK				0x2F
+#define ISS_DA_FSC(x)				(x & ISS_DA_FSC_MASK)
+
+#define PLATFORM_SYSCALL_TRAP_NO		0x80000000
+
+#define ARM64_SYSCALL_CODE_REG_NUM		(16)
+
+#define ARM64_CLINE_SHIFT			6
+
+/*
+ * Timer definitions.
+ */
+#define CNTKCTL_EL1_PL0PTEN			(0x1 << 9)		/* 1: EL0 access to physical timer regs permitted */
+#define CNTKCTL_EL1_PL0VTEN			(0x1 << 8)		/* 1: EL0 access to virtual timer regs permitted */
+#define CNTKCTL_EL1_EVENTI_MASK			(0x000000f0)		/* Mask for bits describing which bit to use for triggering event stream */
+#define CNTKCTL_EL1_EVENTI_SHIFT		(0x4)			/* Shift for same */
+#define CNTKCTL_EL1_EVENTDIR			(0x1 << 3)		/* 1: one-to-zero transition of specified bit causes event */
+#define CNTKCTL_EL1_EVNTEN			(0x1 << 2)		/* 1: enable event stream */
+#define CNTKCTL_EL1_PL0VCTEN			(0x1 << 1)		/* 1: EL0 access to physical timebase + frequency reg enabled */
+#define CNTKCTL_EL1_PL0PCTEN			(0x1 << 0)		/* 1: EL0 access to virtual timebase + frequency reg enabled */
+
+#endif /* _ARM64_PROC_REG_H_ */

arch/arm64/link.exp

@@ -0,0 +1 @@
+_start

arch/arm64/mmu.c

@@ -0,0 +1,543 @@
+/*
+ * Copyright (C) 2012-2014 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+/*
+ * See docs/mmu64.txt for an in-depth version of what is going on here.
+ * Short version:
+ * - H6, H7 use 4KB page granule.
+ * - H8 uses 16KB page granule (4KB not supported by hardware).
+ * - Start at L1 for 4KB page granule.
+ * - Start at L2 for 16KB page granule.
+ * - Set virtual memory limit so the first level (L0/L1) has just two entries.
+ * - ...then never touch the top level so we don't suffer from NS/S coherency.
+ */
+
+#include <debug.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <arch/arm/arm.h>
+#include <arch/arm64/proc_reg.h>
+#ifndef TEST
+# include <sys.h>
+# include <arch.h>
+# include <platform.h>
+# include <platform/memmap.h>
+#endif
+
+#define LAST_TABLE_LEVEL	3
+
+extern uint64_t arm_read_sctlr();
+
+// Start level page table entries
+static uint64_t *tt_top_level;
+
+// Allocation area and current allocation pointer.
+static addr_t tt_alloc_start, tt_alloc_next, tt_alloc_end;
+
+#ifdef TEST
+
+// Mock test interfaces for mmu.c, implemented in mmu_test.c
+
+void mmu_printf(int verbosity, const char *fmt, ...);
+unsigned get_page_granule_shift(void);
+void mmu_get_memmap(addr_t *page_tables_base,
+		    size_t *page_tables_size);
+void platform_mmu_setup(bool resume);
+void get_table_entry_hook(unsigned level, addr_t table_base, size_t index);
+void set_table_entry_hook(unsigned level, addr_t table_base, size_t index,
+			  uint64_t value);
+
+#else  // !defined(TEST)
+
+// Real interfaces, designed to be inlined. These have no host-side coverage,
+// so they should be no-ops or simple constants.
+
+// No printf spew.
+static void mmu_printf(int verbosity, const char *fmt, ...)
+{
+	(void) verbosity;
+	(void) fmt;
+}
+
+// Constants.
+static unsigned get_page_granule_shift(void) { return PAGE_GRANULE_SHIFT; }
+static void mmu_get_memmap(addr_t *page_tables_base,
+			   size_t *page_tables_size)
+{
+	*page_tables_base = PAGE_TABLES_BASE;
+	*page_tables_size = PAGE_TABLES_SIZE;
+}
+
+// Remove expensive validity checking.
+static void get_table_entry_hook(unsigned level, addr_t table_base,
+				 size_t index)
+{
+	(void) level; (void) table_base; (void) index;
+}
+static void set_table_entry_hook(unsigned level, addr_t table_base,
+				 size_t index, uint64_t value)
+{
+	(void) level; (void) table_base; (void) index; (void) value;
+}
+
+#endif  // !defined(TEST)
+
+#define mmu_spew(x...) mmu_printf(1, x)
+#define mmu_extra_spew(x...) mmu_printf(2, x)
+
+static void assert_valid_level(unsigned level)
+{
+	ASSERT((get_page_granule_shift() == 12 || level >= 1) &&
+	       level <= LAST_TABLE_LEVEL);
+}
+
+static bool level_permits_blocks(unsigned level)
+{
+	assert_valid_level(level);
+	// No granule supports L0 blocks.
+	// 4KB granule supports L1 blocks.
+	// All granules support L2 and L3 blocks.
+	return (level == 1 && get_page_granule_shift() == 12) || level >= 2;
+}
+
+static uint64_t get_table_entry(unsigned level, addr_t table_base,
+				size_t index)
+{
+	get_table_entry_hook(level, table_base, index);
+	mmu_extra_spew("get_table_entry(%u, 0x%llx, %u) ",
+		       level, (unsigned long long) table_base,
+		       (unsigned) index);
+	// No possible aliases, known aligned.
+	uint64_t result = *(uint64_t *) (table_base + index * 8);
+	mmu_extra_spew("= 0x%016llx\n", (unsigned long long) result);
+	return result;
+}
+
+static void set_table_entry(unsigned level, addr_t table_base, size_t index,
+			    uint64_t value)
+{
+	set_table_entry_hook(level, table_base, index, value);
+	mmu_extra_spew("set_table_entry(%u, 0x%llx, %u, 0x%llx)\n",
+		       level, (unsigned long long) table_base, (unsigned) index,
+		       (unsigned long long) value);
+	*((volatile uint64_t *) (table_base + index * 8)) = value;
+}
+
+static unsigned get_start_level(void)
+{
+	switch (get_page_granule_shift()) {
+	case 12: return 1;
+	case 14: return 2;
+	case 16: return 2;
+	default: panic("Bad granule");
+	}
+}
+
+static unsigned get_virt_address_bits(void)
+{
+	switch (get_page_granule_shift()) {
+	case 12: return 36;
+	case 14: return 36;
+	case 16: return 36;
+	default: panic("Bad granule");
+	}
+}
+
+static uint64_t get_mair(void)
+{
+	return (MAIR_WRITEBACK << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_WRITEBACK)) |
+		(MAIR_DISABLE << MAIR_ATTR_SHIFT(CACHE_ATTRINDX_DISABLE));
+}
+
+static uint64_t get_tcr(void)
+{
+	// Form the contents of the TCR register for the selected
+	// virtual address limit and page granule.
+	uint64_t tcr;
+	uint32_t txsz = 64 - get_virt_address_bits();
+#if WITH_EL3
+	switch (get_page_granule_shift()) {
+	case 12: tcr = TCR_ELx_TG0_GRANULE_4KB; break;
+	case 14: tcr = TCR_ELx_TG0_GRANULE_16KB; break;
+	case 16: tcr = TCR_ELx_TG0_GRANULE_64KB; break;
+	default: panic("Bad granule");
+	}
+	tcr |=	TCR_EL3_PS_36BITS |
+		TCR_ELx_SH0_OUTER | TCR_ELx_ORGN0_WRITEBACK | TCR_ELx_IRGN0_WRITEBACK |
+		(txsz << TCR_ELx_T0SZ_SHIFT);
+#else
+	switch (get_page_granule_shift()) {
+	case 12: tcr = TCR_ELx_TG0_GRANULE_4KB  | TCR_EL1_TG1_GRANULE_4KB;  break;
+	case 14: tcr = TCR_ELx_TG0_GRANULE_16KB | TCR_EL1_TG1_GRANULE_16KB; break;
+	case 16: tcr = TCR_ELx_TG0_GRANULE_64KB | TCR_EL1_TG1_GRANULE_64KB; break;
+	default: panic("Bad granule");
+	}
+	tcr |=	TCR_EL1_IPS_36BITS |
+		TCR_ELx_SH0_OUTER | TCR_ELx_ORGN0_WRITEBACK | TCR_ELx_IRGN0_WRITEBACK |
+		(txsz << TCR_ELx_T0SZ_SHIFT) |
+		TCR_EL1_EPD1_TTBR1_DISABLED |
+		TCR_EL1_SH1_OUTER | TCR_EL1_ORGN1_WRITEBACK | TCR_EL1_IRGN1_WRITEBACK |
+		(txsz << TCR_EL1_T1SZ_SHIFT);
+#endif
+	return tcr;
+}
+
+static uint64_t get_table_flags(void)
+{
+	// All table walks after the start level are non-secure.
+	return ARM_LPAE_NSTABLE | ARM_LPAE_TYPE_TABLE | ARM_LPAE_VALID;
+}
+
+static uint64_t get_block_flags(unsigned level, arm_mmu_extended_attr_t attr)
+{
+	assert_valid_level(level);
+	RELEASE_ASSERT(level_permits_blocks(level));
+	bool normal, writable, executable;
+	switch (attr) {
+	case kARMMMUDeviceR:
+		normal = false; writable = false; executable = false; break;
+	case kARMMMUDeviceRX:
+		normal = false; writable = false; executable = true; break;
+	case kARMMMUDeviceRW:
+		normal = false; writable = true;  executable = false; break;
+	case kARMMMUNormalR:
+		normal = true;  writable = false; executable = false; break;
+	case kARMMMUNormalRX:
+		normal = true;  writable = false; executable = true; break;
+	case kARMMMUNormalRW:
+		normal = true;  writable = true;  executable = false; break;
+	default:
+		panic("Bad attr %d", (int) attr);
+	}
+	// Always set Access Flag, Non-Secure and Valid.
+	uint64_t value = ARM_LPAE_AF | ARM_LPAE_NS | ARM_LPAE_VALID;
+	if (level < LAST_TABLE_LEVEL) {
+		value |= ARM_LPAE_TYPE_L0L1L2BLOCK;
+	} else {
+		value |= ARM_LPAE_TYPE_L3BLOCK;
+	}
+	uint64_t sh, attr_indx;
+	if (normal) {
+		sh = SH_OUTER_MEMORY;
+		attr_indx = CACHE_ATTRINDX_WRITEBACK;
+	} else {
+		sh = SH_NONE;
+		attr_indx = CACHE_ATTRINDX_DISABLE;
+	}
+	value |= ARM_LPAE_SH(sh);
+	value |= ARM_LPAE_ATTRINDX(attr_indx);
+	uint64_t ap = writable ? 0 : 2;
+	value |= ARM_LPAE_AP(ap);
+	if (!executable) {
+		value |= ARM_LPAE_PXN | ARM_LPAE_XN;
+	}
+	return value;
+}
+
+static uint64_t get_virt_size_bytes(void)
+{
+	return 1ULL << get_virt_address_bits();
+}
+
+static uint64_t get_phys_size_bytes(void)
+{
+	return 1ULL << 40;
+}
+
+static size_t get_level_shift(unsigned level)
+{
+	assert_valid_level(level);
+	switch (get_page_granule_shift()) {
+	case 12: return 39 - (level * 9);
+	case 14: return 47 - (level * 11);
+	case 16: return 55 - (level * 13);
+	default: panic("Granule 2^%u not supported\n",
+		       get_page_granule_shift());
+	}
+}
+
+static size_t get_level_entries(unsigned level)
+{
+	assert_valid_level(level);
+	// 1 granule per sub-table, 8 bytes per entry.
+	return ((size_t) 1) << (get_page_granule_shift() - 3);
+}
+
+static addr_t tt_alloc(unsigned level)
+{
+	// Allocate a translation table. The start level was
+	// allocated as tt_top_level in arm_mmu_init and should not be
+	// requested to be allocated. All levels are the
+	// same size: one granule.
+	RELEASE_ASSERT(level > get_start_level() && level <= LAST_TABLE_LEVEL);
+	size_t bytes = ((size_t) 1) << get_page_granule_shift();
+	mmu_spew("Allocate L%u size 0x%llx\n", level,
+		 (unsigned long long) bytes);
+	if (bytes > tt_alloc_end || tt_alloc_next > tt_alloc_end - bytes) {
+		panic("Out of table allocation space L%u\n", level);
+	}
+	addr_t ret = tt_alloc_next;
+	// Alignment is guaranteed by arm_mmu_init() and all
+	// allocations being the same size (1 granule), but it's cheap
+	// to double-check allocation alignment.
+	if ((ret & (bytes - 1)) != 0) {
+		panic("Bad aligned pte alloc 0x%llx\n",
+		      (unsigned long long) ret);
+	}
+	tt_alloc_next += bytes;
+	mmu_spew("Allocated L%u size 0x%llx at 0x%llx\n",
+		 level, (unsigned long long) bytes, (unsigned long long) ret);
+	return ret;
+}
+
+static size_t map_at_level(unsigned level, addr_t table_base,
+			   addr_t vaddr, addr_t paddr, size_t size,
+			   arm_mmu_extended_attr_t attr,
+			   bool blank_table)
+{
+	mmu_spew("map_at_level(level=%u,\n"
+		 "             table_base=0x%llx,\n"
+		 "             vaddr=0x%llx,\n"
+		 "             paddr=0x%llx,\n"
+		 "             size=0x%llx,\n"
+		 "             attr=%u\n",
+		 level, (unsigned long long) table_base,
+		 (unsigned long long) vaddr, (unsigned long long) paddr,
+		 (unsigned long long) size, (unsigned) attr);
+
+	// Check this mapping is bounded by virtual and physical address limits.
+	assert_valid_level(level);
+	RELEASE_ASSERT(size <= get_virt_size_bytes() &&
+	               vaddr <= get_virt_size_bytes() - size);
+	RELEASE_ASSERT(size <= get_phys_size_bytes() &&
+	               paddr <= get_phys_size_bytes() - size);
+
+	// Calculate constants about this translation level.
+	size_t level_shift = get_level_shift(level);
+	size_t level_entries = get_level_entries(level);
+	size_t level_size = 1ULL << level_shift;
+	size_t level_mask = level_size - 1;
+	bool permit_block = level_permits_blocks(level);
+	size_t index = (vaddr >> level_shift) & (level_entries - 1);
+	mmu_spew("shift:%d entries:%d size:0x%llx mask:0x%llx "
+		 "block:%d index:%d\n",
+		 (int) level_shift,
+		 (int) level_entries,
+		 (unsigned long long) level_size,
+		 (unsigned long long) level_mask,
+		 (int) permit_block,
+		 (int) index);
+
+	// Can we make block entries here? Must be permitted at this
+	// level, have enough bytes remaining, and both virtual and
+	// physical addresses aligned to a block.
+	if (permit_block &&
+	    size >= level_size &&
+	    ((vaddr | paddr) & level_mask) == 0) {
+		// Map contiguous blocks.
+		mmu_spew("Array of blocks\n");
+		size_t mapped_size = 0;
+		size_t remaining_size = size;
+		// Common block flags for all entries at this level.
+		uint64_t block_flags = get_block_flags(level, attr);
+		while (remaining_size >= level_size && index < level_entries) {
+			// Check the existing entry is unused.
+			uint64_t entry = get_table_entry(level, table_base,
+							 index);
+			if (entry != 0) {
+				// Not difficult but no use case.
+				panic("Remapping an existing L%u at 0x%016llx",
+				      level, (unsigned long long) vaddr);
+			}
+			// Form an entry.
+			uint64_t ent = block_flags | paddr;
+			// Write a block entry to the table.
+			set_table_entry(level, table_base, index, ent);
+			// Next index.
+			mapped_size += level_size;
+			remaining_size -= level_size;
+			paddr += level_size;
+			vaddr += level_size;
+			++index;
+		}
+		// Must make forward progress... and not too much.
+		RELEASE_ASSERT(mapped_size > 0 && mapped_size <= size);
+		return mapped_size;
+	} else {
+		// Sub-divide into a next level table.
+		mmu_spew("Sub-divide\n");
+
+		// Only so much granularity available.
+		RELEASE_ASSERT(level < LAST_TABLE_LEVEL);
+	
+		// Get the existing entry.
+		uint64_t entry = blank_table ? 0 : get_table_entry(level, table_base, index);
+		if ((entry & ARM_LPAE_VALID) &&
+		    (entry & ARM_LPAE_TYPE_MASK) == ARM_LPAE_TYPE_L0L1L2BLOCK) {
+			// Not hard but no use case.
+			panic("Breaking down blocks not implemented");
+		}
+
+		unsigned sub_level = level + 1;
+		bool blank_sub_table;
+		addr_t sub_base;
+		if (entry & ARM_LPAE_VALID) {
+			mmu_spew("Existing entry\n");
+			sub_base = entry & ARM_LPAE_TABLE_MASK;
+			blank_sub_table = false;
+		} else {
+			mmu_spew("Allocating sub-level table\n");
+			sub_base = tt_alloc(sub_level);
+			// Form an entry.
+			uint64_t ent = get_table_flags() | sub_base;
+			// Store an entry pointing to the sub-level table.
+			set_table_entry(level, table_base, index, ent);
+			blank_sub_table = true;
+		}
+		// Recurse at next level.
+		return map_at_level(sub_level, sub_base,
+				    vaddr, paddr, size, attr, blank_sub_table);
+	}
+}
+
+void arm_mmu_map_range(addr_t vaddr, addr_t paddr, size_t size,
+		       arm_mmu_extended_attr_t attr)
+{
+	// Ensure all addresses and sizes are at least L3 aligned.
+	size_t l3_align = ((size_t) 1) << get_level_shift(LAST_TABLE_LEVEL);
+	RELEASE_ASSERT(((vaddr | paddr | size) & (l3_align - 1)) == 0);
+	while (size > 0) {
+		// Map at the top level first, allowing the mapper function
+		// to break it down into smaller chunks as necessary.
+		size_t mapped = map_at_level(get_start_level(),
+					     (addr_t) tt_top_level,
+					     vaddr, paddr, size, attr, false);
+		RELEASE_ASSERT(mapped <= size);
+		size -= mapped;
+		vaddr += mapped;
+		paddr += mapped;
+	}
+
+	if ((arm_read_sctlr() & SCTLR_M_ENABLED) != 0) {
+		arm_flush_tlbs();
+	}
+}
+
+void arm_mmu_map_ro(addr_t addr, size_t size)
+{
+	arm_mmu_map_range(addr, addr, size, kARMMMUNormalR);
+}
+
+void arm_mmu_map_rw(addr_t addr, size_t size)
+{
+	arm_mmu_map_range(addr, addr, size, kARMMMUNormalRW);
+}
+
+void arm_mmu_map_rx(addr_t addr, size_t size)
+{
+	arm_mmu_map_range(addr, addr, size, kARMMMUNormalRX);
+}
+
+void arm_mmu_map_device_ro(addr_t addr, size_t size)
+{
+	arm_mmu_map_range(addr, addr, size, kARMMMUDeviceR);
+}
+
+void arm_mmu_map_device_rw(addr_t addr, size_t size)
+{
+	arm_mmu_map_range(addr, addr, size, kARMMMUDeviceRW);
+}
+
+void arm_mmu_map_device_rx(addr_t addr, size_t size)
+{
+	arm_mmu_map_range(addr, addr, size, kARMMMUDeviceRX);
+}
+
+void arm_mmu_init(bool resume)
+{
+	// Setup memory attributes.
+	arm_write_mair(get_mair());
+
+	// Setup translation control register.
+	arm_write_tcr(get_tcr());
+
+	/*
+	 * Sub-level page table allocations starts in the book-keeping
+	 * region after all stacks, rounded up. Allocation rounded down.
+	 */
+	size_t granule_mask = (((size_t) 1) << get_page_granule_shift()) - 1;
+	addr_t page_tables_base;
+	size_t page_tables_size;
+	mmu_get_memmap(&page_tables_base,
+		       &page_tables_size);
+
+	RELEASE_ASSERT((page_tables_base & granule_mask) == 0);
+	RELEASE_ASSERT((page_tables_size & granule_mask) == 0);
+
+	tt_alloc_start = page_tables_base;
+	tt_alloc_next = tt_alloc_start;
+	tt_alloc_end = (page_tables_base + page_tables_size) & ~granule_mask;
+
+	// allocate top-level page table
+	tt_top_level = (uint64_t *)tt_alloc_start;
+	tt_alloc_next += 1 << get_page_granule_shift();
+
+	// Defer to platform code for the memory map setup.
+	platform_mmu_setup(resume);
+
+	// Write the translation table base.
+	arm_write_ttbr0(tt_top_level);
+
+	// Flush TLBs.
+	arm_flush_tlbs();
+}
+
+
+#ifdef TEST
+// static->global stubs for mmu_test.c
+uint64_t *mmu_get_tt_top_level(void) { return tt_top_level; }
+unsigned mmu_get_start_level(void) { return get_start_level(); }
+uint64_t mmu_get_tcr(void) { return get_tcr(); }
+uint64_t mmu_tt_alloc(unsigned level) { return tt_alloc(level); }
+size_t mmu_get_level_entries(unsigned level) {
+	return get_level_entries(level);
+}
+uint64_t mmu_get_table_flags(void) {
+	return get_table_flags();
+}
+uint64_t mmu_get_block_flags(unsigned level, arm_mmu_extended_attr_t attr) {
+	return get_block_flags(level, attr);
+}
+
+size_t mmu_map_at_level(unsigned level, addr_t table_base,
+			addr_t vaddr, addr_t paddr, size_t size,
+			arm_mmu_extended_attr_t attr)
+{
+	return map_at_level(level, table_base, vaddr, paddr, size, attr, false);
+}
+
+void mmu_get_tt_alloc_range(addr_t *start, addr_t *next, addr_t *end)
+{
+	*start = tt_alloc_start;
+	*next = tt_alloc_next;
+	*end = tt_alloc_end;
+}
+
+void mmu_reset_state(void)
+{
+	tt_top_level = 0;
+	tt_alloc_start = 0;
+	tt_alloc_next = 0;
+	tt_alloc_end = 0;
+}
+#endif  // defined(TEST)

arch/arm64/mmu_test.c

@@ -0,0 +1,846 @@
+/*
+ * Copyright (C) 2014 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <stdbool.h>
+#include <assert.h>
+#include <inttypes.h>
+#include <setjmp.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <arch/arm/arm.h>
+#include <arch/arm64/proc_reg.h>
+
+#define VERBOSITY	FAILURE
+
+#define FAILURE		0
+#define INFO		1
+#define SPEW		2
+
+static size_t tests_run, tests_passed;
+
+enum Reason {
+	REASON_OK = 0,
+	REASON_EXPECTED_PANIC,
+	REASON_UNEXPECTED_PANIC,
+	REASON_PARAMS,
+	REASON_LEVEL,
+	REASON_OOM,
+	REASON_PTE_ALIGNMENT,
+	REASON_PTE_BASE,
+	REASON_PTE_INDEX,
+	REASON_PTE_ACCESS,
+	REASON_PTE_MISMATCH
+};
+
+static void test_unwind(enum Reason reason) __attribute__((noreturn));
+
+#define TEST_ASSERT(reason, c)						\
+	do {								\
+		if (!(c)) {						\
+			printf("!!! Test assertion failure: %s\n",	\
+			       #c );					\
+			test_unwind(reason);				\
+		}							\
+	} while (0)
+
+struct State {
+	// Environment constants for an individual test.
+	const char *test_name;
+	const char *expect_panic_func;
+	const char *expect_panic_substr;
+	size_t level_entries[4];
+	void *page_tables_base;
+	size_t page_tables_size;
+	size_t total_stack_size;
+	unsigned page_granule_shift;
+	jmp_buf jmp_env;
+
+	// Updated during test.
+	bool arm_flush_tlbs_called;
+
+	// Callbacks from test.
+	void (^mmu_setup_func)();
+	void (^get_table_entry_hook)();
+	void (^set_table_entry_hook)();
+};
+
+static struct State state;
+
+unsigned get_page_granule_shift(void) { return state.page_granule_shift; }
+
+void mmu_get_memmap(addr_t *page_tables_base,
+		    size_t *page_tables_size)
+{
+	*page_tables_base = (addr_t) state.page_tables_base;
+	*page_tables_size = state.page_tables_size;
+}
+
+// Access to mmu.c static functions and variables.
+uint64_t *mmu_get_tt_top_level(void);
+unsigned mmu_get_start_level(void);
+uint64_t mmu_get_tcr(void);
+addr_t mmu_tt_alloc(unsigned level);
+size_t mmu_get_level_entries(unsigned level);
+uint64_t mmu_get_table_flags(void);
+uint64_t mmu_get_block_flags(unsigned level, arm_mmu_extended_attr_t attr);
+size_t mmu_map_at_level(unsigned level, addr_t table_base,
+			addr_t vaddr, addr_t paddr, size_t size,
+			arm_mmu_extended_attr_t attr);
+void mmu_get_tt_alloc_range(addr_t *start, addr_t *next, addr_t *end);
+void mmu_reset_state(void);
+
+const char *reason_to_string(int reason)
+{
+	switch (reason) {
+	case REASON_OK: return "OK";
+	case REASON_EXPECTED_PANIC: return "Expected panic() called";
+	case REASON_UNEXPECTED_PANIC: return "Unexpected panic() called";
+	case REASON_PARAMS: return "Bad parameters detected";
+	case REASON_LEVEL: return "Invalid level used";
+	case REASON_OOM: return "Test ran out of memory for structures";
+	case REASON_PTE_ALIGNMENT: return "Bad page table alignment";
+	case REASON_PTE_BASE: return "Page table base does not match allocated";
+	case REASON_PTE_INDEX: return "Page table index out of bounds";
+	case REASON_PTE_ACCESS: return "Page table unexpected access";
+	case REASON_PTE_MISMATCH: return "Page table value mismatch";
+	default:
+		printf("Bad reason code %d\n", (int) reason);
+		abort();
+	}
+}
+
+void mmu_printf(int verbosity, const char *fmt, ...)
+{
+	if (VERBOSITY >= verbosity) {
+		va_list va;
+		va_start(va, fmt);
+		vprintf(fmt, va);
+		va_end(va);
+	}
+}
+
+static void test_unwind(enum Reason reason)
+{
+	assert(reason != REASON_OK);
+	longjmp(state.jmp_env, (int) reason);
+}
+
+void _panic(const char *func, const char *fmt, ...)
+{
+	char *buf = malloc(4096);
+	va_list va;
+
+	va_start(va, fmt);
+	vsnprintf(buf, 4096, fmt, va);
+	va_end(va);
+
+	bool expected = state.expect_panic_func && state.expect_panic_substr &&
+		strstr(func, state.expect_panic_func) != NULL &&
+		strstr(buf, state.expect_panic_substr) != NULL;
+
+	if (!expected) {
+		mmu_printf(FAILURE, "!!! Unexpected panic in function %s: %s\n", func, buf);
+	} else {
+		mmu_printf(INFO, "Expected panic in function %s: %s\n", func, buf);
+	}
+
+	free(buf);
+
+	if (expected) {
+		test_unwind(REASON_EXPECTED_PANIC);
+	} else if (strstr(buf, "Out of table allocation space")) {
+		test_unwind(REASON_OOM);
+	} else {
+		test_unwind(REASON_UNEXPECTED_PANIC);
+	}
+}
+
+static void expect_panic(const char *func, const char *substr)
+{
+	state.expect_panic_func = func;
+	state.expect_panic_substr = substr;
+}
+
+uint64_t check_valid_index(unsigned level, addr_t table_base, size_t index)
+{
+	if (level < mmu_get_start_level() || level > 3) {
+		mmu_printf(FAILURE, "L%u invalid level\n", level);
+		test_unwind(REASON_LEVEL);
+	}
+	void *p = NULL;
+
+	size_t table_size =
+		mmu_get_level_entries(level) * sizeof(uint64_t);
+	size_t table_entries = mmu_get_level_entries(level);
+	uint64_t alloc_start, alloc_next, alloc_end;
+	mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
+	if (table_base < alloc_start ||
+	    table_size > alloc_next ||
+	    table_base > alloc_next - table_size) {
+		mmu_printf(FAILURE, "L%u base 0x%" PRIx64 " size 0x%" PRIx64
+			   " outside allocated space\n",
+			   level,
+			   (uint64_t) table_base,
+			   (uint64_t) table_size);
+		test_unwind(REASON_PTE_BASE);
+	}
+	if (index > table_entries) {
+		mmu_printf(FAILURE, "L%u entry %u >= table_entries %u\n",
+			   level, (unsigned) index,
+			   (unsigned) table_entries);
+		test_unwind(REASON_PTE_INDEX);
+	}
+	p = (void *) (table_base + index * 8);
+
+	uint64_t result;
+	memcpy(&result, p, sizeof(result));
+	return result;
+}
+
+static void get_table_entry_internal(unsigned level, addr_t table_base,
+				     size_t index)
+{
+	mmu_printf(SPEW, "get_table_entry(%u, 0x%" PRIx64 ", %u) ",
+		   level, (uint64_t) table_base, (unsigned) index);
+	uint64_t result = check_valid_index(level, table_base, index);
+	mmu_printf(SPEW, "= 0x%016" PRIx64 "\n", result);
+}
+
+void get_table_entry_hook(unsigned level, addr_t table_base, size_t index)
+{
+	if (state.get_table_entry_hook) {
+		state.get_table_entry_hook(level, table_base, index);
+		return;
+	} else {
+		get_table_entry_internal(level, table_base, index);
+	}
+}
+
+static void set_table_entry_internal(unsigned level, addr_t table_base,
+				     size_t index, uint64_t value)
+{
+	mmu_printf(SPEW, "set_table_entry(%u, 0x%" PRIx64 ", %u, "
+		   "0x%" PRIx64 ") ",
+		   level, (uint64_t) table_base, (unsigned) index,
+		   value);
+	uint64_t old = check_valid_index(level, table_base, index);
+	mmu_printf(SPEW, " was 0x%016" PRIx64 "\n", old);
+}
+
+void set_table_entry_hook(unsigned level, addr_t table_base, size_t index,
+			  uint64_t value)
+{
+	if (state.set_table_entry_hook) {
+		state.set_table_entry_hook(level, table_base, index, value);
+		return;
+	} else {
+		set_table_entry_internal(level, table_base, index, value);
+	}
+}
+
+// Mock HAL.
+
+uint64_t arm_read_sctlr(void)
+{
+	mmu_printf(INFO, "arm_read_sctlr()\n");
+	return 0;
+}
+
+void arm_write_mair(uint64_t mair)
+{
+	mmu_printf(INFO, "arm_write_mair(0x%016" PRIx64 ")\n", (uint64_t) mair);
+}
+
+void arm_write_tcr(uint64_t tcr)
+{
+	mmu_printf(INFO, "arm_write_tcr(0x%016" PRIx64 ")\n", (uint64_t) tcr);
+}
+
+void arm_write_ttbr0(void *ttbr0)
+{
+	mmu_printf(INFO, "arm_write_ttbr0(%p)\n", ttbr0);
+	if ((addr_t) ttbr0 != (addr_t) mmu_get_tt_top_level()) {
+		mmu_printf(FAILURE, "TTBR0 written with %p expected %p\n",
+		           ttbr0, mmu_get_tt_top_level());
+		test_unwind(REASON_PTE_BASE);
+	}
+}
+
+void arm_flush_tlbs(void)
+{
+	mmu_printf(INFO, "arm_flush_tlbs()\n");
+	state.arm_flush_tlbs_called = true;
+}
+
+void platform_mmu_setup(bool resume)
+{
+	(void) resume;
+	uint64_t alloc_start, alloc_next, alloc_end;
+	mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
+	mmu_printf(INFO, "platform_mmu_setup(%d)\n", (int) resume);
+	mmu_printf(INFO, "tt_top_level = %p\n", mmu_get_tt_top_level());
+	mmu_printf(INFO, "page_tables_base = %p\n", state.page_tables_base);
+	mmu_printf(INFO, "page_tables_size = 0x%" PRIx64 "\n",
+	           (uint64_t) state.page_tables_size);
+	mmu_printf(INFO, "tt_alloc_start = 0x%" PRIx64 "\n"
+	           "tt_alloc_next = 0x%" PRIx64 "\n"
+	           "tt_alloc_end = 0x%" PRIx64 "\n",
+	           (uint64_t) alloc_start,
+	           (uint64_t) alloc_next,
+	           (uint64_t) alloc_end);
+	if (state.mmu_setup_func) {
+		state.mmu_setup_func();
+	}
+}
+
+// Test setup functions.
+
+static void reset_state(void)
+{
+	mmu_reset_state();
+	assert(state.page_tables_base == NULL);  // Prevent leak.
+	memset(&state, 0, sizeof(state));
+}
+
+static void mmu_setup_flat(void)
+{
+	uint64_t io_base = 0x200000000ULL;
+	uint64_t io_size = 256ULL * 1024 * 1024;
+	uint64_t sdram_base = 0x800000000ULL;
+	uint64_t sdram_len = 1ULL << 30;
+	arm_mmu_map_rw(sdram_base, sdram_len);
+	arm_mmu_map_device_rw(io_base, io_size);
+}
+
+static void mmu_setup_fail_remap_existing(void)
+{
+	uint64_t io_base = 0x200000000ULL;
+	uint64_t io_size = 256ULL * 1024 * 1024;
+	arm_mmu_map_rw(io_base, io_size);
+	// Map again
+	arm_mmu_map_device_rw(io_base, io_size);
+}
+
+static void mmu_setup_fail_breaking_down(void)
+{
+	uint64_t io_base = 0x200000000ULL;
+	uint64_t io_size = 256ULL * 1024 * 1024;
+	arm_mmu_map_rw(io_base, io_size);
+	// Map one L3 page again
+	arm_mmu_map_rw(io_base, 4096);
+}
+
+static void mmu_setup_misaligned(void)
+{
+	uint64_t sdram_base = 0x800000000ULL;
+	uint64_t sdram_len = 1ULL << 30;
+	arm_mmu_map_rw(sdram_base, sdram_len + 512);
+}
+
+static void test_delete(void)
+{
+	uint64_t alloc_start, alloc_next, alloc_end;
+	mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
+	mmu_printf(INFO, "Allocation space used: 0x%" PRIx64 "\n",
+	           alloc_next - alloc_start);
+	mmu_printf(INFO, "Allocation space remaining: 0x%" PRIx64 "\n",
+	           alloc_end - alloc_next);
+	if (state.page_tables_base) {
+		free(state.page_tables_base);
+		state.page_tables_base = NULL;
+	}
+	reset_state();
+}
+
+static int mmu_test(void)
+{
+	mmu_printf(INFO, "Test \"%s\" running\n", state.test_name);
+	int ret = posix_memalign(&state.page_tables_base,
+				 1 << state.page_granule_shift,
+				 state.page_tables_size);
+	assert(ret == 0);
+	memset(state.page_tables_base, 0, state.page_tables_size);
+	arm_mmu_init(false);
+	// Calls back on platform_mmu_setup(resume)
+	if (state.arm_flush_tlbs_called) {
+		return true;
+	} else {
+		mmu_printf(FAILURE, "!!! arm_flush_tlbs() was not called\n");
+		return false;
+	}
+}
+
+static void do_test(enum Reason expect_reason, int (^test_func)(void))
+{
+	assert(state.test_name != NULL);
+	mmu_printf(INFO, "----------------------------------------"
+	                 "--------------------------------------\n");
+	mmu_printf(INFO, "Test: %s\n", state.test_name);
+
+	// Increment number of tests run.
+	++tests_run;
+
+	// Set a jump environment for panic/abort exit path.
+	int reason_code = setjmp(state.jmp_env);
+	if (reason_code != 0) {
+		// Returns non-zero if longjmp() invoked to return here.
+		if (reason_code == (int) expect_reason) {
+			// Failure for an expected reason: exercising error
+			// detection.
+			mmu_printf(INFO, "Test \"%s\" aborted for expected reason: %s\n",
+			           state.test_name,
+			           reason_to_string(expect_reason));
+			++tests_passed;
+		} else {
+			mmu_printf(FAILURE, "!!! Test \"%s\" failed: %s\n",
+			           state.test_name,
+			           reason_to_string(reason_code));
+			mmu_printf(FAILURE, "!!! Expected outcome: %s\n",
+			           reason_to_string(expect_reason));
+		}
+		test_delete();
+		reset_state();
+		return;
+	}
+
+	if (test_func()) {
+		mmu_printf(INFO, "Test \"%s\" completed, passed\n",
+		           state.test_name);
+		++tests_passed;
+	} else {
+		mmu_printf(FAILURE, "!!! Test \"%s\" failed at completion\n",
+		           state.test_name);
+		mmu_printf(FAILURE, "!!! Expected outcome: %s\n",
+		           reason_to_string(expect_reason));
+	}
+
+	test_delete();
+	reset_state();
+}
+
+static void setup_standard_granule(size_t granule)
+{
+	switch (granule) {
+	case 4096:
+		state.page_granule_shift = 12;
+		state.page_tables_size = 4096 * 4;
+		break;
+
+	case 16384:
+		state.page_granule_shift = 14;
+		state.page_tables_size = 16384 * 4;
+		break;
+
+	case 65536:
+		state.page_granule_shift = 16;
+		state.page_tables_size = 65536 * 4;
+		break;
+
+	default:
+		mmu_printf(FAILURE, "Bad granule %u\n", (unsigned) granule);
+		abort();
+	}
+}
+
+static void test_granule(size_t bytes)
+{
+	state.mmu_setup_func = ^{ mmu_setup_flat(); };
+	setup_standard_granule(bytes);
+	do_test(REASON_OK, ^{ return mmu_test(); });
+}
+
+static void test_granule_4kb(void)
+{
+	state.test_name = "4KB granule";
+	test_granule(4096);
+}
+
+static void test_granule_16kb(void)
+{
+	state.test_name = "16KB granule";
+	test_granule(16384);
+}
+
+static void test_granule_64kb(void)
+{
+	state.test_name = "64KB granule";
+	test_granule(65536);
+}
+
+static void test_fail_oom(void)
+{
+	state.test_name = "Detect out-of-memory PTE allocation";
+	state.mmu_setup_func = ^{ mmu_setup_flat(); };
+	setup_standard_granule(4096);
+	// Too small.
+	state.page_tables_size = 16384;
+	expect_panic("tt_alloc", "Out of table allocation space L2");
+	do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
+}
+
+static void test_fail_range_misaligned(void)
+{
+	state.test_name = "Detect misaligned mapping request";
+	state.mmu_setup_func = ^{ mmu_setup_misaligned(); };
+	setup_standard_granule(4096);
+	// Expect vaddr, paddr, size to misaligned.
+	expect_panic("arm_mmu_map_range", "vaddr | paddr | size");
+	do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
+}
+
+static void test_fail_bad_granule(void)
+{
+	state.test_name = "Detect bad granule size";
+	state.mmu_setup_func = ^{
+		// Alter the granule to an illegal value.
+		++state.page_granule_shift;
+		mmu_setup_flat();
+	};
+	setup_standard_granule(4096);
+	expect_panic("get_level_shift", "Granule 2^13");
+	do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
+}
+
+static void test_fail_remap_existing(void)
+{
+	state.test_name = "Detect remapping existing page";
+	state.mmu_setup_func = ^{ mmu_setup_fail_remap_existing(); };
+	setup_standard_granule(4096);
+	expect_panic("map_at_level", "Remapping an existing L2");
+	do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
+}
+
+static void test_fail_breaking_down(void)
+{
+	state.test_name = "Detect breaking down existing mapping";
+	state.mmu_setup_func = ^{ mmu_setup_fail_breaking_down(); };
+	setup_standard_granule(4096);
+	expect_panic("map_at_level", "Breaking down");
+	do_test(REASON_EXPECTED_PANIC, ^{ return mmu_test(); });
+}
+
+static void test_get_tcr(void)
+{
+	state.test_name = "get_tcr()";
+	do_test(REASON_OK,
+		^{
+			state.page_granule_shift = 12;
+			mmu_get_tcr();
+			state.page_granule_shift = 14;
+			mmu_get_tcr();
+			state.page_granule_shift = 16;
+			mmu_get_tcr();
+			return true;
+		});
+}
+
+static void test_fail_get_tcr_bad_granule(void)
+{
+	state.test_name = "Detect get_tcr() with bad granule";
+	expect_panic("get_tcr", "Bad granule");
+	state.page_granule_shift = 13;
+	do_test(REASON_EXPECTED_PANIC, ^{ mmu_get_tcr(); return false; });
+}
+
+static void test_get_block_flags_for_level(unsigned level)
+{
+	// [63:59] SBZ
+	// [58:55] S/W use SBZ
+	// [54:53] XN,PXN = execute never, 0 for RX, 3 for R/RW
+	// [52]    Contiguous hint, SBZ
+	// [51:12] SBZ and address bits, also SBZ
+	// [11]    nG - non-global, we don't use ASID - SBZ
+	// [10]    AF - access flag, should set - SBO
+	// [9:8]   SH - shareability - 0 for device, 2 for normal
+	// [7:6]   AP - access permission - 0 for RW, 2 for RO
+	// [5]     NS - non-secure, should set - SBO
+	// [4:2]   AttrIndx - 0 for device, 1 for normal
+	// [1]     Walk - L0,L1,L2 0=Block; L3 1=Block
+	// [0]     V - 0=fault, 1=valid
+	const uint64_t sbo = (1 << 10) | (1 << 5) | (1 << 0);
+	const uint64_t normal = (2 << 8) | (1 << 2);
+	const uint64_t device = (0 << 8) | (0 << 2);
+	const uint64_t read_write = 0 << 6;
+	const uint64_t read_only = 2 << 6;
+	const uint64_t execute = 0ULL << 53;
+	const uint64_t no_execute = 3ULL << 53;
+	static const struct {
+		arm_mmu_extended_attr_t attr;
+		uint64_t expect;
+	} tests[] = {
+		{ kARMMMUDeviceR, sbo | device | read_only | no_execute },
+		{ kARMMMUDeviceRX, sbo | device | read_only | execute },
+		{ kARMMMUDeviceRW, sbo | device | read_write | no_execute },
+		{ kARMMMUNormalR, sbo | normal | read_only | no_execute },
+		{ kARMMMUNormalRX, sbo | normal | read_only | execute },
+		{ kARMMMUNormalRW, sbo | normal | read_write | no_execute },
+	};
+	for (size_t i = 0; i < sizeof(tests) / sizeof(tests[0]); ++i) {
+		arm_mmu_extended_attr_t attr = tests[i].attr;
+		uint64_t expect = tests[i].expect;
+		if (level == 3) {
+			expect |= 1 << 1;  // "Walk" flag
+		}
+		uint64_t value = mmu_get_block_flags(level, attr);
+		if (value != expect) {
+			mmu_printf(FAILURE, "get_block_flags level %u test %u enum %u\n",
+			           level, (unsigned) i, (unsigned) attr);
+			mmu_printf(FAILURE, "got 0x%" PRIx64 "\n", value);
+			mmu_printf(FAILURE, "expect 0x%" PRIx64 "\n", expect);
+			test_unwind(REASON_PTE_MISMATCH);
+		}
+	}
+}
+
+static void test_get_block_flags(void)
+{
+	state.test_name = "get_block_flags()";
+	state.page_granule_shift = 12;
+	do_test(REASON_OK, ^{
+			for (unsigned level = 1; level <= 3; ++level) {
+				test_get_block_flags_for_level(level);
+			}
+			return true;
+		});
+}
+
+static void test_get_table_flags(void)
+{
+	state.test_name = "get_tte_flags()";
+	state.page_granule_shift = 12;
+	// [63]    NS
+	// [62:61] AP
+	// [60]    XN
+	// [59]    PXN
+	// [58:52] ignored
+	// [51:48] zero, sbz
+	// [47:12] TableOutputAddress
+	// [11:2]  ignored
+	// [1]     type=1, sbo
+	// [0]     valid=1, sbo
+	const uint64_t expect = (1ULL << 63) | (1ULL << 1) | (1ULL << 0);
+	do_test(REASON_OK, ^{
+			uint64_t value = mmu_get_table_flags();
+			if (value != expect) {
+				mmu_printf(FAILURE, "get_table_flags() mismatch\n");
+				mmu_printf(FAILURE, "Got 0x%" PRIx64 "\n", value);
+				mmu_printf(FAILURE, "Expect 0x%" PRIx64 "\n", expect);
+				test_unwind(REASON_PTE_MISMATCH);
+			}
+			return true;
+		});
+}
+
+static void test_fail_get_block_flags_bad_attr(void)
+{
+	state.test_name = "Detect get_block_flags() bad attribute";
+	state.page_granule_shift = 12;
+	expect_panic("get_block_flags", "Bad attr");
+	do_test(REASON_EXPECTED_PANIC,
+		^{ mmu_get_block_flags(3, -1); return false; });
+}
+
+static void test_fail_get_block_flags_bad_level(void)
+{
+	state.test_name = "Detect get_pte_flags() bad level";
+	state.page_granule_shift = 16;
+	expect_panic("get_block_flags", "level_permits_blocks");
+	// Should not allow a block at L1 with 64KB granule.
+	do_test(REASON_EXPECTED_PANIC,
+		^{ mmu_get_block_flags(1, kARMMMUNormalRW); return false; });
+}
+
+static void test_tt_alloc(unsigned granule_shift)
+{
+	state.test_name = "tt_alloc()";
+	state.page_granule_shift = granule_shift;
+	state.page_tables_size = (1 << granule_shift) * 3;
+	state.mmu_setup_func = ^{};
+	do_test(REASON_OK,
+		^{
+			arm_mmu_init(false);
+			if (granule_shift == 12)
+				mmu_tt_alloc(2);
+			mmu_tt_alloc(3);
+			return true;
+		});
+}
+
+static void test_tt_alloc_bad_level(void)
+{
+	state.test_name = "Detect tt_alloc() bad level";
+	state.page_granule_shift = 14;
+	state.page_tables_size = 16384 * 3;
+	state.mmu_setup_func = ^{};
+	expect_panic("tt_alloc", "level > ");
+	do_test(REASON_EXPECTED_PANIC,
+		^{
+			arm_mmu_init(false);
+			mmu_tt_alloc(1);
+			return false;
+		});
+}
+
+static void test_tt_alloc_bad_alignment(void)
+{
+	state.test_name = "Detect tt_alloc() bad alignment";
+	state.page_granule_shift = 14;
+	state.page_tables_size = 65536 * 3;  // Enough space.
+	state.mmu_setup_func = ^{};
+	expect_panic("tt_alloc", "Bad aligned pte alloc");
+	do_test(REASON_EXPECTED_PANIC,
+		^{
+			arm_mmu_init(false);
+			mmu_tt_alloc(3);
+			// Allocation is aligned, so only way to trip it up
+			// is to artificially tamper with the granule.
+			state.page_granule_shift = 16;
+			mmu_tt_alloc(3);
+			return false;
+		});
+}
+
+static void test_map_at_top_level_large(void)
+{
+	state.test_name = "map_at_level() at top level traverses only L1";
+	setup_standard_granule(4096);
+	__block uint64_t alloc_start, alloc_next, alloc_end;
+	state.mmu_setup_func = ^{
+		mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
+		mmu_map_at_level(1,
+				 (addr_t) mmu_get_tt_top_level(),
+				 1ULL << 30, 1ULL << 30, 1ULL << 30,
+				 kARMMMUNormalRW);
+	};
+	__block size_t calls = 0;
+	state.set_table_entry_hook = ^(unsigned level,
+				       uint64_t base,
+				       size_t index,
+				       uint64_t value) {
+		(void) value;
+		++calls;
+		mmu_printf(INFO, "calls:%u %u 0x%" PRIx64 " 0x%" PRIx64
+		           " 0x%" PRIx64 "\n",
+		           (unsigned) calls,
+		           level, base, (uint64_t) index, value);
+		if (calls == 1) {
+			if (level != 1 ||
+			    base != (uint64_t) mmu_get_tt_top_level() ||
+			    index != 1) {
+				mmu_printf(FAILURE, "Did not write L0 index 0\n");
+				test_unwind(REASON_PTE_ACCESS);
+			}
+		} else {
+			mmu_printf(FAILURE, "Too many calls to get_table_entry()\n");
+			test_unwind(REASON_PTE_ACCESS);
+		}
+	};
+	do_test(REASON_OK,
+		^{
+			if (!mmu_test()) {
+				return false;
+			}
+			return true;
+		});
+}
+
+static void test_map_at_l3(void)
+{
+	state.test_name = "map_at_level() at l3";
+	setup_standard_granule(16384);
+	__block uint64_t alloc_start = 0, alloc_next = 0, alloc_end = 0;
+	__block uint64_t l3_base = 0;
+	state.mmu_setup_func = ^{
+		mmu_get_tt_alloc_range(&alloc_start, &alloc_next, &alloc_end);
+		l3_base = mmu_tt_alloc(3);
+		mmu_map_at_level(3, l3_base,
+				 (1ULL << 30) + 16384 * 3,
+				 16384 * 7,
+				 16384,
+				 kARMMMUDeviceRW);
+	};
+	__block size_t calls = 0;
+	state.set_table_entry_hook = ^(unsigned level,
+				       uint64_t base,
+				       size_t index,
+				       uint64_t value) {
+		(void) value;
+		++calls;
+		mmu_printf(INFO, "calls:%u %u 0x%" PRIx64 " 0x%" PRIx64
+		           " 0x%" PRIx64 "\n",
+		           (unsigned) calls,
+		           level, base, (uint64_t) index, value);
+		if (calls == 1) {
+			if (level != 3 ||
+			    base != l3_base ||
+			    index != 3) {
+				mmu_printf(FAILURE, "Did not write L3 index 3\n");
+				test_unwind(REASON_PTE_ACCESS);
+			}
+		} else {
+			mmu_printf(FAILURE, "Too many calls to get_table_entry()\n");
+			test_unwind(REASON_PTE_ACCESS);
+		}
+	};
+	do_test(REASON_OK,
+		^{
+			if (!mmu_test()) {
+				return false;
+			}
+			return true;
+		});
+}
+
+int main()
+{
+	reset_state();
+
+	test_granule_4kb();
+	test_granule_16kb();
+	test_granule_64kb();
+
+	test_fail_oom();
+	test_fail_range_misaligned();
+	test_fail_bad_granule();
+
+	test_fail_remap_existing();
+	test_fail_breaking_down();
+
+	test_get_tcr();
+	// test_fail_get_tcr_bad_granule();	!!!FIXME!!! <rdar://problem/19225114> Unexpected panic in function get_virt_address_bits: Bad granule
+
+	test_get_block_flags();
+	test_get_table_flags();
+	test_fail_get_block_flags_bad_attr();
+	test_fail_get_block_flags_bad_level();
+
+	test_tt_alloc(12);
+	test_tt_alloc(14);
+	test_tt_alloc(16);
+	test_tt_alloc_bad_level();
+	test_tt_alloc_bad_alignment();
+
+	test_map_at_top_level_large();
+	test_map_at_l3();
+
+	mmu_printf(INFO, "%" PRIu64 "/%" PRIu64 " tests passed\n",
+	           (uint64_t) tests_passed, (uint64_t) tests_run);
+	assert(tests_passed <= tests_run);
+	if (tests_passed == tests_run) {
+		mmu_printf(INFO, "Pass\n");
+		return 0;
+	} else {
+		mmu_printf(INFO, "!!! Fail\n");
+		return 1;
+	}
+}

arch/arm64/rules.mk

@@ -0,0 +1,176 @@
+# Copyright (C) 2011-2014 Apple Inc. All rights reserved.
+#
+# This document is the property of Apple Computer, Inc.
+# It is considered confidential and proprietary.
+#
+# This document may not be reproduced or transmitted in any form,
+# in whole or in part, without the express written permission of
+# Apple Computer, Inc.
+#
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+#
+# ARM CPU config 
+#
+
+# sanity check
+ifeq ($(ARM_CPU),)
+$(error ARM_CPU is not set)
+endif
+
+ifneq ($(ARM_ARCH),)
+$(info ARM_ARCH does not need to be set at the platform level)
+endif
+
+ARCH_ARMV8_CPUS := apple-cyclone apple-typhoon apple-typhoon-capri apple-twister apple-hurricane-zephyr
+
+################################################################################
+# Per-CPU configuration 
+################################################################################
+
+#
+# Apple Cyclone/Typhoon
+#
+ifneq (,$(filter $(ARCH_ARMV8_CPUS),$(ARM_CPU)))
+ARM_ARCH ?= arm64
+ARM_ARCH_OPTIONS := ARCH_ARMv8=1 WITH_VFP=1 WITH_VFP_ALWAYS_ON=1
+OPTIONS += \
+	CPU_APPLE_CYCLONE=1 \
+	L1_CACHELINE_SIZE=64 \
+	L1_CACHELINE_SHIFT=6 \
+	CPU_CACHELINE_SIZE=L1_CACHELINE_SIZE \
+	CPU_CACHELINE_SHIFT=L1_CACHELINE_SHIFT \
+	WITH_HW_TIMER=1
+
+ALL_OBJS += \
+	$(LOCAL_DIR)/mmu.o \
+	$(LOCAL_DIR)/timer.o
+
+ifneq (,$(filter apple-cyclone apple-typhoon,$(ARM_CPU)))
+WITH_EL3 := 1
+OPTIONS += \
+	L1_CACHEINDEX_SHIFT=9 \
+	L1_CACHEWAY_SHIFT=1 \
+	L2_CACHE_SIZE=1048576 \
+	L2_CACHEINDEX_SHIFT=11 \
+	L2_CACHEWAY_SHIFT=3 \
+	L2_CACHEWAY_COUNT=8 \
+	L2_CACHELINE_SHIFT=6 \
+	PAGE_GRANULE_SHIFT=12 \
+	PAGE_SIZE=4096 \
+	WITH_EL3=1 \
+	WITH_MCC_AS_RAM=1
+
+#4k-algined linker pages
+GLOBAL_LDFLAGS		+=	-Wl,-segalign,0x1000
+
+else ifneq (,$(filter apple-typhoon-capri,$(ARM_CPU)))
+WITH_EL3 := 1
+OPTIONS += \
+	L1_CACHEINDEX_SHIFT=9 \
+	L1_CACHEWAY_SHIFT=1 \
+	L2_CACHE_SIZE=2097152 \
+	L2_CACHEINDEX_SHIFT=12 \
+	L2_CACHEWAY_SHIFT=3 \
+	L2_CACHEWAY_COUNT=8 \
+	L2_CACHELINE_SHIFT=6 \
+	PAGE_GRANULE_SHIFT=12 \
+	PAGE_SIZE=4096 \
+	WITH_EL3=1 \
+	WITH_MCC_AS_RAM=1
+
+#4k-algined linker pages
+GLOBAL_LDFLAGS		+=	-Wl,-segalign,0x1000
+
+else ifneq (,$(filter apple-twister,$(ARM_CPU)))
+WITH_EL3 := 1
+OPTIONS += \
+	L1_CACHEINDEX_SHIFT=8 \
+	L1_CACHEWAY_SHIFT=2 \
+	L2_CACHE_SIZE=3145728 \
+	L2_CACHEINDEX_SHIFT=12 \
+	L2_CACHEWAY_SHIFT=4 \
+	L2_CACHEWAY_COUNT=12 \
+	L2_CACHELINE_SHIFT=6 \
+	PAGE_GRANULE_SHIFT=14 \
+	PAGE_SIZE=16384 \
+	WITH_EL3=1 \
+	WITH_MCC_AS_RAM=1
+
+#16k-algined linker pages
+GLOBAL_LDFLAGS		+=	-Wl,-segalign,0x4000
+
+else ifneq (,$(filter apple-hurricane-zephyr,$(ARM_CPU)))
+OPTIONS += \
+	L1_CACHEINDEX_SHIFT=8 \
+	L1_CACHEWAY_SHIFT=2 \
+	L2_CACHE_SIZE=3145728 \
+	L2_CACHEINDEX_SHIFT=11 \
+	L2_CACHEWAY_SHIFT=4 \
+	L2_CACHEWAY_COUNT=12 \
+	L2_CACHELINE_SHIFT=7 \
+	PAGE_GRANULE_SHIFT=14 \
+	PAGE_SIZE=16384 \
+	WITH_L2_AS_RAM=1
+
+#16k-algined linker pages
+GLOBAL_LDFLAGS		+=	-Wl,-segalign,0x4000
+
+endif
+
+endif
+
+################################################################################
+# General and conditional configuration
+################################################################################
+
+# sanity check
+ifeq ($(ARM_ARCH),)
+$(error ARM_ARCH not set, no valid CPU selected)
+endif
+
+# build for the proper cpu
+OPTIONS			+= ARCH_ARM=1
+GLOBAL_ALLFLAGS		+= -arch $(ARM_ARCH) $(addprefix -D,$(ARM_ARCH_OPTIONS))
+GLOBAL_LDFLAGS		+= -arch $(ARM_ARCH)
+
+# Tell the build system how libraries should be tagged
+LIBRARY_TAG		+= $(ARM_ARCH)
+LIBRARY_OPTIONS		+= ARCH=arm64 ARM_ARCH=$(ARM_ARCH)
+
+GLOBAL_INCLUDES		+= arch/arm/include
+GLOBAL_INCLUDES		+= $(LOCAL_DIR)/include
+
+# Defaults for stack sizes (stacks laid out from top to bottom as listed)
+INTERRUPT_STACK_SIZE	?= 8192
+BOOTSTRAP_STACK_SIZE	?= 4096
+EXCEPTION_STACK_SIZE	?= 4096
+OPTIONS += \
+	INTERRUPT_STACK_SIZE=$(INTERRUPT_STACK_SIZE) \
+	EXCEPTION_STACK_SIZE=$(EXCEPTION_STACK_SIZE) \
+	BOOTSTRAP_STACK_SIZE=$(BOOTSTRAP_STACK_SIZE)
+
+# Force start.o and asm.o to link first
+ALL_OBJS := \
+	$(LOCAL_DIR)/start.o \
+	$(LOCAL_DIR)/asm.o \
+	$(ALL_OBJS)
+
+# Code common to all cores
+ALL_OBJS += \
+	$(LOCAL_DIR)/cpu.o \
+	$(LOCAL_DIR)/context.o \
+	$(LOCAL_DIR)/entropy.o \
+	$(LOCAL_DIR)/fp.o \
+	$(LOCAL_DIR)/handlers.o \
+	$(LOCAL_DIR)/task.o
+
+ifeq ($(WITH_EL3),1)
+ALL_OBJS += \
+	$(LOCAL_DIR)/exceptions_el3.o
+else
+ALL_OBJS += \
+	$(LOCAL_DIR)/exceptions_el1.o
+endif
+
+LINKER_EXPORTS := $(LOCAL_DIR)/link.exp

arch/arm64/start.S

@@ -0,0 +1,262 @@
+/*
+ * Copyright (c) 2011-2014 Apple Inc.  All rights reserved.	
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+
+#include <arch/arm64/proc_reg.h>
+#include <platform/memmap.h>
+
+	.text
+	.align		12
+	.no_dead_strip 	_start
+	.globl 		_start	
+_start:
+	// Relocate, if needed
+	adrp	x0, _start@page
+	add	x0, x0, _start@pageoff
+	ldr	x1, L_TEXT_BASE
+
+#ifdef	PLATFORM_START_FUNCTION
+	/*
+	 * x0 and x1 must be preserved
+	 * 'eq' will be true if no text relocation is required
+	 */
+	bl	PLATFORM_START_FUNCTION
+#endif
+
+	cmp	x1, x0
+	b.eq	L__continue_start
+
+
+#if !CPU_APPLE_CYCLONE
+	mov	x2, #(SCTLR_I_ENABLED)
+	msr	SCTLR_ELx, x2
+	ic	iallu
+#endif
+
+	mov	lr, x1
+	ldr	x2, L_TEXT_END
+	ldr	x3, L_TEXT_BASE
+	sub	x2, x2, x3
+L__relocate_loop:
+	ldp	x3, x4, [x0], #16
+	stp	x3, x4, [x1], #16
+	subs	x2, x2, #16
+	b.ne	L__relocate_loop
+	ret
+
+L__continue_start:
+	// Disable all interrupts
+	msr	DAIFSet, #(DAIFSC_ALL)
+
+	// Setup return address (virtual)
+	adrp	lr, __main@page
+	add	lr, lr, __main@pageoff
+
+	// Set up exception vectors
+	adrp	x10, _exception_vector_base@page
+	add	x10, x10, _exception_vector_base@pageoff
+	msr	VBAR_ELx, x10
+
+	// bzero stacks region
+	ldr	x10, L_STACKS_BASE
+	ldr	x11, L_STACKS_SIZE
+	add	x11, x11, x10
+	mov	x12, #0
+L__stacks_zero_loop:
+	stp	x12, x12, [x10], #16
+	cmp	x10, x11
+	b.ne	L__stacks_zero_loop
+
+	// bzero page tables region
+	ldr	x10, L_PAGE_TABLES_BASE
+	ldr	x11, L_PAGE_TABLES_SIZE
+	add	x11, x11, x10
+	mov	x12, #0
+L__tables_zero_loop:
+	stp	x12, x12, [x10], #16
+	cmp	x10, x11
+	b.ne	L__tables_zero_loop
+
+	// Set up stack pointers
+	ldr	x10, L_STACKS_BASE
+
+	// exception stack goes at very bottom
+	add	x10, x10, #(EXCEPTION_STACK_SIZE)
+	mov	sp, x10					// set SPx to exception stack
+
+	// bootstrap stack above exception stack
+	msr	SPSel, #0				// switch to SP0
+	add	x10, x10, #(BOOTSTRAP_STACK_SIZE)
+	mov	sp, x10					// set SP0 to bootstrap stack
+
+	// Save interrupt stack top, to be stored after bss is cleared
+	// interrupt stack goes at top so that it can share with bootstrap
+	// stack after bootstrap task exits
+	add	x20, x10, #(INTERRUPT_STACK_SIZE)
+
+	// SecureROM: relocate data to SRAM 
+#if APPLICATION_SECUREROM
+	ldr	x10, L_DATA_RO_START
+	mov	x11, #(PAGE_SIZE - 1)
+	add	x10, x10, x11
+	bic	x10, x10, x11
+	ldr	x11, L_DATA_BASE
+	cmp	x10, x11
+	b.eq	L__do_bss
+
+	ldr	x12, L_DATA_END
+L__relocate_data_loop:
+	ldp	x13, x14, [x10], #16
+	stp	x13, x14, [x11], #16
+	cmp	x11, x12
+	b.ne	L__relocate_data_loop
+#endif // APPLICATION_SECUREROM
+
+	// Clear the bss section
+	// We requested all the sections to be aligned by 64.
+	// Section length should be 4 byte aligned.
+L__do_bss:
+	ldr	x10, L_BSS_START
+	ldr	x11, L_BSS_END
+	mov	x12, #15
+	bic	x12, x11, x12
+	mov	x13, #0
+L__bss_loop_stp:
+	stp	x13, x13, [x10], #16
+	cmp	x10, x12
+	b.ne	L__bss_loop_stp
+	cmp	x11, x12
+	b.eq	L__bss_done
+L__bss_loop_word:
+	str	w13, [x10], #4
+	cmp	x10, x11
+	b.lt	L__bss_loop_word
+L__bss_done:
+
+	// Store interrupt stack top
+	adrp	x11, _interrupt_stack_top@page
+	add	x11, x11, _interrupt_stack_top@pageoff
+	str	x20, [x11]
+
+#if defined(HEAP_GUARD) && (HEAP_GUARD != SRAM_END)
+	// Clear the heap guard page, which will never be mapped
+	mov	x13, #0
+	ldr	x10, L_HEAP_GUARD
+	mov	x11, #(PAGE_SIZE)
+	add	x11, x10, x11
+L__loop_heap_guard:
+	stp	x13, x13, [x10], #16
+	stp	x13, x13, [x10], #16
+	stp	x13, x13, [x10], #16
+	stp	x13, x13, [x10], #16
+	cmp	x10, x11
+	b.ne	L__loop_heap_guard
+#endif
+
+#ifdef BOOT_TRAMPOLINE_BASE
+	// copy the trampoline into a non-writeable page
+	ldr	x3, L_boot_trampoline_start
+	ldr	x4, L_boot_trampoline_end
+	ldr	x5, L_boot_trampoline_dst
+1:
+	ldp	x6, x7, [x3], #16
+	stp	x6, x7, [x5], #16
+	cmp	x3, x4
+	b.lo	1b
+#endif
+
+
+	// Branch to main
+	ret
+
+
+	.org	0x200
+L__build_banner_string:
+	.ascii CONFIG_PROGNAME_STRING
+	.ascii " for "
+	.ascii CONFIG_BOARD_STRING
+	.ascii ", Copyright 2007-2016, Apple Inc."
+
+	.org	0x240
+L__build_style_string:
+	.ascii BUILD_STYLE
+
+/* This will get replaced by macho_post_process.py so that the build tag
+   is accurate even for iterative desktop builds that don't rebuild start.o */
+	.org	0x280
+_build_tag_string_contents:
+
+	.org	0x300
+	.globl	_build_banner_string
+_build_banner_string:
+	.8byte	L__build_banner_string
+
+	.balign 8
+	.globl	_build_style_string
+_build_style_string:
+	.8byte	L__build_style_string
+
+	.balign	8
+	.globl	_build_tag_string
+_build_tag_string:
+	.8byte	_build_tag_string_contents
+
+	.balign	8
+	// Linked start and end addresses of the text
+L_TEXT_BASE:
+	.8byte	(TEXT_BASE)
+L_TEXT_END:
+	.8byte	section$start$__DATA$__zerofill
+L_TEXT_SIZE:
+	.8byte	segment$end$__TEXT - segment$start$__TEXT
+
+	// start address of the data in the read only text */
+L_DATA_RO_START:
+	.8byte	segment$end$__TEXT
+
+	// Linked start and end addresses of the data
+L_DATA_BASE:
+	.8byte	segment$start$__DATA
+L_DATA_END:
+	.8byte	section$start$__DATA$__zerofill
+
+	// Linked start and end addresses of the zero-filled range
+L_BSS_START:
+	.8byte	section$start$__DATA$__zerofill
+L_BSS_END:
+	.8byte	segment$end$__DATA
+
+L_STACKS_BASE:
+	.8byte	(STACKS_BASE)
+L_STACKS_SIZE:
+	.8byte	(STACKS_SIZE)
+L_PAGE_TABLES_BASE:
+	.8byte	(PAGE_TABLES_BASE)
+L_PAGE_TABLES_SIZE:
+	.8byte	(PAGE_TABLES_SIZE)
+
+#if defined(HEAP_GUARD) && (HEAP_GUARD != SRAM_END)
+L_HEAP_GUARD:
+	.8byte	(HEAP_GUARD)
+#endif
+#ifdef BOOT_TRAMPOLINE_BASE
+L_boot_trampoline_start:
+	.8byte	_boot_handoff_trampoline
+L_boot_trampoline_end:
+	.8byte	_boot_handoff_trampoline_end
+L_boot_trampoline_dst:
+	.8byte	BOOT_TRAMPOLINE_BASE
+#endif
+
+	.data
+ 
+	.globl	_interrupt_stack_top
+_interrupt_stack_top:
+	.space	8

arch/arm64/task.c

@@ -0,0 +1,52 @@
+/*
+ * Copyright (C) 2006 Apple Computer, Inc. All rights reserved.
+ *
+ * This document is the property of Apple Computer, Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Computer, Inc.
+ */
+#include <debug.h>
+#include <sys/types.h>
+#include <sys.h>
+#include <sys/task.h>
+
+static void arch_task_trampoline(void) __noreturn;
+
+static void 
+arch_task_trampoline(void)
+{
+	/* exit the critical section we implicitly started with */
+	exit_critical_section();
+
+	dprintf(DEBUG_SPEW, "arch_task_trampoline: task %p (%s) routine %p, arg %p\n", 
+			current_task, current_task->name, current_task->routine, current_task->arg);
+
+	/* call the start routine */
+	int ret = current_task->routine(current_task->arg);
+
+	dprintf(DEBUG_SPEW, "arch_task_trampoline: task %p (%s) exits with %d\n",
+			current_task, current_task->name, ret);
+
+	task_exit(ret);
+
+	/* never return */
+}
+
+int 
+arch_task_create(struct task *t)
+{
+	int i;
+
+	for (i=0; i < 29; i++)
+		t->arch.regs[i] = 0;
+
+	/* set the default starting point to be arch_task_trampoline */
+	t->arch.lr = (uintptr_t)&arch_task_trampoline;
+	t->arch.sp = (uintptr_t)t->stack_base + t->stack_len;
+	t->arch.fp = 0;
+
+	return 0;
+}

arch/arm64/test.mk

@@ -0,0 +1,56 @@
+# Copyright (C) 2014 Apple Inc. All rights reserved.
+#
+# This document is the property of Apple Computer, Inc.
+# It is considered confidential and proprietary.
+#
+# This document may not be reproduced or transmitted in any form,
+# in whole or in part, without the express written permission of
+# Apple Computer, Inc.
+#
+
+CC			:= $(shell xcodebuild -sdk macosx.internal -find clang)
+GCOV			:= $(shell xcodebuild -sdk macosx.internal -find gcov)
+
+TEST_DIR		:= host_tests
+
+INCDIRS			:= . \
+			   include \
+			   ../arm/include \
+			   ../../include
+
+CFLAGS			:= -arch x86_64 \
+			   -std=c11 \
+			   -g -Wall -Werror -O \
+			   $(addprefix -I,$(INCDIRS)) \
+			   -DARCH_ARMv8 -DTEST
+
+TESTS			:= mmu_test
+
+TEST_PASSES		:= $(patsubst %,$(TEST_DIR)/%.pass,$(TESTS))
+
+.PHONY:			all
+all:			host-tests
+
+.PHONY:			host-tests
+host-tests:		$(TEST_PASSES)
+
+.PHONY:			clean
+clean:
+	rm -rf $(TEST_DIR)
+	rm -f *.gcno *.gcda
+
+$(TEST_DIR)/mmu_test.pass:	$(TEST_DIR)/mmu_test
+	@echo "Test running $(notdir $<)"
+	@rm -f mmu_test.gcda mmu.gcda
+	@$< 2>&1 1>$(TEST_DIR)/mmu_test.out || \
+		{ cat $(TEST_DIR)/mmu_test.out; false; }
+	@echo "Test passed $(notdir $<)"
+	@$(GCOV) -a mmu.c
+	@mv -f mmu.c.gcov $(TEST_DIR)
+	@grep '#####' $(TEST_DIR)/mmu.c.gcov || true
+	@touch $@
+
+$(TEST_DIR)/mmu_test:		mmu_test.c mmu.c
+	@echo "Build $(notdir $@)"
+	@mkdir -p $(dir $@)
+	@$(CC) -o $@ $^ $(CFLAGS) --coverage

arch/arm64/tests-mmu.mk

@@ -0,0 +1,30 @@
+# Copyright (C) 2014 Apple Inc. All rights reserved.
+#
+# This document is the property of Apple Inc.
+# It is considered confidential and proprietary.
+#
+# This document may not be reproduced or transmitted in any form,
+# in whole or in part, without the express written permission of
+# Apple Inc.
+#
+
+LOCAL_DIR := $(GET_LOCAL_DIR)
+
+TEST_NAME := arm64-mmu
+
+TEST_OBJS := \
+	$(LOCAL_DIR)/mmu.o \
+
+TEST_SUPPORT_OBJS := \
+	$(LOCAL_DIR)/mmu_test.o
+
+TEST_CFLAGS := \
+	-DCPU_CACHELINE_SIZE=64 \
+	-DARCH_ARMv8=1 \
+	-DTEST=1 \
+	-Iarch/arm64/include/ \
+	-Iarch/arm/include
+
+TEST_INCLUDES := \
+	arch/arm64/include/ \
+	arch/arm/include

arch/arm64/timer.c

@@ -0,0 +1,147 @@
+/*
+ * Copyright (C) 2011 Apple Inc. All rights reserved.
+ *
+ * This document is the property of Apple Inc.
+ * It is considered confidential and proprietary.
+ *
+ * This document may not be reproduced or transmitted in any form,
+ * in whole or in part, without the express written permission of
+ * Apple Inc.
+ */
+#include <arch.h>
+#include <debug.h>
+#include <platform.h>
+#include <platform/soc/hwclocks.h>
+#include <platform/timer.h>
+
+extern void arm_write_cntp_ctl(uint64_t);
+extern uint64_t arm_read_cntp_ctl();
+extern void arm_write_cntp_tval(uint64_t);
+extern uint64_t arm_read_cntpct();
+
+static void timer_deadline(void *arg);
+static void (* timer_deadline_func)(void);
+
+#define DECR_MAX_COUNT	(INT32_MAX)
+
+int timer_init(uint32_t timer)
+{
+	uint64_t	val;
+
+	val = (0<<0) | (1<<1);
+	arm_write_cntp_ctl(val);
+
+	return 0;
+}
+
+void timer_stop_all(void)
+{
+	uint64_t	val;
+
+	val = (0<<0) | (1<<1);
+	arm_write_cntp_ctl(val);
+}
+
+void timer_deadline_enter(uint64_t deadline, void (* func)(void))
+{
+	uint64_t	ticks;
+	uint64_t	decr;
+	uint64_t	val;
+
+	timer_deadline_func = func;
+	if (func) {
+		/* convert absolute deadline to relative time */
+		ticks = timer_get_ticks();
+		if (deadline < ticks) {
+//			printf(" %llu too soon (it is now %llu)\n", deadline, ticks);
+			deadline = 0;
+		} else {
+			deadline -= ticks;
+		}
+
+		/* clamp the deadline to our maximum */
+		decr = (deadline >= DECR_MAX_COUNT) ? DECR_MAX_COUNT : deadline;
+		// printf(" using decrementer count %u\n", decr);
+
+		/*
+		 * Reset the decrementer by programming the max count, clearing
+		 * any pending interrupt status, then programming the desired count.
+		 * Note the timer's value is a signed 32 bit number
+		 */
+		arm_write_cntp_tval(DECR_MAX_COUNT);
+		val = (1<<0) | (0<<1);
+		arm_write_cntp_ctl(val);
+		arm_write_cntp_tval(decr);
+	} else {
+		/* no deadline, disable the decrementer */
+		val = (0<<0) | (1<<1);
+		arm_write_cntp_ctl(val);
+	}
+}
+
+void arm_fiq()
+{
+	uint64_t	val;
+
+	_irq_enter_critical_section();
+
+	val = arm_read_cntp_ctl();
+	if (val & (1 << 2)) {
+		/* acknowledge the interrupt and disable the timer */
+		val = (0<<0) | (1<<1);
+		arm_write_cntp_ctl(val);
+
+		/* if we have a callback, invoke it now */
+		if (timer_deadline_func)
+			timer_deadline_func();
+	}
+
+	_irq_exit_critical_section();
+}
+
+#ifdef OSC_FREQ
+
+utime_t timer_ticks_to_usecs(uint64_t ticks)
+{
+#if (OSC_FREQ % 1000000) == 0
+	return ticks / (OSC_FREQ / 1000000);
+#else
+#error "The code below has overflow issues. Make sure you really want to use it"
+	return (ticks * 1000 * 1000) / OSC_FREQ;
+#endif
+}
+
+uint64_t timer_usecs_to_ticks(utime_t usecs)
+{
+#if (OSC_FREQ % 1000000) == 0
+	return usecs * (OSC_FREQ / 1000000);
+#else
+#error "The code below has overflow issues. Make sure you really want to use it"
+	uint64_t timer_scale = ((uint64_t)(OSC_FREQ) << 20) / (1000 * 1000);
+	return (ticks * timer_scale) >> 20;
+#endif
+}
+
+uint32_t timer_get_tick_rate(void)
+{
+	return OSC_FREQ;
+}
+
+#endif
+
+uint64_t timer_get_ticks(void)
+{
+	return (arm_read_cntpct());
+}
+
+void arm_no_wfe_spin(uint32_t usecs)
+{
+	uint64_t delta_time = (uint64_t)usecs * (OSC_FREQ / 1000000);
+	uint64_t start_time;
+
+	start_time = timer_get_ticks();
+	while (true) {
+		if ((timer_get_ticks() - start_time) > delta_time)
+			break;
+	}
+}