changelog shortlog tags branches changeset file revisions annotate raw help

Mercurial > hg > plan9front / sys/src/9/bcm64/l.s

revision 7199: ba62683c0e2d
child 7207: 19de954d8073
     1.1new file mode 100644
     1.2--- /dev/null
     1.3+++ b/sys/src/9/bcm64/l.s
     1.4@@ -0,0 +1,749 @@
     1.5+#include "mem.h"
     1.6+#include "sysreg.h"
     1.7+
     1.8+#undef	SYSREG
     1.9+#define	SYSREG(op0,op1,Cn,Cm,op2)	SPR(((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5))
    1.10+
    1.11+TEXT _start(SB), 1, $-4
    1.12+	MOV	$setSB-KZERO(SB), R28
    1.13+	BL	svcmode<>(SB)
    1.14+
    1.15+	/* use dedicated stack pointer per exception level */
    1.16+	MOVWU	$1, R1
    1.17+	MSR	R1, SPSel
    1.18+
    1.19+	BL	mmudisable<>(SB)
    1.20+
    1.21+	/* invalidate local caches */
    1.22+	BL	cachedinv(SB)
    1.23+	BL	cacheiinv(SB)
    1.24+
    1.25+	MOV	$(MACHADDR(0)-KZERO), R27
    1.26+	MRS	MPIDR_EL1, R1
    1.27+	ANDW	$(MAXMACH-1), R1
    1.28+	MOVWU	$MACHSIZE, R2
    1.29+	MULW	R1, R2, R2
    1.30+	SUB	R2, R27
    1.31+
    1.32+	ADD	$(MACHSIZE-16), R27, R2
    1.33+	MOV	R2, SP
    1.34+
    1.35+	CBNZ	R1, _startup
    1.36+
    1.37+	/* clear page table and machs */
    1.38+	MOV	$(L1-KZERO), R1
    1.39+	MOV	$(MACHADDR(-1)-KZERO), R2
    1.40+_zerol1:
    1.41+	MOV	ZR, (R1)8!
    1.42+	CMP	R1, R2
    1.43+	BNE	_zerol1
    1.44+
    1.45+	/* clear BSS */
    1.46+	MOV	$edata-KZERO(SB), R1
    1.47+	MOV	$end-KZERO(SB), R2
    1.48+_zerobss:
    1.49+	MOV	ZR, (R1)8!
    1.50+	CMP	R1, R2
    1.51+	BNE	_zerobss
    1.52+
    1.53+	/* setup page tables */
    1.54+	MOV	$(L1-KZERO), R0
    1.55+	BL	mmu0init(SB)
    1.56+
    1.57+	BL	cachedwbinv(SB)
    1.58+	BL	l2cacheuwbinv(SB)
    1.59+	SEVL
    1.60+_startup:
    1.61+	WFE
    1.62+	BL	mmuenable<>(SB)
    1.63+
    1.64+	MOV	$0, R26
    1.65+	ORR	$KZERO, R27
    1.66+	MSR	R27, TPIDR_EL1
    1.67+	MOV	$setSB(SB), R28
    1.68+
    1.69+	BL	main(SB)
    1.70+
    1.71+TEXT	stop<>(SB), 1, $-4
    1.72+_stop:
    1.73+	WFE
    1.74+	B	_stop
    1.75+
    1.76+TEXT sev(SB), 1, $-4
    1.77+	SEV
    1.78+	WFE
    1.79+	RETURN
    1.80+
    1.81+TEXT PUTC(SB), 1, $-4
    1.82+	MOVWU $(0x3F000000+0x215040), R14
    1.83+	MOVB R0, (R14)
    1.84+	RETURN
    1.85+
    1.86+TEXT svcmode<>(SB), 1, $-4
    1.87+	MSR	$0xF, DAIFSet
    1.88+	MRS	CurrentEL, R0
    1.89+	ANDW	$(3<<2), R0
    1.90+	CMPW	$(1<<2), R0
    1.91+	BEQ	el1
    1.92+	CMPW	$(2<<2), R0
    1.93+	BEQ	el2
    1.94+	B	stop<>(SB)
    1.95+el2:
    1.96+	MOV	$0, R0
    1.97+	MSR	R0, MDCR_EL2
    1.98+	ISB	$SY
    1.99+
   1.100+	/* HCR = RW, HCD, SWIO, BSU, FB */
   1.101+	MOVWU	$(1<<31 | 1<<29 | 1<<2 | 0<<10 | 0<<9), R0
   1.102+	MSR	R0, HCR_EL2
   1.103+	ISB	$SY
   1.104+
   1.105+	/* SCTLR = RES1 */
   1.106+	MOVWU	$(3<<4 | 1<<11 | 1<<16 | 1<<18 | 3<<22 | 3<<28), R0
   1.107+	ISB	$SY
   1.108+	MSR	R0, SCTLR_EL2
   1.109+	ISB	$SY
   1.110+
   1.111+	/* set VMID to zero */
   1.112+	MOV	$0, R0
   1.113+	MSR	R0, VTTBR_EL2
   1.114+	ISB	$SY
   1.115+
   1.116+	MOVWU	$(0xF<<6 | 4), R0
   1.117+	MSR	R0, SPSR_EL2
   1.118+	MSR	LR, ELR_EL2
   1.119+	ERET
   1.120+el1:
   1.121+	RETURN
   1.122+
   1.123+TEXT mmudisable<>(SB), 1, $-4
   1.124+#define SCTLRCLR \
   1.125+	/* RES0 */	( 3<<30 \
   1.126+	/* RES0 */	| 1<<27 \
   1.127+	/* UCI */	| 1<<26 \
   1.128+	/* EE */	| 1<<25 \
   1.129+	/* RES0 */	| 1<<21 \
   1.130+	/* E0E */	| 1<<24 \
   1.131+	/* WXN */	| 1<<19 \
   1.132+	/* nTWE */	| 1<<18 \
   1.133+	/* RES0 */	| 1<<17 \
   1.134+	/* nTWI */	| 1<<16 \
   1.135+	/* UCT */	| 1<<15 \
   1.136+	/* DZE */	| 1<<14 \
   1.137+	/* RES0 */	| 1<<13 \
   1.138+	/* RES0 */	| 1<<10 \
   1.139+	/* UMA */	| 1<<9 \
   1.140+	/* SA0 */	| 1<<4 \
   1.141+	/* SA */	| 1<<3 \
   1.142+	/* A */		| 1<<1 )
   1.143+#define SCTLRSET \
   1.144+	/* RES1 */	( 3<<28 \
   1.145+	/* RES1 */	| 3<<22 \
   1.146+	/* RES1 */	| 1<<20 \
   1.147+	/* RES1 */	| 1<<11 )
   1.148+#define SCTLRMMU \
   1.149+	/* I */		( 1<<12 \
   1.150+	/* C */		| 1<<2 \
   1.151+	/* M */		| 1<<0 )
   1.152+
   1.153+	/* initialise SCTLR, MMU and caches off */
   1.154+	ISB	$SY
   1.155+	MRS	SCTLR_EL1, R0
   1.156+	BIC	$(SCTLRCLR | SCTLRMMU), R0
   1.157+	ORR	$SCTLRSET, R0
   1.158+	ISB	$SY
   1.159+	MSR	R0, SCTLR_EL1
   1.160+	ISB	$SY
   1.161+
   1.162+	B	flushlocaltlb(SB)
   1.163+
   1.164+TEXT mmuenable<>(SB), 1, $-4
   1.165+	/* return to virtual */
   1.166+	ORR	$KZERO, LR
   1.167+	MOV	LR, -16(RSP)!
   1.168+
   1.169+	BL	cachedwbinv(SB)
   1.170+	BL	flushlocaltlb(SB)
   1.171+
   1.172+	/* memory attributes */
   1.173+#define MAIRINIT \
   1.174+	( 0xFF << MA_MEM_WB*8 \
   1.175+	| 0x33 << MA_MEM_WT*8 \
   1.176+	| 0x44 << MA_MEM_UC*8 \
   1.177+	| 0x00 << MA_DEV_nGnRnE*8 \
   1.178+	| 0x04 << MA_DEV_nGnRE*8 \
   1.179+	| 0x08 << MA_DEV_nGRE*8 \
   1.180+	| 0x0C << MA_DEV_GRE*8 )
   1.181+	MOV	$MAIRINIT, R1
   1.182+	MSR	R1, MAIR_EL1
   1.183+	ISB	$SY
   1.184+
   1.185+	/* translation control */
   1.186+#define TCRINIT \
   1.187+	/* TBI1 */	( 0<<38 \
   1.188+	/* TBI0 */	| 0<<37 \
   1.189+	/* AS */	| 0<<36 \
   1.190+	/* TG1 */	| (((3<<16|1<<14|2<<12)>>PGSHIFT)&3)<<30 \
   1.191+	/* SH1 */	| SHARE_INNER<<28 \
   1.192+	/* ORGN1 */	| CACHE_WB<<26 \
   1.193+	/* IRGN1 */	| CACHE_WB<<24 \
   1.194+	/* EPD1 */	| 0<<23 \
   1.195+	/* A1 */	| 0<<22 \
   1.196+	/* T1SZ */	| (64-EVASHIFT)<<16 \
   1.197+	/* TG0 */	| (((1<<16|2<<14|0<<12)>>PGSHIFT)&3)<<14 \
   1.198+	/* SH0 */	| SHARE_INNER<<12 \
   1.199+	/* ORGN0 */	| CACHE_WB<<10 \
   1.200+	/* IRGN0 */	| CACHE_WB<<8 \
   1.201+	/* EPD0 */	| 0<<7 \
   1.202+	/* T0SZ */	| (64-EVASHIFT)<<0 )
   1.203+	MOV	$TCRINIT, R1
   1.204+	MRS	ID_AA64MMFR0_EL1, R2
   1.205+	ANDW	$0xF, R2	// IPS
   1.206+	ADD	R2<<32, R1
   1.207+	MSR	R1, TCR_EL1
   1.208+	ISB	$SY
   1.209+
   1.210+	/* load the page tables */
   1.211+	MOV	$(L1TOP-KZERO), R0
   1.212+	ISB	$SY
   1.213+	MSR	R0, TTBR0_EL1
   1.214+	MSR	R0, TTBR1_EL1
   1.215+	ISB	$SY
   1.216+
   1.217+	/* enable MMU and caches */
   1.218+	MRS	SCTLR_EL1, R1
   1.219+	ORR	$SCTLRMMU, R1
   1.220+	ISB	$SY
   1.221+	MSR	R1, SCTLR_EL1
   1.222+	ISB	$SY
   1.223+
   1.224+	MOV	RSP, R1
   1.225+	ORR	$KZERO, R1
   1.226+	MOV	R1, RSP
   1.227+	MOV	(RSP)16!, LR
   1.228+	B	cacheiinv(SB)
   1.229+
   1.230+TEXT touser(SB), 1, $-4
   1.231+	MSR	$0x3, DAIFSet	// interrupts off
   1.232+	MOVWU	$0x10028, R1	// entry
   1.233+	MOVWU	$0, R2		// psr
   1.234+	MSR	R0, SP_EL0	// sp
   1.235+	MSR	R1, ELR_EL1
   1.236+	MSR	R2, SPSR_EL1
   1.237+	ERET
   1.238+
   1.239+TEXT cas(SB), 1, $-4
   1.240+TEXT cmpswap(SB), 1, $-4
   1.241+	MOVW	ov+8(FP), R1
   1.242+	MOVW	nv+16(FP), R2
   1.243+_cas1:
   1.244+	LDXRW	(R0), R3
   1.245+	CMP	R3, R1
   1.246+	BNE	_cas0
   1.247+	STXRW	R2, (R0), R4
   1.248+	CBNZ	R4, _cas1
   1.249+	MOVW	$1, R0
   1.250+	DMB	$ISH
   1.251+	RETURN
   1.252+_cas0:
   1.253+	CLREX
   1.254+	MOVW	$0, R0
   1.255+	RETURN
   1.256+
   1.257+TEXT tas(SB), 1, $-4
   1.258+TEXT _tas(SB), 1, $-4
   1.259+	MOVW	$0xdeaddead, R2
   1.260+_tas1:
   1.261+	LDXRW	(R0), R1
   1.262+	STXRW	R2, (R0), R3
   1.263+	CBNZ	R3, _tas1
   1.264+	MOVW	R1, R0
   1.265+
   1.266+TEXT coherence(SB), 1, $-4
   1.267+	DMB	$ISH
   1.268+	RETURN
   1.269+
   1.270+TEXT islo(SB), 1, $-4
   1.271+	MRS	DAIF, R0
   1.272+	AND	$(0x2<<6), R0
   1.273+	EOR	$(0x2<<6), R0
   1.274+	RETURN
   1.275+
   1.276+TEXT splhi(SB), 1, $-4
   1.277+	MRS	DAIF, R0
   1.278+	MSR	$0x2, DAIFSet
   1.279+	RETURN
   1.280+
   1.281+TEXT splfhi(SB), 1, $-4
   1.282+	MRS	DAIF, R0
   1.283+	MSR	$0x3, DAIFSet
   1.284+	RETURN
   1.285+
   1.286+TEXT spllo(SB), 1, $-4
   1.287+	MSR	$0x3, DAIFClr
   1.288+	RETURN
   1.289+
   1.290+TEXT splflo(SB), 1, $-4
   1.291+	MSR	$0x1, DAIFClr
   1.292+	RETURN
   1.293+
   1.294+TEXT splx(SB), 1, $-4
   1.295+	MSR	R0, DAIF
   1.296+	RETURN
   1.297+
   1.298+TEXT cycles(SB), 1, $-4
   1.299+TEXT lcycles(SB), 1, $-4
   1.300+	MRS	PMCCNTR_EL0, R0
   1.301+	RETURN
   1.302+
   1.303+TEXT setlabel(SB), 1, $-4
   1.304+	MOV	LR, 8(R0)
   1.305+	MOV	SP, R1
   1.306+	MOV	R1, 0(R0)
   1.307+	MOVW	$0, R0
   1.308+	RETURN
   1.309+
   1.310+TEXT gotolabel(SB), 1, $-4
   1.311+	MOV	8(R0), LR	/* link */
   1.312+	MOV	0(R0), R1	/* sp */
   1.313+	MOV	R1, SP
   1.314+	MOVW	$1, R0
   1.315+	RETURN
   1.316+
   1.317+TEXT returnto(SB), 1, $-4
   1.318+	MOV	R0, 0(SP)
   1.319+	RETURN
   1.320+
   1.321+TEXT getfar(SB), 1, $-4
   1.322+	MRS	FAR_EL1, R0
   1.323+	RETURN
   1.324+
   1.325+TEXT setttbr(SB), 1, $-4
   1.326+	DSB	$ISHST
   1.327+	MSR	R0, TTBR0_EL1
   1.328+	DSB	$ISH
   1.329+	ISB	$SY
   1.330+
   1.331+	B	cacheiinv(SB)
   1.332+
   1.333+TEXT magic(SB), 1, $-4
   1.334+	DSB	$SY
   1.335+	ISB	$SY
   1.336+	DSB	$SY
   1.337+	ISB	$SY
   1.338+	DSB	$SY
   1.339+	ISB	$SY
   1.340+	DSB	$SY
   1.341+	ISB	$SY
   1.342+	RETURN
   1.343+
   1.344+/*
   1.345+ * TLB maintenance operations.
   1.346+ * these broadcast to all cpu's in the cluser
   1.347+ * (inner sharable domain).
   1.348+ */
   1.349+TEXT flushasidva(SB), 1, $-4
   1.350+TEXT tlbivae1is(SB), 1, $-4
   1.351+	DSB	$ISHST
   1.352+	TLBI	R0, 0,8,3,1	/* VAE1IS */
   1.353+	DSB	$ISH
   1.354+	ISB	$SY
   1.355+	RETURN
   1.356+
   1.357+TEXT flushasidvall(SB), 1, $-4
   1.358+TEXT tlbivale1is(SB), 1, $-4
   1.359+	DSB	$ISHST
   1.360+	TLBI	R0, 0,8,3,5	/* VALE1IS */
   1.361+	DSB	$ISH
   1.362+	ISB	$SY
   1.363+	RETURN
   1.364+
   1.365+TEXT flushasid(SB), 1, $-4
   1.366+TEXT tlbiaside1is(SB), 1, $-4
   1.367+	DSB	$ISHST
   1.368+	TLBI	R0, 0,8,3,2	/* ASIDE1IS */
   1.369+	DSB	$ISH
   1.370+	ISB	$SY
   1.371+	RETURN
   1.372+
   1.373+TEXT flushtlb(SB), 1, $-4
   1.374+TEXT tlbivmalle1is(SB), 1, $-4
   1.375+	DSB	$ISHST
   1.376+	TLBI	R0, 0,8,3,0	/* VMALLE1IS */
   1.377+	DSB	$ISH
   1.378+	ISB	$SY
   1.379+	RETURN
   1.380+
   1.381+/*
   1.382+ * flush the tlb of this cpu. no broadcast.
   1.383+ */
   1.384+TEXT flushlocaltlb(SB), 1, $-4
   1.385+TEXT tlbivmalle1(SB), 1, $-4
   1.386+	DSB	$NSHST
   1.387+	TLBI	R0, 0,8,7,0	/* VMALLE1 */
   1.388+	DSB	$NSH
   1.389+	ISB	$SY
   1.390+	RETURN
   1.391+
   1.392+TEXT fpsaveregs(SB), 1, $-4
   1.393+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 0)  /* MOV { V0, V1, V2, V3  }, (R0)64! */
   1.394+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 4)  /* MOV { V4, V5, V6, V7  }, (R0)64! */
   1.395+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 8)  /* MOV { V8, V9, V10,V11 }, (R0)64! */
   1.396+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 12) /* MOV { V12,V13,V14,V15 }, (R0)64! */
   1.397+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 16) /* MOV { V16,V17,V18,V19 }, (R0)64! */
   1.398+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 20) /* MOV { V20,V21,V22,V23 }, (R0)64! */
   1.399+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 24) /* MOV { V24,V25,V26,V27 }, (R0)64! */
   1.400+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 28) /* MOV { V28,V29,V30,V31 }, (R0)64! */
   1.401+	RETURN
   1.402+
   1.403+TEXT fploadregs(SB), 1, $-4
   1.404+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 0)  /* MOV (R0)64!, { V0, V1, V2, V3  } */
   1.405+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 4)  /* MOV (R0)64!, { V4, V5, V6, V7  } */
   1.406+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 8)  /* MOV (R0)64!, { V8, V9, V10,V11 } */
   1.407+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 12) /* MOV (R0)64!, { V12,V13,V14,V15 } */
   1.408+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 16) /* MOV (R0)64!, { V16,V17,V18,V19 } */
   1.409+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 20) /* MOV (R0)64!, { V20,V21,V22,V23 } */
   1.410+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 24) /* MOV (R0)64!, { V24,V25,V26,V27 } */
   1.411+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 28) /* MOV (R0)64!, { V28,V29,V30,V31 } */
   1.412+	RETURN
   1.413+
   1.414+// syscall or trap from EL0
   1.415+TEXT vsys0(SB), 1, $-4
   1.416+	LSRW	$26, R0, R17	// ec
   1.417+	CMPW	$0x15, R17	// SVC trap?
   1.418+	BNE	_itsatrap	// nope.
   1.419+
   1.420+	MOV	R26, 224(RSP)	// special
   1.421+	MOV	R27, 232(RSP)	// special
   1.422+	MOV	R28, 240(RSP)	// sb
   1.423+	MOV	R29, 248(RSP)	// special
   1.424+
   1.425+	MRS	SP_EL0, R1
   1.426+	MRS	ELR_EL1, R2
   1.427+	MRS	SPSR_EL1, R3
   1.428+
   1.429+	MOV	R0, 288(RSP)	// type
   1.430+	MOV	R1, 264(RSP)	// sp
   1.431+	MOV	R2, 272(RSP)	// pc
   1.432+	MOV	R3, 280(RSP)	// psr
   1.433+
   1.434+	MOV	$setSB(SB), R28
   1.435+	MRS	TPIDR_EL1, R27
   1.436+	MOV	16(R27), R26
   1.437+
   1.438+	ADD	$16, RSP, R0	// ureg
   1.439+	BL	syscall(SB)
   1.440+
   1.441+TEXT forkret(SB), 1, $-4
   1.442+	MSR	$0x3, DAIFSet	// interrupts off
   1.443+
   1.444+	ADD	$16, RSP, R0	// ureg
   1.445+
   1.446+	MOV	16(RSP), R0	// ret
   1.447+	MOV	264(RSP), R1	// sp
   1.448+	MOV	272(RSP), R2	// pc
   1.449+	MOV	280(RSP), R3	// psr
   1.450+
   1.451+	MSR	R1, SP_EL0
   1.452+	MSR	R2, ELR_EL1
   1.453+	MSR	R3, SPSR_EL1
   1.454+
   1.455+	MOV	224(RSP), R26	// special
   1.456+	MOV	232(RSP), R27	// special
   1.457+	MOV	240(RSP), R28	// sb
   1.458+	MOV	248(RSP), R29	// special
   1.459+
   1.460+	MOV	256(RSP), R30	// link
   1.461+
   1.462+	ADD	$TRAPFRAMESIZE, RSP
   1.463+	ERET
   1.464+
   1.465+TEXT itsatrap<>(SB), 1, $-4
   1.466+_itsatrap:
   1.467+	MOV	R1, 24(RSP)
   1.468+	MOV	R2, 32(RSP)
   1.469+	MOV	R3, 40(RSP)
   1.470+	MOV	R4, 48(RSP)
   1.471+	MOV	R5, 56(RSP)
   1.472+	MOV	R6, 64(RSP)
   1.473+	MOV	R7, 72(RSP)
   1.474+	MOV	R8, 80(RSP)
   1.475+	MOV	R9, 88(RSP)
   1.476+	MOV	R10, 96(RSP)
   1.477+	MOV	R11, 104(RSP)
   1.478+	MOV	R12, 112(RSP)
   1.479+	MOV	R13, 120(RSP)
   1.480+	MOV	R14, 128(RSP)
   1.481+	MOV	R15, 136(RSP)
   1.482+	MOV	R16, 144(RSP)
   1.483+
   1.484+	MOV	R18, 160(RSP)
   1.485+	MOV	R19, 168(RSP)
   1.486+	MOV	R20, 176(RSP)
   1.487+	MOV	R21, 184(RSP)
   1.488+	MOV	R22, 192(RSP)
   1.489+	MOV	R23, 200(RSP)
   1.490+	MOV	R24, 208(RSP)
   1.491+	MOV	R25, 216(RSP)
   1.492+
   1.493+// trap/irq/fiq/serr from EL0
   1.494+TEXT vtrap0(SB), 1, $-4
   1.495+	MOV	R26, 224(RSP)	// special
   1.496+	MOV	R27, 232(RSP)	// special
   1.497+	MOV	R28, 240(RSP)	// sb
   1.498+	MOV	R29, 248(RSP)	// special
   1.499+
   1.500+	MRS	SP_EL0, R1
   1.501+	MRS	ELR_EL1, R2
   1.502+	MRS	SPSR_EL1, R3
   1.503+
   1.504+	MOV	R0, 288(RSP)	// type
   1.505+	MOV	R1, 264(RSP)	// sp
   1.506+	MOV	R2, 272(RSP)	// pc
   1.507+	MOV	R3, 280(RSP)	// psr
   1.508+
   1.509+	MOV	$setSB(SB), R28
   1.510+	MRS	TPIDR_EL1, R27
   1.511+	MOV	16(R27), R26
   1.512+
   1.513+	ADD	$16, RSP, R0	// ureg
   1.514+	BL	trap(SB)
   1.515+
   1.516+TEXT noteret(SB), 1, $-4
   1.517+	MSR	$0x3, DAIFSet	// interrupts off
   1.518+
   1.519+	ADD	$16, RSP, R0	// ureg
   1.520+
   1.521+	MOV	264(RSP), R1	// sp
   1.522+	MOV	272(RSP), R2	// pc
   1.523+	MOV	280(RSP), R3	// psr
   1.524+
   1.525+	MSR	R1, SP_EL0
   1.526+	MSR	R2, ELR_EL1
   1.527+	MSR	R3, SPSR_EL1
   1.528+
   1.529+	MOV	224(RSP), R26	// special
   1.530+	MOV	232(RSP), R27	// special
   1.531+	MOV	240(RSP), R28	// sb
   1.532+	MOV	248(RSP), R29	// special
   1.533+
   1.534+_intrreturn:
   1.535+	MOV	16(RSP), R0
   1.536+	MOV	24(RSP), R1
   1.537+	MOV	32(RSP), R2
   1.538+	MOV	40(RSP), R3
   1.539+	MOV	48(RSP), R4
   1.540+	MOV	56(RSP), R5
   1.541+	MOV	64(RSP), R6
   1.542+	MOV	72(RSP), R7
   1.543+	MOV	80(RSP), R8
   1.544+	MOV	88(RSP), R9
   1.545+	MOV	96(RSP), R10
   1.546+	MOV	104(RSP), R11
   1.547+	MOV	112(RSP), R12
   1.548+	MOV	120(RSP), R13
   1.549+	MOV	128(RSP), R14
   1.550+	MOV	136(RSP), R15
   1.551+	MOV	144(RSP), R16
   1.552+	MOV	152(RSP), R17
   1.553+	MOV	160(RSP), R18
   1.554+	MOV	168(RSP), R19
   1.555+	MOV	176(RSP), R20
   1.556+	MOV	184(RSP), R21
   1.557+	MOV	192(RSP), R22
   1.558+	MOV	200(RSP), R23
   1.559+	MOV	208(RSP), R24
   1.560+	MOV	216(RSP), R25
   1.561+
   1.562+	MOV	256(RSP), R30	// link
   1.563+
   1.564+	ADD	$TRAPFRAMESIZE, RSP
   1.565+	ERET
   1.566+
   1.567+// irq/fiq/trap/serr from EL1
   1.568+TEXT vtrap1(SB), 1, $-4
   1.569+	MOV	R29, 248(RSP)	// special
   1.570+
   1.571+	ADD	$TRAPFRAMESIZE, RSP, R1
   1.572+	MRS	ELR_EL1, R2
   1.573+	MRS	SPSR_EL1, R3
   1.574+
   1.575+	MOV	R0, 288(RSP)	// type
   1.576+	MOV	R1, 264(RSP)	// sp
   1.577+	MOV	R2, 272(RSP)	// pc
   1.578+	MOV	R3, 280(RSP)	// psr
   1.579+
   1.580+	ADD	$16, RSP, R0	// ureg
   1.581+	BL	trap(SB)
   1.582+
   1.583+	MSR	$0x3, DAIFSet	// interrupts off
   1.584+
   1.585+	MOV	272(RSP), R2	// pc
   1.586+	MOV	280(RSP), R3	// psr
   1.587+
   1.588+	MSR	R2, ELR_EL1
   1.589+	MSR	R3, SPSR_EL1
   1.590+
   1.591+	MOV	248(RSP), R29	// special
   1.592+	B	_intrreturn	
   1.593+
   1.594+// vector tables
   1.595+TEXT vsys(SB), 1, $-4
   1.596+	SUB	$TRAPFRAMESIZE, RSP
   1.597+
   1.598+	MOV	R0, 16(RSP)
   1.599+	MOV	R30, 256(RSP)	// link
   1.600+
   1.601+	MOV	R17, 152(RSP)	// temp
   1.602+
   1.603+	MRS	ESR_EL1, R0	// type
   1.604+
   1.605+_vsyspatch:
   1.606+	B	_vsyspatch	// branch to vsys0() patched in
   1.607+
   1.608+TEXT vtrap(SB), 1, $-4
   1.609+	SUB	$TRAPFRAMESIZE, RSP
   1.610+
   1.611+	MOV	R0, 16(RSP)
   1.612+	MOV	R1, 24(RSP)
   1.613+	MOV	R2, 32(RSP)
   1.614+	MOV	R3, 40(RSP)
   1.615+	MOV	R4, 48(RSP)
   1.616+	MOV	R5, 56(RSP)
   1.617+	MOV	R6, 64(RSP)
   1.618+	MOV	R7, 72(RSP)
   1.619+	MOV	R8, 80(RSP)
   1.620+	MOV	R9, 88(RSP)
   1.621+	MOV	R10, 96(RSP)
   1.622+	MOV	R11, 104(RSP)
   1.623+	MOV	R12, 112(RSP)
   1.624+	MOV	R13, 120(RSP)
   1.625+	MOV	R14, 128(RSP)
   1.626+	MOV	R15, 136(RSP)
   1.627+	MOV	R16, 144(RSP)
   1.628+	MOV	R17, 152(RSP)
   1.629+	MOV	R18, 160(RSP)
   1.630+	MOV	R19, 168(RSP)
   1.631+	MOV	R20, 176(RSP)
   1.632+	MOV	R21, 184(RSP)
   1.633+	MOV	R22, 192(RSP)
   1.634+	MOV	R23, 200(RSP)
   1.635+	MOV	R24, 208(RSP)
   1.636+	MOV	R25, 216(RSP)
   1.637+
   1.638+	MOV	R30, 256(RSP)	// link
   1.639+
   1.640+	MRS	ESR_EL1, R0	// type
   1.641+
   1.642+_vtrappatch:
   1.643+	B	_vtrappatch	// branch to vtrapX() patched in
   1.644+
   1.645+TEXT virq(SB), 1, $-4
   1.646+	SUB	$TRAPFRAMESIZE, RSP
   1.647+
   1.648+	MOV	R0, 16(RSP)
   1.649+	MOV	R1, 24(RSP)
   1.650+	MOV	R2, 32(RSP)
   1.651+	MOV	R3, 40(RSP)
   1.652+	MOV	R4, 48(RSP)
   1.653+	MOV	R5, 56(RSP)
   1.654+	MOV	R6, 64(RSP)
   1.655+	MOV	R7, 72(RSP)
   1.656+	MOV	R8, 80(RSP)
   1.657+	MOV	R9, 88(RSP)
   1.658+	MOV	R10, 96(RSP)
   1.659+	MOV	R11, 104(RSP)
   1.660+	MOV	R12, 112(RSP)
   1.661+	MOV	R13, 120(RSP)
   1.662+	MOV	R14, 128(RSP)
   1.663+	MOV	R15, 136(RSP)
   1.664+	MOV	R16, 144(RSP)
   1.665+	MOV	R17, 152(RSP)
   1.666+	MOV	R18, 160(RSP)
   1.667+	MOV	R19, 168(RSP)
   1.668+	MOV	R20, 176(RSP)
   1.669+	MOV	R21, 184(RSP)
   1.670+	MOV	R22, 192(RSP)
   1.671+	MOV	R23, 200(RSP)
   1.672+	MOV	R24, 208(RSP)
   1.673+	MOV	R25, 216(RSP)
   1.674+
   1.675+	MOV	R30, 256(RSP)	// link
   1.676+
   1.677+	MOV	$(1<<32), R0	// type irq
   1.678+
   1.679+_virqpatch:
   1.680+	B	_virqpatch	// branch to vtrapX() patched in
   1.681+
   1.682+TEXT vfiq(SB), 1, $-4
   1.683+	SUB	$TRAPFRAMESIZE, RSP
   1.684+
   1.685+	MOV	R0, 16(RSP)
   1.686+	MOV	R1, 24(RSP)
   1.687+	MOV	R2, 32(RSP)
   1.688+	MOV	R3, 40(RSP)
   1.689+	MOV	R4, 48(RSP)
   1.690+	MOV	R5, 56(RSP)
   1.691+	MOV	R6, 64(RSP)
   1.692+	MOV	R7, 72(RSP)
   1.693+	MOV	R8, 80(RSP)
   1.694+	MOV	R9, 88(RSP)
   1.695+	MOV	R10, 96(RSP)
   1.696+	MOV	R11, 104(RSP)
   1.697+	MOV	R12, 112(RSP)
   1.698+	MOV	R13, 120(RSP)
   1.699+	MOV	R14, 128(RSP)
   1.700+	MOV	R15, 136(RSP)
   1.701+	MOV	R16, 144(RSP)
   1.702+	MOV	R17, 152(RSP)
   1.703+	MOV	R18, 160(RSP)
   1.704+	MOV	R19, 168(RSP)
   1.705+	MOV	R20, 176(RSP)
   1.706+	MOV	R21, 184(RSP)
   1.707+	MOV	R22, 192(RSP)
   1.708+	MOV	R23, 200(RSP)
   1.709+	MOV	R24, 208(RSP)
   1.710+	MOV	R25, 216(RSP)
   1.711+
   1.712+	MOV	R30, 256(RSP)	// link
   1.713+	MOV	$(2<<32), R0	// type fiq
   1.714+
   1.715+_vfiqpatch:
   1.716+	B	_vfiqpatch	// branch to vtrapX() patched in
   1.717+
   1.718+TEXT vserr(SB), 1, $-4
   1.719+	SUB	$TRAPFRAMESIZE, RSP
   1.720+
   1.721+	MOV	R0, 16(RSP)
   1.722+	MOV	R1, 24(RSP)
   1.723+	MOV	R2, 32(RSP)
   1.724+	MOV	R3, 40(RSP)
   1.725+	MOV	R4, 48(RSP)
   1.726+	MOV	R5, 56(RSP)
   1.727+	MOV	R6, 64(RSP)
   1.728+	MOV	R7, 72(RSP)
   1.729+	MOV	R8, 80(RSP)
   1.730+	MOV	R9, 88(RSP)
   1.731+	MOV	R10, 96(RSP)
   1.732+	MOV	R11, 104(RSP)
   1.733+	MOV	R12, 112(RSP)
   1.734+	MOV	R13, 120(RSP)
   1.735+	MOV	R14, 128(RSP)
   1.736+	MOV	R15, 136(RSP)
   1.737+	MOV	R16, 144(RSP)
   1.738+	MOV	R17, 152(RSP)
   1.739+	MOV	R18, 160(RSP)
   1.740+	MOV	R19, 168(RSP)
   1.741+	MOV	R20, 176(RSP)
   1.742+	MOV	R21, 184(RSP)
   1.743+	MOV	R22, 192(RSP)
   1.744+	MOV	R23, 200(RSP)
   1.745+	MOV	R24, 208(RSP)
   1.746+	MOV	R25, 216(RSP)
   1.747+
   1.748+	MOV	R30, 256(RSP)	// link
   1.749+
   1.750+	MRS	ESR_EL1, R0
   1.751+	ORR	$(3<<32), R0	// type
   1.752+_vserrpatch:
   1.753+	B	_vserrpatch	// branch to vtrapX() patched in