changelog shortlog tags branches files raw gz bz2 help

Mercurial > hg > plan9front / changeset: bcm64: add experimental work in progress arm64 kernel for raspberry pi 3

changeset 7199: ba62683c0e2d
parent 7198: bee572b18071
child 7200: 4b67ffcd2c61
author: cinap_lenrek@felloff.net
date: Fri, 03 May 2019 23:14:57 +0200
files: sys/src/9/bcm64/archbcm3.c sys/src/9/bcm64/cache.v8.s sys/src/9/bcm64/clock.c sys/src/9/bcm64/dat.h sys/src/9/bcm64/devgen.c sys/src/9/bcm64/fns.h sys/src/9/bcm64/fpu.c sys/src/9/bcm64/init9.s sys/src/9/bcm64/l.s sys/src/9/bcm64/main.c sys/src/9/bcm64/mem.h sys/src/9/bcm64/mkfile sys/src/9/bcm64/mmu.c sys/src/9/bcm64/pi3 sys/src/9/bcm64/sysreg.c sys/src/9/bcm64/sysreg.h sys/src/9/bcm64/trap.c
description: bcm64: add experimental work in progress arm64 kernel for raspberry pi 3
     1.1new file mode 100644
     1.2--- /dev/null
     1.3+++ b/sys/src/9/bcm64/archbcm3.c
     1.4@@ -0,0 +1,168 @@
     1.5+/*
     1.6+ * bcm2836 (e.g.raspberry pi 3) architecture-specific stuff
     1.7+ */
     1.8+
     1.9+#include "u.h"
    1.10+#include "../port/lib.h"
    1.11+#include "mem.h"
    1.12+#include "dat.h"
    1.13+#include "fns.h"
    1.14+#include "../port/error.h"
    1.15+#include "io.h"
    1.16+#include "sysreg.h"
    1.17+
    1.18+typedef struct Mbox Mbox;
    1.19+typedef struct Mboxes Mboxes;
    1.20+
    1.21+#define	POWERREGS	(VIRTIO+0x100000)
    1.22+
    1.23+Soc soc = {
    1.24+	.dramsize	= GiB,
    1.25+	.physio		= 0x3F000000,
    1.26+	.busdram	= 0xC0000000,
    1.27+	.busio		= 0x7E000000,
    1.28+	.armlocal	= 0x40000000,
    1.29+};
    1.30+
    1.31+enum {
    1.32+	Wdogfreq	= 65536,
    1.33+	Wdogtime	= 10,	/* seconds, ≤ 15 */
    1.34+};
    1.35+
    1.36+/*
    1.37+ * Power management / watchdog registers
    1.38+ */
    1.39+enum {
    1.40+	Rstc		= 0x1c>>2,
    1.41+		Password	= 0x5A<<24,
    1.42+		CfgMask		= 0x03<<4,
    1.43+		CfgReset	= 0x02<<4,
    1.44+	Rsts		= 0x20>>2,
    1.45+	Wdog		= 0x24>>2,
    1.46+};
    1.47+
    1.48+/*
    1.49+ * Arm local regs for smp
    1.50+ */
    1.51+struct Mbox {
    1.52+	u32int	doorbell;
    1.53+	u32int	mbox1;
    1.54+	u32int	mbox2;
    1.55+	u32int	startcpu;
    1.56+};
    1.57+struct Mboxes {
    1.58+	Mbox	set[4];
    1.59+	Mbox	clr[4];
    1.60+};
    1.61+
    1.62+enum {
    1.63+	Mboxregs	= 0x80,
    1.64+};
    1.65+
    1.66+void
    1.67+archreset(void)
    1.68+{
    1.69+}
    1.70+
    1.71+void
    1.72+archreboot(void)
    1.73+{
    1.74+	u32int *r;
    1.75+
    1.76+	r = (u32int*)POWERREGS;
    1.77+	r[Wdog] = Password | 1;
    1.78+	r[Rstc] = Password | (r[Rstc] & ~CfgMask) | CfgReset;
    1.79+	coherence();
    1.80+	for(;;)
    1.81+		;
    1.82+}
    1.83+
    1.84+void
    1.85+wdogfeed(void)
    1.86+{
    1.87+	u32int *r;
    1.88+
    1.89+	r = (u32int*)POWERREGS;
    1.90+	r[Wdog] = Password | (Wdogtime * Wdogfreq);
    1.91+	r[Rstc] = Password | (r[Rstc] & ~CfgMask) | CfgReset;
    1.92+}
    1.93+
    1.94+void
    1.95+wdogoff(void)
    1.96+{
    1.97+	u32int *r;
    1.98+
    1.99+	r = (u32int*)POWERREGS;
   1.100+	r[Rstc] = Password | (r[Rstc] & ~CfgMask);
   1.101+}
   1.102+
   1.103+
   1.104+char *
   1.105+cputype2name(char *buf, int size)
   1.106+{
   1.107+	u32int r, part;
   1.108+	char *p;
   1.109+
   1.110+	r = sysrd(MIDR_EL1);
   1.111+	part = (r >> 4) & 0xFFF;
   1.112+	switch(part){
   1.113+	case 0xc07:
   1.114+		p = seprint(buf, buf + size, "Cortex-A7");
   1.115+		break;
   1.116+	case 0xd03:
   1.117+		p = seprint(buf, buf + size, "Cortex-A53");
   1.118+		break;
   1.119+	default:
   1.120+		p = seprint(buf, buf + size, "Unknown-%#x", part);
   1.121+		break;
   1.122+	}
   1.123+	seprint(p, buf + size, " r%udp%ud", (r >> 20) & 0xF, r & 0xF);
   1.124+	return buf;
   1.125+}
   1.126+
   1.127+void
   1.128+cpuidprint(void)
   1.129+{
   1.130+	char name[64];
   1.131+
   1.132+	cputype2name(name, sizeof name);
   1.133+	iprint("cpu%d: %dMHz ARM %s\n", m->machno, m->cpumhz, name);
   1.134+}
   1.135+
   1.136+int
   1.137+getncpus(void)
   1.138+{
   1.139+	int n, max;
   1.140+	char *p;
   1.141+	n = 4;
   1.142+	if(n > MAXMACH)
   1.143+		n = MAXMACH;
   1.144+	p = getconf("*ncpu");
   1.145+	if(p && (max = atoi(p)) > 0 && n > max)
   1.146+		n = max;
   1.147+	return n;
   1.148+}
   1.149+
   1.150+void
   1.151+mboxclear(uint cpu)
   1.152+{
   1.153+	Mboxes *mb;
   1.154+
   1.155+	mb = (Mboxes*)(ARMLOCAL + Mboxregs);
   1.156+	mb->clr[cpu].mbox1 = 1;
   1.157+}
   1.158+
   1.159+void
   1.160+wakecpu(uint cpu)
   1.161+{
   1.162+	Mboxes *mb;
   1.163+
   1.164+	mb = (Mboxes*)(ARMLOCAL + Mboxregs);
   1.165+	mb->set[cpu].mbox1 = 1;
   1.166+}
   1.167+
   1.168+void
   1.169+archbcm3link(void)
   1.170+{
   1.171+//	addclock0link(wdogfeed, HZ);
   1.172+}
     2.1new file mode 100644
     2.2--- /dev/null
     2.3+++ b/sys/src/9/bcm64/cache.v8.s
     2.4@@ -0,0 +1,212 @@
     2.5+#include "sysreg.h"
     2.6+
     2.7+#undef	SYSREG
     2.8+#define	SYSREG(op0,op1,Cn,Cm,op2)	SPR(((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5))
     2.9+
    2.10+/*
    2.11+ * instruction cache operations
    2.12+ */
    2.13+TEXT cacheiinvse(SB), 1, $-4
    2.14+	MOVWU	len+8(FP), R2
    2.15+	ADD	R0, R2
    2.16+
    2.17+	MRS	DAIF, R11
    2.18+	MSR	$0x2, DAIFSet
    2.19+	MOVWU	$1, R10
    2.20+	MSR	R10, CSSELR_EL1
    2.21+	ISB	$SY
    2.22+	MRS	CCSIDR_EL1, R4
    2.23+
    2.24+	ANDW	$7, R4
    2.25+	ADDW	$4, R4		// log2(linelen)
    2.26+	LSL	R4, R10
    2.27+	LSR	R4, R0
    2.28+	LSL	R4, R0
    2.29+
    2.30+_iinvse:
    2.31+	IC	R0, 3,7,5,1	// IVAU
    2.32+	ADD	R10, R0
    2.33+	CMP	R0, R2
    2.34+	BGT	_iinvse
    2.35+	DSB	$NSH
    2.36+	ISB	$SY
    2.37+	MSR	R11, DAIF
    2.38+	RETURN
    2.39+
    2.40+TEXT cacheiinv(SB), 1, $-4
    2.41+	IC	R0, 0,7,5,0	// IALLU
    2.42+	DSB	$NSH
    2.43+	ISB	$SY
    2.44+	RETURN
    2.45+
    2.46+TEXT cacheuwbinv(SB), 1, $0
    2.47+	BL	cachedwbinv(SB)
    2.48+	BL	cacheiinv(SB)
    2.49+	RETURN
    2.50+
    2.51+/*
    2.52+ * data cache operations
    2.53+ */
    2.54+TEXT cachedwbse(SB), 1, $-4
    2.55+	MOV	LR, R29
    2.56+	BL	cachedva<>(SB)
    2.57+TEXT dccvac(SB), 1, $-4
    2.58+	DC	R0, 3,7,10,1	// CVAC
    2.59+	RETURN
    2.60+
    2.61+TEXT cacheduwbse(SB), 1, $-4
    2.62+	MOV	LR, R29
    2.63+	BL	cachedva<>(SB)
    2.64+TEXT dccvau(SB), 1, $-4
    2.65+	DC	R0, 3,7,11,1	// CVAU
    2.66+	RETURN
    2.67+
    2.68+TEXT cachedinvse(SB), 1, $-4
    2.69+	MOV	LR, R29
    2.70+	BL	cachedva<>(SB)
    2.71+TEXT dcivac(SB), 1, $-4
    2.72+	DC	R0, 0,7,6,1	// IVAC
    2.73+	RETURN
    2.74+
    2.75+TEXT cachedwbinvse(SB), 1, $-4
    2.76+	MOV	LR, R29
    2.77+	BL	cachedva<>(SB)
    2.78+TEXT dccivac(SB), 1, $-4
    2.79+	DC	R0, 3,7,14,1	// CIVAC
    2.80+	RETURN
    2.81+
    2.82+TEXT cachedva<>(SB), 1, $-4
    2.83+	MOV	LR, R1
    2.84+	MOVWU	len+8(FP), R2
    2.85+	ADD	R0, R2
    2.86+
    2.87+	MRS	DAIF, R11
    2.88+	MSR	$0x2, DAIFSet
    2.89+	MOVWU	$0, R10
    2.90+	MSR	R10, CSSELR_EL1
    2.91+	ISB	$SY
    2.92+	MRS	CCSIDR_EL1, R4
    2.93+
    2.94+	ANDW	$7, R4
    2.95+	ADDW	$4, R4		// log2(linelen)
    2.96+	MOVWU	$1, R10
    2.97+	LSL	R4, R10
    2.98+	LSR	R4, R0
    2.99+	LSL	R4, R0
   2.100+
   2.101+	DSB	$SY
   2.102+	ISB	$SY
   2.103+_cachedva:
   2.104+	BL	(R1)
   2.105+	ADD	R10, R0
   2.106+	CMP	R0, R2
   2.107+	BGT	_cachedva
   2.108+	DSB	$SY
   2.109+	ISB	$SY
   2.110+	MSR	R11, DAIF
   2.111+	RET	R29
   2.112+
   2.113+/*
   2.114+ * l1 cache operations
   2.115+ */
   2.116+TEXT cachedwb(SB), 1, $-4
   2.117+	MOVWU	$0, R0
   2.118+_cachedwb:
   2.119+	MOV	LR, R29
   2.120+	BL	cachedsw<>(SB)
   2.121+TEXT dccsw(SB), 1, $-4
   2.122+	DC	R0, 0,7,10,2	// CSW
   2.123+	RETURN
   2.124+
   2.125+TEXT cachedinv(SB), 1, $-4
   2.126+	MOVWU	$0, R0
   2.127+_cachedinv:
   2.128+	MOV	LR, R29
   2.129+	BL	cachedsw<>(SB)
   2.130+TEXT dcisw(SB), 1, $-4
   2.131+	DC	R0, 0,7,6,2	// ISW
   2.132+	RETURN
   2.133+
   2.134+TEXT cachedwbinv(SB), 1, $-4
   2.135+	MOVWU	$0, R0
   2.136+_cachedwbinv:
   2.137+	MOV	LR, R29
   2.138+	BL	cachedsw<>(SB)
   2.139+TEXT dccisw(SB), 1, $-4
   2.140+	DC	R0, 0,7,14,2	// CISW
   2.141+	RETURN
   2.142+
   2.143+/*
   2.144+ * l2 cache operations
   2.145+ */
   2.146+TEXT l2cacheuwb(SB), 1, $-4
   2.147+	MOVWU	$1, R0
   2.148+	B	_cachedwb
   2.149+TEXT l2cacheuinv(SB), 1, $-4
   2.150+	MOVWU	$1, R0
   2.151+	B	_cachedinv
   2.152+TEXT l2cacheuwbinv(SB), 1, $-4
   2.153+	MOVWU	$1, R0
   2.154+	B	_cachedwbinv
   2.155+
   2.156+TEXT cachesize(SB), 1, $-4
   2.157+	MRS	DAIF, R11
   2.158+	MSR	$0x2, DAIFSet
   2.159+	MSR	R0, CSSELR_EL1
   2.160+	ISB	$SY
   2.161+	MRS	CCSIDR_EL1, R0
   2.162+	MSR	R11, DAIF
   2.163+	RETURN
   2.164+
   2.165+TEXT cachedsw<>(SB), 1, $-4
   2.166+	MOV	LR, R1
   2.167+
   2.168+	MRS	DAIF, R11
   2.169+	MSR	$0x2, DAIFSet
   2.170+	ADDW	R0, R0, R8
   2.171+	MSR	R8, CSSELR_EL1
   2.172+	ISB	$SY
   2.173+	MRS	CCSIDR_EL1, R4
   2.174+
   2.175+	LSR	$3, R4, R7
   2.176+	ANDW	$1023, R7	// lastway
   2.177+	ADDW	$1, R7, R5	// #ways
   2.178+
   2.179+	LSR	$13, R4, R2
   2.180+	ANDW	$32767, R2	// lastset
   2.181+	ADDW	$1, R2		// #sets
   2.182+
   2.183+	ANDW	$7, R4
   2.184+	ADDW	$4, R4		// log2(linelen)
   2.185+
   2.186+	MOVWU	$32, R3		// wayshift = 32 - log2(#ways)
   2.187+_countlog2ways:
   2.188+	CBZ	R7, _loop	// lastway == 0?
   2.189+	LSR	$1, R7		// lastway >>= 1
   2.190+	SUB	$1, R3		// wayshift--
   2.191+	B _countlog2ways
   2.192+_loop:
   2.193+	DSB	$SY
   2.194+	ISB	$SY
   2.195+_nextway:
   2.196+	MOVWU	$0, R6		// set
   2.197+_nextset:
   2.198+	LSL	R3, R7, R0	// way<<wayshift
   2.199+	LSL	R4, R6, R9	// set<<log2(linelen)
   2.200+	ORRW	R8, R0		// level
   2.201+	ORRW	R9, R0		// setway
   2.202+
   2.203+	BL	(R1)		// op(setway)
   2.204+
   2.205+	ADDW	$1, R6		// set++
   2.206+	CMPW	R2, R6
   2.207+	BLT	_nextset
   2.208+
   2.209+	ADDW	$1, R7		// way++
   2.210+	CMPW	R5, R7
   2.211+	BLT	_nextway
   2.212+
   2.213+	DSB	$SY
   2.214+	ISB	$SY
   2.215+	MSR	R11, DAIF
   2.216+	RET	R29
     3.1new file mode 100644
     3.2--- /dev/null
     3.3+++ b/sys/src/9/bcm64/clock.c
     3.4@@ -0,0 +1,267 @@
     3.5+/*
     3.6+ * bcm283[56] timers
     3.7+ *	System timers run at 1MHz (timers 1 and 2 are used by GPU)
     3.8+ *	ARM timer usually runs at 250MHz (may be slower in low power modes)
     3.9+ *	Cycle counter runs at 700MHz (unless overclocked)
    3.10+ *    All are free-running up-counters
    3.11+ *
    3.12+ * Use system timer 3 (64 bits) for hzclock interrupts and fastticks
    3.13+ *   For smp on bcm2836, use local generic timer for interrupts on cpu1-3
    3.14+ * Use ARM timer (32 bits) for perfticks
    3.15+ * Use ARM timer to force immediate interrupt
    3.16+ * Use cycle counter for cycles()
    3.17+ */
    3.18+
    3.19+#include "u.h"
    3.20+#include "../port/lib.h"
    3.21+#include "mem.h"
    3.22+#include "dat.h"
    3.23+#include "fns.h"
    3.24+#include "io.h"
    3.25+#include "ureg.h"
    3.26+#include "sysreg.h"
    3.27+
    3.28+enum {
    3.29+	SYSTIMERS	= VIRTIO+0x3000,
    3.30+	ARMTIMER	= VIRTIO+0xB400,
    3.31+
    3.32+	Localctl	= 0x00,
    3.33+	Prescaler	= 0x08,
    3.34+	GPUirqroute	= 0x0C,
    3.35+
    3.36+	SystimerFreq	= 1*Mhz,
    3.37+	MaxPeriod	= SystimerFreq / HZ,
    3.38+	MinPeriod	= 10,
    3.39+};
    3.40+
    3.41+typedef struct Systimers Systimers;
    3.42+typedef struct Armtimer Armtimer;
    3.43+
    3.44+struct Systimers {
    3.45+	u32int	cs;
    3.46+	u32int	clo;
    3.47+	u32int	chi;
    3.48+	u32int	c0;
    3.49+	u32int	c1;
    3.50+	u32int	c2;
    3.51+	u32int	c3;
    3.52+};
    3.53+
    3.54+struct Armtimer {
    3.55+	u32int	load;
    3.56+	u32int	val;
    3.57+	u32int	ctl;
    3.58+	u32int	irqack;
    3.59+	u32int	irq;
    3.60+	u32int	maskedirq;
    3.61+	u32int	reload;
    3.62+	u32int	predivider;
    3.63+	u32int	count;
    3.64+};
    3.65+
    3.66+enum {
    3.67+	CntPrescaleShift= 16,	/* freq is sys_clk/(prescale+1) */
    3.68+	CntPrescaleMask	= 0xFF,
    3.69+	CntEnable	= 1<<9,
    3.70+	TmrDbgHalt	= 1<<8,
    3.71+	TmrEnable	= 1<<7,
    3.72+	TmrIntEnable	= 1<<5,
    3.73+	TmrPrescale1	= 0x00<<2,
    3.74+	TmrPrescale16	= 0x01<<2,
    3.75+	TmrPrescale256	= 0x02<<2,
    3.76+	CntWidth16	= 0<<1,
    3.77+	CntWidth32	= 1<<1,
    3.78+
    3.79+	/* generic timer (cortex-a7) */
    3.80+	Enable	= 1<<0,
    3.81+	Imask	= 1<<1,
    3.82+	Istatus = 1<<2,
    3.83+};
    3.84+
    3.85+static void
    3.86+clockintr(Ureg *ureg, void *)
    3.87+{
    3.88+	Systimers *tn;
    3.89+
    3.90+	if(m->machno != 0)
    3.91+		panic("cpu%d: unexpected system timer interrupt", m->machno);
    3.92+	tn = (Systimers*)SYSTIMERS;
    3.93+	/* dismiss interrupt */
    3.94+	tn->cs = 1<<3;
    3.95+	timerintr(ureg, 0);
    3.96+}
    3.97+
    3.98+static void
    3.99+localclockintr(Ureg *ureg, void *)
   3.100+{
   3.101+	if(m->machno == 0)
   3.102+		panic("cpu0: Unexpected local generic timer interrupt");
   3.103+	timerintr(ureg, 0);
   3.104+}
   3.105+
   3.106+void
   3.107+clockshutdown(void)
   3.108+{
   3.109+	Armtimer *tm;
   3.110+
   3.111+	tm = (Armtimer*)ARMTIMER;
   3.112+	tm->ctl = 0;
   3.113+}
   3.114+
   3.115+void
   3.116+clockinit(void)
   3.117+{
   3.118+	Systimers *tn;
   3.119+	Armtimer *tm;
   3.120+	ulong t0, t1, tstart, tend;
   3.121+
   3.122+	syswr(PMCR_EL0, 1<<6 | 7);
   3.123+	syswr(PMCNTENSET, 1<<31);
   3.124+	syswr(PMUSERENR_EL0, 1<<2);
   3.125+
   3.126+	syswr(CNTP_TVAL_EL0, ~0UL);
   3.127+	if(m->machno == 0){
   3.128+		syswr(CNTP_CTL_EL0, Imask);
   3.129+
   3.130+		*(u32int*)(ARMLOCAL + GPUirqroute) = 0;
   3.131+
   3.132+		/* input clock is 19.2Mhz crystal */
   3.133+		*(u32int*)(ARMLOCAL + Localctl) = 0;
   3.134+		/* divide by (2^31/Prescaler) */
   3.135+		*(u32int*)(ARMLOCAL + Prescaler) = (((uvlong)SystimerFreq<<31)/19200000)&~1UL;
   3.136+	} else {
   3.137+		syswr(CNTP_CTL_EL0, Enable);
   3.138+		intrenable(IRQcntpns, localclockintr, nil, 0, "clock");
   3.139+	}
   3.140+
   3.141+	tn = (Systimers*)SYSTIMERS;
   3.142+	tstart = tn->clo;
   3.143+	do{
   3.144+		t0 = lcycles();
   3.145+	}while(tn->clo == tstart);
   3.146+	tend = tstart + (SystimerFreq/100);
   3.147+	do{
   3.148+		t1 = lcycles();
   3.149+	}while(tn->clo < tend);
   3.150+	t1 -= t0;
   3.151+	m->cpuhz = 100 * t1;
   3.152+	m->cpumhz = (m->cpuhz + Mhz/2 - 1) / Mhz;
   3.153+	m->cyclefreq = m->cpuhz;
   3.154+
   3.155+	if(m->machno == 0){
   3.156+		tn->cs = 1<<3;
   3.157+		tn->c3 = tn->clo - 1;
   3.158+		intrenable(IRQtimer3, clockintr, nil, 0, "clock");
   3.159+
   3.160+		tm = (Armtimer*)ARMTIMER;
   3.161+		tm->load = 0;
   3.162+		tm->ctl = TmrPrescale1|CntEnable|CntWidth32;
   3.163+	}
   3.164+}
   3.165+
   3.166+void
   3.167+timerset(uvlong next)
   3.168+{
   3.169+	Systimers *tn;
   3.170+	uvlong now;
   3.171+	long period;
   3.172+
   3.173+	now = fastticks(nil);
   3.174+	period = next - now;
   3.175+	if(period < MinPeriod)
   3.176+		period = MinPeriod;
   3.177+	else if(period > MaxPeriod)
   3.178+		period = MaxPeriod;
   3.179+	if(m->machno)
   3.180+		syswr(CNTP_TVAL_EL0, period);
   3.181+	else{
   3.182+		tn = (Systimers*)SYSTIMERS;
   3.183+		tn->c3 = tn->clo + period;
   3.184+	}
   3.185+}
   3.186+
   3.187+uvlong
   3.188+fastticks(uvlong *hz)
   3.189+{
   3.190+	Systimers *tn;
   3.191+	ulong lo, hi;
   3.192+	uvlong now;
   3.193+
   3.194+	if(hz)
   3.195+		*hz = SystimerFreq;
   3.196+	tn = (Systimers*)SYSTIMERS;
   3.197+	do{
   3.198+		hi = tn->chi;
   3.199+		lo = tn->clo;
   3.200+	}while(tn->chi != hi);
   3.201+	now = (uvlong)hi<<32 | lo;
   3.202+	return now;
   3.203+}
   3.204+
   3.205+ulong
   3.206+perfticks(void)
   3.207+{
   3.208+	Armtimer *tm;
   3.209+
   3.210+	tm = (Armtimer*)ARMTIMER;
   3.211+	return tm->count;
   3.212+}
   3.213+
   3.214+void
   3.215+armtimerset(int n)
   3.216+{
   3.217+	Armtimer *tm;
   3.218+
   3.219+	tm = (Armtimer*)ARMTIMER;
   3.220+	if(n > 0){
   3.221+		tm->ctl |= TmrEnable|TmrIntEnable;
   3.222+		tm->load = n;
   3.223+	}else{
   3.224+		tm->load = 0;
   3.225+		tm->ctl &= ~(TmrEnable|TmrIntEnable);
   3.226+		tm->irq = 1;
   3.227+	}
   3.228+}
   3.229+
   3.230+ulong
   3.231+µs(void)
   3.232+{
   3.233+	if(SystimerFreq != 1*Mhz)
   3.234+		return fastticks2us(fastticks(nil));
   3.235+	return ((Systimers*)SYSTIMERS)->clo;
   3.236+}
   3.237+
   3.238+void
   3.239+microdelay(int n)
   3.240+{
   3.241+	ulong now;
   3.242+
   3.243+	now = µs();
   3.244+	while(µs() - now < n);
   3.245+}
   3.246+
   3.247+void
   3.248+delay(int n)
   3.249+{
   3.250+	while(--n >= 0)
   3.251+		microdelay(1000);
   3.252+}
   3.253+
   3.254+void
   3.255+synccycles(void)
   3.256+{
   3.257+	static Ref r1, r2;
   3.258+	int s;
   3.259+
   3.260+	s = splhi();
   3.261+	r2.ref = 0;
   3.262+	incref(&r1);
   3.263+	while(r1.ref != conf.nmach)
   3.264+		;
   3.265+//	syswr(PMCR_EL0, 1<<6 | 7);
   3.266+	incref(&r2);
   3.267+	while(r2.ref != conf.nmach)
   3.268+		;
   3.269+	r1.ref = 0;
   3.270+	splx(s);
   3.271+}
     4.1new file mode 100644
     4.2--- /dev/null
     4.3+++ b/sys/src/9/bcm64/dat.h
     4.4@@ -0,0 +1,272 @@
     4.5+/*
     4.6+ * Time.
     4.7+ *
     4.8+ * HZ should divide 1000 evenly, ideally.
     4.9+ * 100, 125, 200, 250 and 333 are okay.
    4.10+ */
    4.11+#define	HZ		100			/* clock frequency */
    4.12+#define	MS2HZ		(1000/HZ)		/* millisec per clock tick */
    4.13+#define	TK2SEC(t)	((t)/HZ)		/* ticks to seconds */
    4.14+
    4.15+enum {
    4.16+	Mhz	= 1000 * 1000,
    4.17+};
    4.18+
    4.19+typedef struct Conf	Conf;
    4.20+typedef struct Confmem	Confmem;
    4.21+typedef struct FPsave	FPsave;
    4.22+typedef struct PFPU	PFPU;
    4.23+typedef struct ISAConf	ISAConf;
    4.24+typedef struct Label	Label;
    4.25+typedef struct Lock	Lock;
    4.26+typedef struct Memcache	Memcache;
    4.27+typedef struct MMMU	MMMU;
    4.28+typedef struct Mach	Mach;
    4.29+typedef struct Page	Page;
    4.30+typedef struct PhysUart	PhysUart;
    4.31+typedef struct PMMU	PMMU;
    4.32+typedef struct Proc	Proc;
    4.33+typedef u64int		PTE;
    4.34+typedef struct Soc	Soc;
    4.35+typedef struct Uart	Uart;
    4.36+typedef struct Ureg	Ureg;
    4.37+typedef uvlong		Tval;
    4.38+typedef void		KMap;
    4.39+
    4.40+#pragma incomplete Ureg
    4.41+
    4.42+#define MAXSYSARG	5	/* for mount(fd, mpt, flag, arg, srv) */
    4.43+
    4.44+/*
    4.45+ *  parameters for sysproc.c
    4.46+ */
    4.47+#define AOUT_MAGIC	(R_MAGIC)
    4.48+
    4.49+struct Lock
    4.50+{
    4.51+	ulong	key;
    4.52+	u32int	sr;
    4.53+	uintptr	pc;
    4.54+	Proc*	p;
    4.55+	Mach*	m;
    4.56+	int	isilock;
    4.57+};
    4.58+
    4.59+struct Label
    4.60+{
    4.61+	uintptr	sp;
    4.62+	uintptr	pc;
    4.63+};
    4.64+
    4.65+struct FPsave
    4.66+{
    4.67+	uvlong	regs[32][2];
    4.68+
    4.69+	ulong	control;
    4.70+	ulong	status;
    4.71+};
    4.72+
    4.73+struct PFPU
    4.74+{
    4.75+	FPsave	fpsave[1];
    4.76+
    4.77+	int	fpstate;
    4.78+};
    4.79+
    4.80+enum
    4.81+{
    4.82+	FPinit,
    4.83+	FPactive,
    4.84+	FPinactive,
    4.85+
    4.86+	/* bits or'd with the state */
    4.87+	FPillegal= 0x100,
    4.88+};
    4.89+
    4.90+struct Confmem
    4.91+{
    4.92+	uintptr	base;
    4.93+	usize	npage;
    4.94+	uintptr	limit;
    4.95+	uintptr	kbase;
    4.96+	uintptr	klimit;
    4.97+};
    4.98+
    4.99+struct Conf
   4.100+{
   4.101+	ulong	nmach;		/* processors */
   4.102+	ulong	nproc;		/* processes */
   4.103+	Confmem	mem[1];		/* physical memory */
   4.104+	ulong	npage;		/* total physical pages of memory */
   4.105+	usize	upages;		/* user page pool */
   4.106+	ulong	copymode;	/* 0 is copy on write, 1 is copy on reference */
   4.107+	ulong	ialloc;		/* max interrupt time allocation in bytes */
   4.108+	ulong	pipeqsize;	/* size in bytes of pipe queues */
   4.109+	ulong	nimage;		/* number of page cache image headers */
   4.110+	ulong	nswap;		/* number of swap pages */
   4.111+	int	nswppo;		/* max # of pageouts per segment pass */
   4.112+	ulong	hz;		/* processor cycle freq */
   4.113+	ulong	mhz;
   4.114+	int	monitor;	/* flag */
   4.115+};
   4.116+
   4.117+/*
   4.118+ *  MMU stuff in Mach.
   4.119+ */
   4.120+struct MMMU
   4.121+{
   4.122+	PTE*	mmul1;		/* l1 for this processor */
   4.123+};
   4.124+
   4.125+/*
   4.126+ *  MMU stuff in proc
   4.127+ */
   4.128+#define NCOLOR	1		/* 1 level cache, don't worry about VCE's */
   4.129+
   4.130+struct PMMU
   4.131+{
   4.132+	Page*	mmul1;
   4.133+	Page*	mmul1tail;
   4.134+
   4.135+	Page*	mmul2;
   4.136+	Page*	mmul2tail;
   4.137+
   4.138+	Page*	mmufree;
   4.139+
   4.140+	int	asid;
   4.141+
   4.142+	uintptr	tpidr;
   4.143+};
   4.144+
   4.145+#include "../port/portdat.h"
   4.146+
   4.147+struct Mach
   4.148+{
   4.149+	int	machno;			/* physical id of processor */
   4.150+	uintptr	splpc;			/* pc of last caller to splhi */
   4.151+
   4.152+	Proc*	proc;			/* current process */
   4.153+
   4.154+	MMMU;
   4.155+	int	flushmmu;		/* flush current proc mmu state */
   4.156+
   4.157+	ulong	ticks;			/* of the clock since boot time */
   4.158+	Label	sched;			/* scheduler wakeup */
   4.159+	Lock	alarmlock;		/* access to alarm list */
   4.160+	void*	alarm;			/* alarms bound to this clock */
   4.161+
   4.162+	Proc*	readied;		/* for runproc */
   4.163+	ulong	schedticks;		/* next forced context switch */
   4.164+
   4.165+	int	cputype;
   4.166+	ulong	delayloop;
   4.167+
   4.168+	/* stats */
   4.169+	int	tlbfault;
   4.170+	int	tlbpurge;
   4.171+	int	pfault;
   4.172+	int	cs;
   4.173+	int	syscall;
   4.174+	int	load;
   4.175+	int	intr;
   4.176+	uvlong	fastclock;		/* last sampled value */
   4.177+	uvlong	inidle;			/* time spent in idlehands() */
   4.178+	ulong	spuriousintr;
   4.179+	int	lastintr;
   4.180+	int	ilockdepth;
   4.181+	Perf	perf;			/* performance counters */
   4.182+
   4.183+	int	cpumhz;
   4.184+	uvlong	cpuhz;			/* speed of cpu */
   4.185+	uvlong	cyclefreq;		/* Frequency of user readable cycle counter */
   4.186+
   4.187+	int	stack[1];
   4.188+};
   4.189+
   4.190+struct
   4.191+{
   4.192+	char	machs[MAXMACH];		/* active CPUs */
   4.193+	int	exiting;		/* shutdown */
   4.194+}active;
   4.195+
   4.196+#define MACHP(n)	((Mach*)MACHADDR(n))
   4.197+
   4.198+extern register Mach* m;			/* R27 */
   4.199+extern register Proc* up;			/* R26 */
   4.200+extern int normalprint;
   4.201+extern ulong memsize;
   4.202+
   4.203+/*
   4.204+ *  a parsed plan9.ini line
   4.205+ */
   4.206+#define NISAOPT		8
   4.207+
   4.208+struct ISAConf {
   4.209+	char	*type;
   4.210+	ulong	port;
   4.211+	int	irq;
   4.212+	ulong	dma;
   4.213+	ulong	mem;
   4.214+	ulong	size;
   4.215+	ulong	freq;
   4.216+
   4.217+	int	nopt;
   4.218+	char	*opt[NISAOPT];
   4.219+};
   4.220+
   4.221+/*
   4.222+ * Horrid. But the alternative is 'defined'.
   4.223+ */
   4.224+#ifdef _DBGC_
   4.225+#define DBGFLG		(dbgflg[_DBGC_])
   4.226+#else
   4.227+#define DBGFLG		(0)
   4.228+#endif /* _DBGC_ */
   4.229+
   4.230+int vflag;
   4.231+extern char dbgflg[256];
   4.232+
   4.233+#define dbgprint	print		/* for now */
   4.234+
   4.235+/*
   4.236+ *  hardware info about a device
   4.237+ */
   4.238+typedef struct {
   4.239+	ulong	port;
   4.240+	int	size;
   4.241+} Devport;
   4.242+
   4.243+struct DevConf
   4.244+{
   4.245+	ulong	intnum;			/* interrupt number */
   4.246+	char	*type;			/* card type, malloced */
   4.247+	int	nports;			/* Number of ports */
   4.248+	Devport	*ports;			/* The ports themselves */
   4.249+};
   4.250+
   4.251+struct Soc {			/* SoC dependent configuration */
   4.252+	ulong	dramsize;
   4.253+	uintptr	physio;
   4.254+	uintptr	busdram;
   4.255+	uintptr	busio;
   4.256+	uintptr	armlocal;
   4.257+	u32int	l1ptedramattrs;
   4.258+	u32int	l2ptedramattrs;
   4.259+};
   4.260+extern Soc soc;
   4.261+
   4.262+#define BUSUNKNOWN -1
   4.263+
   4.264+/*
   4.265+ * GPIO
   4.266+ */
   4.267+enum {
   4.268+	Input	= 0x0,
   4.269+	Output	= 0x1,
   4.270+	Alt0	= 0x4,
   4.271+	Alt1	= 0x5,
   4.272+	Alt2	= 0x6,
   4.273+	Alt3	= 0x7,
   4.274+	Alt4	= 0x3,
   4.275+	Alt5	= 0x2,
   4.276+};
     5.1new file mode 100644
     5.2--- /dev/null
     5.3+++ b/sys/src/9/bcm64/devgen.c
     5.4@@ -0,0 +1,34 @@
     5.5+#include	"u.h"
     5.6+#include	"../port/lib.h"
     5.7+#include	"mem.h"
     5.8+#include	"dat.h"
     5.9+#include	"fns.h"
    5.10+#include	"../port/error.h"
    5.11+
    5.12+/*
    5.13+ * the zeroth element of the table MUST be the directory itself for ..
    5.14+*/
    5.15+int
    5.16+devgen(Chan *c, char *name, Dirtab *tab, int ntab, int i, Dir *dp)
    5.17+{
    5.18+	if(tab == 0)
    5.19+		return -1;
    5.20+	if(i == DEVDOTDOT){
    5.21+		/* nothing */
    5.22+	}else if(name){
    5.23+		for(i=1; i<ntab; i++)
    5.24+			if(strcmp(tab[i].name, name) == 0)
    5.25+				break;
    5.26+		if(i==ntab)
    5.27+			return -1;
    5.28+		tab += i;
    5.29+	}else{
    5.30+		/* skip over the first element, that for . itself */
    5.31+		i++;
    5.32+		if(i >= ntab)
    5.33+			return -1;
    5.34+		tab += i;
    5.35+	}
    5.36+	devdir(c, tab->qid, tab->name, tab->length, eve, tab->perm, dp);
    5.37+	return 1;
    5.38+}
     6.1new file mode 100644
     6.2--- /dev/null
     6.3+++ b/sys/src/9/bcm64/fns.h
     6.4@@ -0,0 +1,168 @@
     6.5+#include "../port/portfns.h"
     6.6+
     6.7+#define	waserror()	(up->nerrlab++, setlabel(&up->errlab[up->nerrlab-1]))
     6.8+
     6.9+/* l.s */
    6.10+extern void sev(void);
    6.11+extern int tas(void *);
    6.12+extern int cmpswap(long*, long, long);
    6.13+extern void coherence(void);
    6.14+extern void idlehands(void);
    6.15+extern uvlong cycles(void);
    6.16+extern int splfhi(void);
    6.17+extern void splflo(void);
    6.18+extern void touser(uintptr sp);
    6.19+extern void forkret(void);
    6.20+extern void noteret(void);
    6.21+extern void returnto(void*);
    6.22+extern void fpsaveregs(void*);
    6.23+extern void fploadregs(void*);
    6.24+extern void magic(void);
    6.25+
    6.26+extern void setttbr(uintptr pa);
    6.27+extern uintptr getfar(void);
    6.28+
    6.29+extern void flushasidva(uintptr asidva);
    6.30+extern void tlbivae1is(uintptr asidva);
    6.31+
    6.32+extern void flushasidvall(uintptr asidva);
    6.33+extern void tlbivale1is(uintptr asidva);
    6.34+
    6.35+extern void flushasid(uintptr asid);
    6.36+extern void tlbiaside1is(uintptr asid);
    6.37+
    6.38+extern void flushtlb(void);
    6.39+extern void tlbivmalle1(void);
    6.40+
    6.41+/* cache */
    6.42+extern ulong cachesize(int level);
    6.43+
    6.44+extern void cacheiinvse(void*, int);
    6.45+extern void cacheuwbinv(void);
    6.46+extern void cacheiinv(void);
    6.47+
    6.48+extern void cachedwbse(void*, int);
    6.49+extern void cacheduwbse(void*, int);
    6.50+extern void cachedinvse(void*, int);
    6.51+extern void cachedwbinvse(void*, int);
    6.52+
    6.53+extern void cachedwb(void);
    6.54+extern void cachedinv(void);
    6.55+extern void cachedwbinv(void);
    6.56+
    6.57+extern void l2cacheuwb(void);
    6.58+extern void l2cacheuinv(void);
    6.59+extern void l2cacheuwbinv(void);
    6.60+
    6.61+/* mmu */
    6.62+#define	getpgcolor(a)	0
    6.63+extern uintptr paddr(void*);
    6.64+#define PADDR(a) paddr((void*)(a))
    6.65+extern uintptr cankaddr(uintptr);
    6.66+extern void* kaddr(uintptr);
    6.67+#define KADDR(a) kaddr(a)
    6.68+extern void kmapinval(void);
    6.69+#define	VA(k)	((uintptr)(k))
    6.70+extern KMap *kmap(Page*);
    6.71+extern void kunmap(KMap*);
    6.72+extern uintptr mmukmap(uintptr, uintptr, usize);
    6.73+
    6.74+extern void mmu0init(uintptr*);
    6.75+extern void mmu0clear(uintptr*);
    6.76+extern void mmu1init(void);
    6.77+
    6.78+extern void putasid(Proc*);
    6.79+
    6.80+/* clock */
    6.81+extern void clockinit(void);
    6.82+extern void synccycles(void);
    6.83+extern void armtimerset(int);
    6.84+
    6.85+/* fpu */
    6.86+extern void fpuinit(void);
    6.87+extern void fpoff(void);
    6.88+extern void fpinit(void);
    6.89+extern void fpclear(void);
    6.90+extern void fpsave(FPsave*);
    6.91+extern void fprestore(FPsave*);
    6.92+extern void mathtrap(Ureg*);
    6.93+
    6.94+/* trap */
    6.95+extern void trapinit(void);
    6.96+extern int userureg(Ureg*);
    6.97+extern void evenaddr(uintptr);
    6.98+extern void setkernur(Ureg*, Proc*);
    6.99+extern void procfork(Proc*);
   6.100+extern void procsetup(Proc*);
   6.101+extern void procsave(Proc*);
   6.102+extern void procrestore(Proc *);
   6.103+extern void trap(Ureg*);
   6.104+extern void syscall(Ureg*);
   6.105+extern void noted(Ureg*, ulong);
   6.106+extern void faultarm64(Ureg*);
   6.107+extern void dumpstack(void);
   6.108+extern void dumpregs(Ureg*);
   6.109+
   6.110+/* irq */
   6.111+extern void intrcpushutdown(void);
   6.112+extern void intrsoff(void);
   6.113+#define intrenable(i, f, a, b, n)	irqenable((i), (f), (a))
   6.114+extern void irqenable(int, void (*)(Ureg*, void*), void*);
   6.115+extern int irq(Ureg*);
   6.116+extern void fiq(Ureg*);
   6.117+
   6.118+/* sysreg */
   6.119+extern uvlong	sysrd(ulong);
   6.120+extern void	syswr(ulong, uvlong);
   6.121+
   6.122+/* gpio */
   6.123+extern void gpiosel(uint, int);
   6.124+extern void gpiopull(uint, int);
   6.125+extern void gpiopullup(uint);
   6.126+extern void gpiopulloff(uint);
   6.127+extern void gpiopulldown(uint);
   6.128+extern void gpioout(uint, int);
   6.129+extern int gpioin(uint);
   6.130+extern void gpioselevent(uint, int, int);
   6.131+extern int gpiogetevent(uint);
   6.132+extern void gpiomeminit(void);
   6.133+
   6.134+/* arch */
   6.135+extern char *cputype2name(char*, int);
   6.136+extern void cpuidprint(void);
   6.137+extern void uartconsinit(void);
   6.138+extern void links(void);
   6.139+extern int getncpus(void);
   6.140+extern int startcpu(uint);
   6.141+extern void okay(int);
   6.142+
   6.143+/* dma */
   6.144+extern uintptr dmaaddr(void*);
   6.145+extern void dmastart(int, int, int, void*, void*, int);
   6.146+extern int dmawait(int);
   6.147+
   6.148+/* vcore */
   6.149+extern void* fbinit(int set, int *width, int *height, int *depth);
   6.150+extern int fbblank(int blank);
   6.151+extern void setpower(int dev, int on);
   6.152+extern int getpower(int dev);
   6.153+extern char* getethermac(void);
   6.154+extern uint getboardrev(void);
   6.155+extern uint getfirmware(void);
   6.156+extern void getramsize(Confmem *mem);
   6.157+extern ulong getclkrate(int clkid);
   6.158+extern void setclkrate(int clkid, ulong hz);
   6.159+extern uint getcputemp(void);
   6.160+extern void vgpinit(void);
   6.161+extern void vgpset(uint port, int on);
   6.162+
   6.163+/* bootargs */
   6.164+extern void bootargsinit(void);
   6.165+extern char *getconf(char *name);
   6.166+extern void setconfenv(void);
   6.167+extern void writeconf(void);
   6.168+
   6.169+/* screen */
   6.170+extern void screeninit(void);
   6.171+
   6.172+extern int isaconfig(char*, int, ISAConf*);
     7.1new file mode 100644
     7.2--- /dev/null
     7.3+++ b/sys/src/9/bcm64/fpu.c
     7.4@@ -0,0 +1,92 @@
     7.5+#include "u.h"
     7.6+#include "../port/lib.h"
     7.7+#include "mem.h"
     7.8+#include "dat.h"
     7.9+#include "fns.h"
    7.10+
    7.11+#include "ureg.h"
    7.12+#include "sysreg.h"
    7.13+
    7.14+/* libc */
    7.15+extern ulong getfcr(void);
    7.16+extern void setfcr(ulong fcr);
    7.17+extern ulong getfsr(void);
    7.18+extern void setfsr(ulong fsr);
    7.19+
    7.20+void
    7.21+fpuinit(void)
    7.22+{
    7.23+	fpoff();
    7.24+}
    7.25+
    7.26+void
    7.27+fpon(void)
    7.28+{
    7.29+	syswr(CPACR_EL1, 3<<20);
    7.30+}
    7.31+
    7.32+void
    7.33+fpoff(void)
    7.34+{
    7.35+	syswr(CPACR_EL1, 0<<20);
    7.36+}
    7.37+
    7.38+void
    7.39+fpinit(void)
    7.40+{
    7.41+	fpon();
    7.42+	setfcr(0);
    7.43+	setfsr(0);
    7.44+}
    7.45+
    7.46+void
    7.47+fpclear(void)
    7.48+{
    7.49+	fpoff();
    7.50+}
    7.51+
    7.52+void
    7.53+fpsave(FPsave *p)
    7.54+{
    7.55+	p->control = getfcr();
    7.56+	p->status = getfsr();
    7.57+	fpsaveregs(p->regs);
    7.58+	fpoff();
    7.59+}
    7.60+
    7.61+void
    7.62+fprestore(FPsave *p)
    7.63+{
    7.64+	fpon();
    7.65+	setfcr(p->control);
    7.66+	setfsr(p->status);
    7.67+	fploadregs(p->regs);
    7.68+}
    7.69+
    7.70+void
    7.71+mathtrap(Ureg*)
    7.72+{
    7.73+	int s;
    7.74+
    7.75+	if((up->fpstate & FPillegal) != 0){
    7.76+		postnote(up, 1, "sys: floating point in note handler", NDebug);
    7.77+		return;
    7.78+	}
    7.79+	switch(up->fpstate){
    7.80+	case FPinit:
    7.81+		s = splhi();
    7.82+		fpinit();
    7.83+		up->fpstate = FPactive;
    7.84+		splx(s);
    7.85+		break;
    7.86+	case FPinactive:
    7.87+		s = splhi();
    7.88+		fprestore(up->fpsave);
    7.89+		up->fpstate = FPactive;
    7.90+		splx(s);
    7.91+		break;
    7.92+	case FPactive:
    7.93+		postnote(up, 1, "sys: floating point error", NDebug);
    7.94+		break;
    7.95+	}
    7.96+}
     8.1new file mode 100644
     8.2--- /dev/null
     8.3+++ b/sys/src/9/bcm64/init9.s
     8.4@@ -0,0 +1,4 @@
     8.5+TEXT main(SB), 1, $8
     8.6+	MOV	$setSB(SB), R28		/* load the SB */
     8.7+	MOV	$boot(SB), R0
     8.8+	B	startboot(SB)
     9.1new file mode 100644
     9.2--- /dev/null
     9.3+++ b/sys/src/9/bcm64/l.s
     9.4@@ -0,0 +1,749 @@
     9.5+#include "mem.h"
     9.6+#include "sysreg.h"
     9.7+
     9.8+#undef	SYSREG
     9.9+#define	SYSREG(op0,op1,Cn,Cm,op2)	SPR(((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5))
    9.10+
    9.11+TEXT _start(SB), 1, $-4
    9.12+	MOV	$setSB-KZERO(SB), R28
    9.13+	BL	svcmode<>(SB)
    9.14+
    9.15+	/* use dedicated stack pointer per exception level */
    9.16+	MOVWU	$1, R1
    9.17+	MSR	R1, SPSel
    9.18+
    9.19+	BL	mmudisable<>(SB)
    9.20+
    9.21+	/* invalidate local caches */
    9.22+	BL	cachedinv(SB)
    9.23+	BL	cacheiinv(SB)
    9.24+
    9.25+	MOV	$(MACHADDR(0)-KZERO), R27
    9.26+	MRS	MPIDR_EL1, R1
    9.27+	ANDW	$(MAXMACH-1), R1
    9.28+	MOVWU	$MACHSIZE, R2
    9.29+	MULW	R1, R2, R2
    9.30+	SUB	R2, R27
    9.31+
    9.32+	ADD	$(MACHSIZE-16), R27, R2
    9.33+	MOV	R2, SP
    9.34+
    9.35+	CBNZ	R1, _startup
    9.36+
    9.37+	/* clear page table and machs */
    9.38+	MOV	$(L1-KZERO), R1
    9.39+	MOV	$(MACHADDR(-1)-KZERO), R2
    9.40+_zerol1:
    9.41+	MOV	ZR, (R1)8!
    9.42+	CMP	R1, R2
    9.43+	BNE	_zerol1
    9.44+
    9.45+	/* clear BSS */
    9.46+	MOV	$edata-KZERO(SB), R1
    9.47+	MOV	$end-KZERO(SB), R2
    9.48+_zerobss:
    9.49+	MOV	ZR, (R1)8!
    9.50+	CMP	R1, R2
    9.51+	BNE	_zerobss
    9.52+
    9.53+	/* setup page tables */
    9.54+	MOV	$(L1-KZERO), R0
    9.55+	BL	mmu0init(SB)
    9.56+
    9.57+	BL	cachedwbinv(SB)
    9.58+	BL	l2cacheuwbinv(SB)
    9.59+	SEVL
    9.60+_startup:
    9.61+	WFE
    9.62+	BL	mmuenable<>(SB)
    9.63+
    9.64+	MOV	$0, R26
    9.65+	ORR	$KZERO, R27
    9.66+	MSR	R27, TPIDR_EL1
    9.67+	MOV	$setSB(SB), R28
    9.68+
    9.69+	BL	main(SB)
    9.70+
    9.71+TEXT	stop<>(SB), 1, $-4
    9.72+_stop:
    9.73+	WFE
    9.74+	B	_stop
    9.75+
    9.76+TEXT sev(SB), 1, $-4
    9.77+	SEV
    9.78+	WFE
    9.79+	RETURN
    9.80+
    9.81+TEXT PUTC(SB), 1, $-4
    9.82+	MOVWU $(0x3F000000+0x215040), R14
    9.83+	MOVB R0, (R14)
    9.84+	RETURN
    9.85+
    9.86+TEXT svcmode<>(SB), 1, $-4
    9.87+	MSR	$0xF, DAIFSet
    9.88+	MRS	CurrentEL, R0
    9.89+	ANDW	$(3<<2), R0
    9.90+	CMPW	$(1<<2), R0
    9.91+	BEQ	el1
    9.92+	CMPW	$(2<<2), R0
    9.93+	BEQ	el2
    9.94+	B	stop<>(SB)
    9.95+el2:
    9.96+	MOV	$0, R0
    9.97+	MSR	R0, MDCR_EL2
    9.98+	ISB	$SY
    9.99+
   9.100+	/* HCR = RW, HCD, SWIO, BSU, FB */
   9.101+	MOVWU	$(1<<31 | 1<<29 | 1<<2 | 0<<10 | 0<<9), R0
   9.102+	MSR	R0, HCR_EL2
   9.103+	ISB	$SY
   9.104+
   9.105+	/* SCTLR = RES1 */
   9.106+	MOVWU	$(3<<4 | 1<<11 | 1<<16 | 1<<18 | 3<<22 | 3<<28), R0
   9.107+	ISB	$SY
   9.108+	MSR	R0, SCTLR_EL2
   9.109+	ISB	$SY
   9.110+
   9.111+	/* set VMID to zero */
   9.112+	MOV	$0, R0
   9.113+	MSR	R0, VTTBR_EL2
   9.114+	ISB	$SY
   9.115+
   9.116+	MOVWU	$(0xF<<6 | 4), R0
   9.117+	MSR	R0, SPSR_EL2
   9.118+	MSR	LR, ELR_EL2
   9.119+	ERET
   9.120+el1:
   9.121+	RETURN
   9.122+
   9.123+TEXT mmudisable<>(SB), 1, $-4
   9.124+#define SCTLRCLR \
   9.125+	/* RES0 */	( 3<<30 \
   9.126+	/* RES0 */	| 1<<27 \
   9.127+	/* UCI */	| 1<<26 \
   9.128+	/* EE */	| 1<<25 \
   9.129+	/* RES0 */	| 1<<21 \
   9.130+	/* E0E */	| 1<<24 \
   9.131+	/* WXN */	| 1<<19 \
   9.132+	/* nTWE */	| 1<<18 \
   9.133+	/* RES0 */	| 1<<17 \
   9.134+	/* nTWI */	| 1<<16 \
   9.135+	/* UCT */	| 1<<15 \
   9.136+	/* DZE */	| 1<<14 \
   9.137+	/* RES0 */	| 1<<13 \
   9.138+	/* RES0 */	| 1<<10 \
   9.139+	/* UMA */	| 1<<9 \
   9.140+	/* SA0 */	| 1<<4 \
   9.141+	/* SA */	| 1<<3 \
   9.142+	/* A */		| 1<<1 )
   9.143+#define SCTLRSET \
   9.144+	/* RES1 */	( 3<<28 \
   9.145+	/* RES1 */	| 3<<22 \
   9.146+	/* RES1 */	| 1<<20 \
   9.147+	/* RES1 */	| 1<<11 )
   9.148+#define SCTLRMMU \
   9.149+	/* I */		( 1<<12 \
   9.150+	/* C */		| 1<<2 \
   9.151+	/* M */		| 1<<0 )
   9.152+
   9.153+	/* initialise SCTLR, MMU and caches off */
   9.154+	ISB	$SY
   9.155+	MRS	SCTLR_EL1, R0
   9.156+	BIC	$(SCTLRCLR | SCTLRMMU), R0
   9.157+	ORR	$SCTLRSET, R0
   9.158+	ISB	$SY
   9.159+	MSR	R0, SCTLR_EL1
   9.160+	ISB	$SY
   9.161+
   9.162+	B	flushlocaltlb(SB)
   9.163+
   9.164+TEXT mmuenable<>(SB), 1, $-4
   9.165+	/* return to virtual */
   9.166+	ORR	$KZERO, LR
   9.167+	MOV	LR, -16(RSP)!
   9.168+
   9.169+	BL	cachedwbinv(SB)
   9.170+	BL	flushlocaltlb(SB)
   9.171+
   9.172+	/* memory attributes */
   9.173+#define MAIRINIT \
   9.174+	( 0xFF << MA_MEM_WB*8 \
   9.175+	| 0x33 << MA_MEM_WT*8 \
   9.176+	| 0x44 << MA_MEM_UC*8 \
   9.177+	| 0x00 << MA_DEV_nGnRnE*8 \
   9.178+	| 0x04 << MA_DEV_nGnRE*8 \
   9.179+	| 0x08 << MA_DEV_nGRE*8 \
   9.180+	| 0x0C << MA_DEV_GRE*8 )
   9.181+	MOV	$MAIRINIT, R1
   9.182+	MSR	R1, MAIR_EL1
   9.183+	ISB	$SY
   9.184+
   9.185+	/* translation control */
   9.186+#define TCRINIT \
   9.187+	/* TBI1 */	( 0<<38 \
   9.188+	/* TBI0 */	| 0<<37 \
   9.189+	/* AS */	| 0<<36 \
   9.190+	/* TG1 */	| (((3<<16|1<<14|2<<12)>>PGSHIFT)&3)<<30 \
   9.191+	/* SH1 */	| SHARE_INNER<<28 \
   9.192+	/* ORGN1 */	| CACHE_WB<<26 \
   9.193+	/* IRGN1 */	| CACHE_WB<<24 \
   9.194+	/* EPD1 */	| 0<<23 \
   9.195+	/* A1 */	| 0<<22 \
   9.196+	/* T1SZ */	| (64-EVASHIFT)<<16 \
   9.197+	/* TG0 */	| (((1<<16|2<<14|0<<12)>>PGSHIFT)&3)<<14 \
   9.198+	/* SH0 */	| SHARE_INNER<<12 \
   9.199+	/* ORGN0 */	| CACHE_WB<<10 \
   9.200+	/* IRGN0 */	| CACHE_WB<<8 \
   9.201+	/* EPD0 */	| 0<<7 \
   9.202+	/* T0SZ */	| (64-EVASHIFT)<<0 )
   9.203+	MOV	$TCRINIT, R1
   9.204+	MRS	ID_AA64MMFR0_EL1, R2
   9.205+	ANDW	$0xF, R2	// IPS
   9.206+	ADD	R2<<32, R1
   9.207+	MSR	R1, TCR_EL1
   9.208+	ISB	$SY
   9.209+
   9.210+	/* load the page tables */
   9.211+	MOV	$(L1TOP-KZERO), R0
   9.212+	ISB	$SY
   9.213+	MSR	R0, TTBR0_EL1
   9.214+	MSR	R0, TTBR1_EL1
   9.215+	ISB	$SY
   9.216+
   9.217+	/* enable MMU and caches */
   9.218+	MRS	SCTLR_EL1, R1
   9.219+	ORR	$SCTLRMMU, R1
   9.220+	ISB	$SY
   9.221+	MSR	R1, SCTLR_EL1
   9.222+	ISB	$SY
   9.223+
   9.224+	MOV	RSP, R1
   9.225+	ORR	$KZERO, R1
   9.226+	MOV	R1, RSP
   9.227+	MOV	(RSP)16!, LR
   9.228+	B	cacheiinv(SB)
   9.229+
   9.230+TEXT touser(SB), 1, $-4
   9.231+	MSR	$0x3, DAIFSet	// interrupts off
   9.232+	MOVWU	$0x10028, R1	// entry
   9.233+	MOVWU	$0, R2		// psr
   9.234+	MSR	R0, SP_EL0	// sp
   9.235+	MSR	R1, ELR_EL1
   9.236+	MSR	R2, SPSR_EL1
   9.237+	ERET
   9.238+
   9.239+TEXT cas(SB), 1, $-4
   9.240+TEXT cmpswap(SB), 1, $-4
   9.241+	MOVW	ov+8(FP), R1
   9.242+	MOVW	nv+16(FP), R2
   9.243+_cas1:
   9.244+	LDXRW	(R0), R3
   9.245+	CMP	R3, R1
   9.246+	BNE	_cas0
   9.247+	STXRW	R2, (R0), R4
   9.248+	CBNZ	R4, _cas1
   9.249+	MOVW	$1, R0
   9.250+	DMB	$ISH
   9.251+	RETURN
   9.252+_cas0:
   9.253+	CLREX
   9.254+	MOVW	$0, R0
   9.255+	RETURN
   9.256+
   9.257+TEXT tas(SB), 1, $-4
   9.258+TEXT _tas(SB), 1, $-4
   9.259+	MOVW	$0xdeaddead, R2
   9.260+_tas1:
   9.261+	LDXRW	(R0), R1
   9.262+	STXRW	R2, (R0), R3
   9.263+	CBNZ	R3, _tas1
   9.264+	MOVW	R1, R0
   9.265+
   9.266+TEXT coherence(SB), 1, $-4
   9.267+	DMB	$ISH
   9.268+	RETURN
   9.269+
   9.270+TEXT islo(SB), 1, $-4
   9.271+	MRS	DAIF, R0
   9.272+	AND	$(0x2<<6), R0
   9.273+	EOR	$(0x2<<6), R0
   9.274+	RETURN
   9.275+
   9.276+TEXT splhi(SB), 1, $-4
   9.277+	MRS	DAIF, R0
   9.278+	MSR	$0x2, DAIFSet
   9.279+	RETURN
   9.280+
   9.281+TEXT splfhi(SB), 1, $-4
   9.282+	MRS	DAIF, R0
   9.283+	MSR	$0x3, DAIFSet
   9.284+	RETURN
   9.285+
   9.286+TEXT spllo(SB), 1, $-4
   9.287+	MSR	$0x3, DAIFClr
   9.288+	RETURN
   9.289+
   9.290+TEXT splflo(SB), 1, $-4
   9.291+	MSR	$0x1, DAIFClr
   9.292+	RETURN
   9.293+
   9.294+TEXT splx(SB), 1, $-4
   9.295+	MSR	R0, DAIF
   9.296+	RETURN
   9.297+
   9.298+TEXT cycles(SB), 1, $-4
   9.299+TEXT lcycles(SB), 1, $-4
   9.300+	MRS	PMCCNTR_EL0, R0
   9.301+	RETURN
   9.302+
   9.303+TEXT setlabel(SB), 1, $-4
   9.304+	MOV	LR, 8(R0)
   9.305+	MOV	SP, R1
   9.306+	MOV	R1, 0(R0)
   9.307+	MOVW	$0, R0
   9.308+	RETURN
   9.309+
   9.310+TEXT gotolabel(SB), 1, $-4
   9.311+	MOV	8(R0), LR	/* link */
   9.312+	MOV	0(R0), R1	/* sp */
   9.313+	MOV	R1, SP
   9.314+	MOVW	$1, R0
   9.315+	RETURN
   9.316+
   9.317+TEXT returnto(SB), 1, $-4
   9.318+	MOV	R0, 0(SP)
   9.319+	RETURN
   9.320+
   9.321+TEXT getfar(SB), 1, $-4
   9.322+	MRS	FAR_EL1, R0
   9.323+	RETURN
   9.324+
   9.325+TEXT setttbr(SB), 1, $-4
   9.326+	DSB	$ISHST
   9.327+	MSR	R0, TTBR0_EL1
   9.328+	DSB	$ISH
   9.329+	ISB	$SY
   9.330+
   9.331+	B	cacheiinv(SB)
   9.332+
   9.333+TEXT magic(SB), 1, $-4
   9.334+	DSB	$SY
   9.335+	ISB	$SY
   9.336+	DSB	$SY
   9.337+	ISB	$SY
   9.338+	DSB	$SY
   9.339+	ISB	$SY
   9.340+	DSB	$SY
   9.341+	ISB	$SY
   9.342+	RETURN
   9.343+
   9.344+/*
   9.345+ * TLB maintenance operations.
   9.346+ * these broadcast to all cpu's in the cluser
   9.347+ * (inner sharable domain).
   9.348+ */
   9.349+TEXT flushasidva(SB), 1, $-4
   9.350+TEXT tlbivae1is(SB), 1, $-4
   9.351+	DSB	$ISHST
   9.352+	TLBI	R0, 0,8,3,1	/* VAE1IS */
   9.353+	DSB	$ISH
   9.354+	ISB	$SY
   9.355+	RETURN
   9.356+
   9.357+TEXT flushasidvall(SB), 1, $-4
   9.358+TEXT tlbivale1is(SB), 1, $-4
   9.359+	DSB	$ISHST
   9.360+	TLBI	R0, 0,8,3,5	/* VALE1IS */
   9.361+	DSB	$ISH
   9.362+	ISB	$SY
   9.363+	RETURN
   9.364+
   9.365+TEXT flushasid(SB), 1, $-4
   9.366+TEXT tlbiaside1is(SB), 1, $-4
   9.367+	DSB	$ISHST
   9.368+	TLBI	R0, 0,8,3,2	/* ASIDE1IS */
   9.369+	DSB	$ISH
   9.370+	ISB	$SY
   9.371+	RETURN
   9.372+
   9.373+TEXT flushtlb(SB), 1, $-4
   9.374+TEXT tlbivmalle1is(SB), 1, $-4
   9.375+	DSB	$ISHST
   9.376+	TLBI	R0, 0,8,3,0	/* VMALLE1IS */
   9.377+	DSB	$ISH
   9.378+	ISB	$SY
   9.379+	RETURN
   9.380+
   9.381+/*
   9.382+ * flush the tlb of this cpu. no broadcast.
   9.383+ */
   9.384+TEXT flushlocaltlb(SB), 1, $-4
   9.385+TEXT tlbivmalle1(SB), 1, $-4
   9.386+	DSB	$NSHST
   9.387+	TLBI	R0, 0,8,7,0	/* VMALLE1 */
   9.388+	DSB	$NSH
   9.389+	ISB	$SY
   9.390+	RETURN
   9.391+
   9.392+TEXT fpsaveregs(SB), 1, $-4
   9.393+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 0)  /* MOV { V0, V1, V2, V3  }, (R0)64! */
   9.394+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 4)  /* MOV { V4, V5, V6, V7  }, (R0)64! */
   9.395+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 8)  /* MOV { V8, V9, V10,V11 }, (R0)64! */
   9.396+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 12) /* MOV { V12,V13,V14,V15 }, (R0)64! */
   9.397+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 16) /* MOV { V16,V17,V18,V19 }, (R0)64! */
   9.398+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 20) /* MOV { V20,V21,V22,V23 }, (R0)64! */
   9.399+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 24) /* MOV { V24,V25,V26,V27 }, (R0)64! */
   9.400+	WORD	$(1<<30 | 3 << 26 | 2<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 28) /* MOV { V28,V29,V30,V31 }, (R0)64! */
   9.401+	RETURN
   9.402+
   9.403+TEXT fploadregs(SB), 1, $-4
   9.404+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 0)  /* MOV (R0)64!, { V0, V1, V2, V3  } */
   9.405+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 4)  /* MOV (R0)64!, { V4, V5, V6, V7  } */
   9.406+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 8)  /* MOV (R0)64!, { V8, V9, V10,V11 } */
   9.407+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 12) /* MOV (R0)64!, { V12,V13,V14,V15 } */
   9.408+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 16) /* MOV (R0)64!, { V16,V17,V18,V19 } */
   9.409+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 20) /* MOV (R0)64!, { V20,V21,V22,V23 } */
   9.410+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 24) /* MOV (R0)64!, { V24,V25,V26,V27 } */
   9.411+	WORD	$(1<<30 | 3 << 26 | 3<<22 | 0x1F<<16 | 3<<10 | 0<<5 | 28) /* MOV (R0)64!, { V28,V29,V30,V31 } */
   9.412+	RETURN
   9.413+
   9.414+// syscall or trap from EL0
   9.415+TEXT vsys0(SB), 1, $-4
   9.416+	LSRW	$26, R0, R17	// ec
   9.417+	CMPW	$0x15, R17	// SVC trap?
   9.418+	BNE	_itsatrap	// nope.
   9.419+
   9.420+	MOV	R26, 224(RSP)	// special
   9.421+	MOV	R27, 232(RSP)	// special
   9.422+	MOV	R28, 240(RSP)	// sb
   9.423+	MOV	R29, 248(RSP)	// special
   9.424+
   9.425+	MRS	SP_EL0, R1
   9.426+	MRS	ELR_EL1, R2
   9.427+	MRS	SPSR_EL1, R3
   9.428+
   9.429+	MOV	R0, 288(RSP)	// type
   9.430+	MOV	R1, 264(RSP)	// sp
   9.431+	MOV	R2, 272(RSP)	// pc
   9.432+	MOV	R3, 280(RSP)	// psr
   9.433+
   9.434+	MOV	$setSB(SB), R28
   9.435+	MRS	TPIDR_EL1, R27
   9.436+	MOV	16(R27), R26
   9.437+
   9.438+	ADD	$16, RSP, R0	// ureg
   9.439+	BL	syscall(SB)
   9.440+
   9.441+TEXT forkret(SB), 1, $-4
   9.442+	MSR	$0x3, DAIFSet	// interrupts off
   9.443+
   9.444+	ADD	$16, RSP, R0	// ureg
   9.445+
   9.446+	MOV	16(RSP), R0	// ret
   9.447+	MOV	264(RSP), R1	// sp
   9.448+	MOV	272(RSP), R2	// pc
   9.449+	MOV	280(RSP), R3	// psr
   9.450+
   9.451+	MSR	R1, SP_EL0
   9.452+	MSR	R2, ELR_EL1
   9.453+	MSR	R3, SPSR_EL1
   9.454+
   9.455+	MOV	224(RSP), R26	// special
   9.456+	MOV	232(RSP), R27	// special
   9.457+	MOV	240(RSP), R28	// sb
   9.458+	MOV	248(RSP), R29	// special
   9.459+
   9.460+	MOV	256(RSP), R30	// link
   9.461+
   9.462+	ADD	$TRAPFRAMESIZE, RSP
   9.463+	ERET
   9.464+
   9.465+TEXT itsatrap<>(SB), 1, $-4
   9.466+_itsatrap:
   9.467+	MOV	R1, 24(RSP)
   9.468+	MOV	R2, 32(RSP)
   9.469+	MOV	R3, 40(RSP)
   9.470+	MOV	R4, 48(RSP)
   9.471+	MOV	R5, 56(RSP)
   9.472+	MOV	R6, 64(RSP)
   9.473+	MOV	R7, 72(RSP)
   9.474+	MOV	R8, 80(RSP)
   9.475+	MOV	R9, 88(RSP)
   9.476+	MOV	R10, 96(RSP)
   9.477+	MOV	R11, 104(RSP)
   9.478+	MOV	R12, 112(RSP)
   9.479+	MOV	R13, 120(RSP)
   9.480+	MOV	R14, 128(RSP)
   9.481+	MOV	R15, 136(RSP)
   9.482+	MOV	R16, 144(RSP)
   9.483+
   9.484+	MOV	R18, 160(RSP)
   9.485+	MOV	R19, 168(RSP)
   9.486+	MOV	R20, 176(RSP)
   9.487+	MOV	R21, 184(RSP)
   9.488+	MOV	R22, 192(RSP)
   9.489+	MOV	R23, 200(RSP)
   9.490+	MOV	R24, 208(RSP)
   9.491+	MOV	R25, 216(RSP)
   9.492+
   9.493+// trap/irq/fiq/serr from EL0
   9.494+TEXT vtrap0(SB), 1, $-4
   9.495+	MOV	R26, 224(RSP)	// special
   9.496+	MOV	R27, 232(RSP)	// special
   9.497+	MOV	R28, 240(RSP)	// sb
   9.498+	MOV	R29, 248(RSP)	// special
   9.499+
   9.500+	MRS	SP_EL0, R1
   9.501+	MRS	ELR_EL1, R2
   9.502+	MRS	SPSR_EL1, R3
   9.503+
   9.504+	MOV	R0, 288(RSP)	// type
   9.505+	MOV	R1, 264(RSP)	// sp
   9.506+	MOV	R2, 272(RSP)	// pc
   9.507+	MOV	R3, 280(RSP)	// psr
   9.508+
   9.509+	MOV	$setSB(SB), R28
   9.510+	MRS	TPIDR_EL1, R27
   9.511+	MOV	16(R27), R26
   9.512+
   9.513+	ADD	$16, RSP, R0	// ureg
   9.514+	BL	trap(SB)
   9.515+
   9.516+TEXT noteret(SB), 1, $-4
   9.517+	MSR	$0x3, DAIFSet	// interrupts off
   9.518+
   9.519+	ADD	$16, RSP, R0	// ureg
   9.520+
   9.521+	MOV	264(RSP), R1	// sp
   9.522+	MOV	272(RSP), R2	// pc
   9.523+	MOV	280(RSP), R3	// psr
   9.524+
   9.525+	MSR	R1, SP_EL0
   9.526+	MSR	R2, ELR_EL1
   9.527+	MSR	R3, SPSR_EL1
   9.528+
   9.529+	MOV	224(RSP), R26	// special
   9.530+	MOV	232(RSP), R27	// special
   9.531+	MOV	240(RSP), R28	// sb
   9.532+	MOV	248(RSP), R29	// special
   9.533+
   9.534+_intrreturn:
   9.535+	MOV	16(RSP), R0
   9.536+	MOV	24(RSP), R1
   9.537+	MOV	32(RSP), R2
   9.538+	MOV	40(RSP), R3
   9.539+	MOV	48(RSP), R4
   9.540+	MOV	56(RSP), R5
   9.541+	MOV	64(RSP), R6
   9.542+	MOV	72(RSP), R7
   9.543+	MOV	80(RSP), R8
   9.544+	MOV	88(RSP), R9
   9.545+	MOV	96(RSP), R10
   9.546+	MOV	104(RSP), R11
   9.547+	MOV	112(RSP), R12
   9.548+	MOV	120(RSP), R13
   9.549+	MOV	128(RSP), R14
   9.550+	MOV	136(RSP), R15
   9.551+	MOV	144(RSP), R16
   9.552+	MOV	152(RSP), R17
   9.553+	MOV	160(RSP), R18
   9.554+	MOV	168(RSP), R19
   9.555+	MOV	176(RSP), R20
   9.556+	MOV	184(RSP), R21
   9.557+	MOV	192(RSP), R22
   9.558+	MOV	200(RSP), R23
   9.559+	MOV	208(RSP), R24
   9.560+	MOV	216(RSP), R25
   9.561+
   9.562+	MOV	256(RSP), R30	// link
   9.563+
   9.564+	ADD	$TRAPFRAMESIZE, RSP
   9.565+	ERET
   9.566+
   9.567+// irq/fiq/trap/serr from EL1
   9.568+TEXT vtrap1(SB), 1, $-4
   9.569+	MOV	R29, 248(RSP)	// special
   9.570+
   9.571+	ADD	$TRAPFRAMESIZE, RSP, R1
   9.572+	MRS	ELR_EL1, R2
   9.573+	MRS	SPSR_EL1, R3
   9.574+
   9.575+	MOV	R0, 288(RSP)	// type
   9.576+	MOV	R1, 264(RSP)	// sp
   9.577+	MOV	R2, 272(RSP)	// pc
   9.578+	MOV	R3, 280(RSP)	// psr
   9.579+
   9.580+	ADD	$16, RSP, R0	// ureg
   9.581+	BL	trap(SB)
   9.582+
   9.583+	MSR	$0x3, DAIFSet	// interrupts off
   9.584+
   9.585+	MOV	272(RSP), R2	// pc
   9.586+	MOV	280(RSP), R3	// psr
   9.587+
   9.588+	MSR	R2, ELR_EL1
   9.589+	MSR	R3, SPSR_EL1
   9.590+
   9.591+	MOV	248(RSP), R29	// special
   9.592+	B	_intrreturn	
   9.593+
   9.594+// vector tables
   9.595+TEXT vsys(SB), 1, $-4
   9.596+	SUB	$TRAPFRAMESIZE, RSP
   9.597+
   9.598+	MOV	R0, 16(RSP)
   9.599+	MOV	R30, 256(RSP)	// link
   9.600+
   9.601+	MOV	R17, 152(RSP)	// temp
   9.602+
   9.603+	MRS	ESR_EL1, R0	// type
   9.604+
   9.605+_vsyspatch:
   9.606+	B	_vsyspatch	// branch to vsys0() patched in
   9.607+
   9.608+TEXT vtrap(SB), 1, $-4
   9.609+	SUB	$TRAPFRAMESIZE, RSP
   9.610+
   9.611+	MOV	R0, 16(RSP)
   9.612+	MOV	R1, 24(RSP)
   9.613+	MOV	R2, 32(RSP)
   9.614+	MOV	R3, 40(RSP)
   9.615+	MOV	R4, 48(RSP)
   9.616+	MOV	R5, 56(RSP)
   9.617+	MOV	R6, 64(RSP)
   9.618+	MOV	R7, 72(RSP)
   9.619+	MOV	R8, 80(RSP)
   9.620+	MOV	R9, 88(RSP)
   9.621+	MOV	R10, 96(RSP)
   9.622+	MOV	R11, 104(RSP)
   9.623+	MOV	R12, 112(RSP)
   9.624+	MOV	R13, 120(RSP)
   9.625+	MOV	R14, 128(RSP)
   9.626+	MOV	R15, 136(RSP)
   9.627+	MOV	R16, 144(RSP)
   9.628+	MOV	R17, 152(RSP)
   9.629+	MOV	R18, 160(RSP)
   9.630+	MOV	R19, 168(RSP)
   9.631+	MOV	R20, 176(RSP)
   9.632+	MOV	R21, 184(RSP)
   9.633+	MOV	R22, 192(RSP)
   9.634+	MOV	R23, 200(RSP)
   9.635+	MOV	R24, 208(RSP)
   9.636+	MOV	R25, 216(RSP)
   9.637+
   9.638+	MOV	R30, 256(RSP)	// link
   9.639+
   9.640+	MRS	ESR_EL1, R0	// type
   9.641+
   9.642+_vtrappatch:
   9.643+	B	_vtrappatch	// branch to vtrapX() patched in
   9.644+
   9.645+TEXT virq(SB), 1, $-4
   9.646+	SUB	$TRAPFRAMESIZE, RSP
   9.647+
   9.648+	MOV	R0, 16(RSP)
   9.649+	MOV	R1, 24(RSP)
   9.650+	MOV	R2, 32(RSP)
   9.651+	MOV	R3, 40(RSP)
   9.652+	MOV	R4, 48(RSP)
   9.653+	MOV	R5, 56(RSP)
   9.654+	MOV	R6, 64(RSP)
   9.655+	MOV	R7, 72(RSP)
   9.656+	MOV	R8, 80(RSP)
   9.657+	MOV	R9, 88(RSP)
   9.658+	MOV	R10, 96(RSP)
   9.659+	MOV	R11, 104(RSP)
   9.660+	MOV	R12, 112(RSP)
   9.661+	MOV	R13, 120(RSP)
   9.662+	MOV	R14, 128(RSP)
   9.663+	MOV	R15, 136(RSP)
   9.664+	MOV	R16, 144(RSP)
   9.665+	MOV	R17, 152(RSP)
   9.666+	MOV	R18, 160(RSP)
   9.667+	MOV	R19, 168(RSP)
   9.668+	MOV	R20, 176(RSP)
   9.669+	MOV	R21, 184(RSP)
   9.670+	MOV	R22, 192(RSP)
   9.671+	MOV	R23, 200(RSP)
   9.672+	MOV	R24, 208(RSP)
   9.673+	MOV	R25, 216(RSP)
   9.674+
   9.675+	MOV	R30, 256(RSP)	// link
   9.676+
   9.677+	MOV	$(1<<32), R0	// type irq
   9.678+
   9.679+_virqpatch:
   9.680+	B	_virqpatch	// branch to vtrapX() patched in
   9.681+
   9.682+TEXT vfiq(SB), 1, $-4
   9.683+	SUB	$TRAPFRAMESIZE, RSP
   9.684+
   9.685+	MOV	R0, 16(RSP)
   9.686+	MOV	R1, 24(RSP)
   9.687+	MOV	R2, 32(RSP)
   9.688+	MOV	R3, 40(RSP)
   9.689+	MOV	R4, 48(RSP)
   9.690+	MOV	R5, 56(RSP)
   9.691+	MOV	R6, 64(RSP)
   9.692+	MOV	R7, 72(RSP)
   9.693+	MOV	R8, 80(RSP)
   9.694+	MOV	R9, 88(RSP)
   9.695+	MOV	R10, 96(RSP)
   9.696+	MOV	R11, 104(RSP)
   9.697+	MOV	R12, 112(RSP)
   9.698+	MOV	R13, 120(RSP)
   9.699+	MOV	R14, 128(RSP)
   9.700+	MOV	R15, 136(RSP)
   9.701+	MOV	R16, 144(RSP)
   9.702+	MOV	R17, 152(RSP)
   9.703+	MOV	R18, 160(RSP)
   9.704+	MOV	R19, 168(RSP)
   9.705+	MOV	R20, 176(RSP)
   9.706+	MOV	R21, 184(RSP)
   9.707+	MOV	R22, 192(RSP)
   9.708+	MOV	R23, 200(RSP)
   9.709+	MOV	R24, 208(RSP)
   9.710+	MOV	R25, 216(RSP)
   9.711+
   9.712+	MOV	R30, 256(RSP)	// link
   9.713+	MOV	$(2<<32), R0	// type fiq
   9.714+
   9.715+_vfiqpatch:
   9.716+	B	_vfiqpatch	// branch to vtrapX() patched in
   9.717+
   9.718+TEXT vserr(SB), 1, $-4
   9.719+	SUB	$TRAPFRAMESIZE, RSP
   9.720+
   9.721+	MOV	R0, 16(RSP)
   9.722+	MOV	R1, 24(RSP)
   9.723+	MOV	R2, 32(RSP)
   9.724+	MOV	R3, 40(RSP)
   9.725+	MOV	R4, 48(RSP)
   9.726+	MOV	R5, 56(RSP)
   9.727+	MOV	R6, 64(RSP)
   9.728+	MOV	R7, 72(RSP)
   9.729+	MOV	R8, 80(RSP)
   9.730+	MOV	R9, 88(RSP)
   9.731+	MOV	R10, 96(RSP)
   9.732+	MOV	R11, 104(RSP)
   9.733+	MOV	R12, 112(RSP)
   9.734+	MOV	R13, 120(RSP)
   9.735+	MOV	R14, 128(RSP)
   9.736+	MOV	R15, 136(RSP)
   9.737+	MOV	R16, 144(RSP)
   9.738+	MOV	R17, 152(RSP)
   9.739+	MOV	R18, 160(RSP)
   9.740+	MOV	R19, 168(RSP)
   9.741+	MOV	R20, 176(RSP)
   9.742+	MOV	R21, 184(RSP)
   9.743+	MOV	R22, 192(RSP)
   9.744+	MOV	R23, 200(RSP)
   9.745+	MOV	R24, 208(RSP)
   9.746+	MOV	R25, 216(RSP)
   9.747+
   9.748+	MOV	R30, 256(RSP)	// link
   9.749+
   9.750+	MRS	ESR_EL1, R0
   9.751+	ORR	$(3<<32), R0	// type
   9.752+_vserrpatch:
   9.753+	B	_vserrpatch	// branch to vtrapX() patched in
    10.1new file mode 100644
    10.2--- /dev/null
    10.3+++ b/sys/src/9/bcm64/main.c
    10.4@@ -0,0 +1,337 @@
    10.5+#include "u.h"
    10.6+#include "tos.h"
    10.7+#include "../port/lib.h"
    10.8+#include "mem.h"
    10.9+#include "dat.h"
   10.10+#include "fns.h"
   10.11+#include "../port/error.h"
   10.12+#include "io.h"
   10.13+#include "init.h"
   10.14+#include "sysreg.h"
   10.15+
   10.16+#include <pool.h>
   10.17+#include <libsec.h>
   10.18+
   10.19+Conf conf;
   10.20+ulong memsize = GiB;
   10.21+
   10.22+/*
   10.23+ *  starting place for first process
   10.24+ */
   10.25+void
   10.26+init0(void)
   10.27+{
   10.28+	char buf[2*KNAMELEN], **sp;
   10.29+
   10.30+	up->nerrlab = 0;
   10.31+	spllo();
   10.32+
   10.33+	/*
   10.34+	 * These are o.k. because rootinit is null.
   10.35+	 * Then early kproc's will have a root and dot.
   10.36+	 */
   10.37+	up->slash = namec("#/", Atodir, 0, 0);
   10.38+	pathclose(up->slash->path);
   10.39+	up->slash->path = newpath("/");
   10.40+	up->dot = cclone(up->slash);
   10.41+	chandevinit();
   10.42+
   10.43+	if(!waserror()){
   10.44+		snprint(buf, sizeof(buf), "%s %s", "ARM64", conffile);
   10.45+		ksetenv("terminal", buf, 0);
   10.46+		ksetenv("cputype", "arm64", 0);
   10.47+		if(cpuserver)
   10.48+			ksetenv("service", "cpu", 0);
   10.49+		else
   10.50+			ksetenv("service", "terminal", 0);
   10.51+		snprint(buf, sizeof(buf), "-a %s", getethermac());
   10.52+		ksetenv("etherargs", buf, 0);
   10.53+
   10.54+		/* convert plan9.ini variables to #e and #ec */
   10.55+		setconfenv();
   10.56+		poperror();
   10.57+	}
   10.58+	kproc("alarm", alarmkproc, 0);
   10.59+
   10.60+	sp = (char**)(USTKTOP-sizeof(Tos) - 8 - sizeof(sp[0])*4);
   10.61+	sp[3] = sp[2] = sp[1] = nil;
   10.62+	strcpy(sp[1] = (char*)&sp[4], "boot");
   10.63+	sp[0] = (void*)&sp[1];
   10.64+
   10.65+	touser((uintptr)sp);
   10.66+
   10.67+	assert(0);			/* shouldn't have returned */
   10.68+}
   10.69+
   10.70+/*
   10.71+ *  create the first process
   10.72+ */
   10.73+void
   10.74+userinit(void)
   10.75+{
   10.76+	Proc *p;
   10.77+	Segment *s;
   10.78+	KMap *k;
   10.79+	Page *pg;
   10.80+
   10.81+	/* no processes yet */
   10.82+	up = nil;
   10.83+
   10.84+	p = newproc();
   10.85+	p->pgrp = newpgrp();
   10.86+	p->egrp = smalloc(sizeof(Egrp));
   10.87+	p->egrp->ref = 1;
   10.88+	p->fgrp = dupfgrp(nil);
   10.89+	p->rgrp = newrgrp();
   10.90+	p->procmode = 0640;
   10.91+
   10.92+	kstrdup(&eve, "");
   10.93+	kstrdup(&p->text, "*init*");
   10.94+	kstrdup(&p->user, eve);
   10.95+
   10.96+	/*
   10.97+	 * Kernel Stack
   10.98+	 */
   10.99+	p->sched.pc = (uintptr)init0;
  10.100+	p->sched.sp = (uintptr)p->kstack+KSTACK-sizeof(up->s.args)-sizeof(uintptr);
  10.101+	p->sched.sp = STACKALIGN(p->sched.sp);
  10.102+	*(void**)p->sched.sp = kproc; // fake
  10.103+
  10.104+	/*
  10.105+	 * User Stack
  10.106+	 *
  10.107+	 * Technically, newpage can't be called here because it
  10.108+	 * should only be called when in a user context as it may
  10.109+	 * try to sleep if there are no pages available, but that
  10.110+	 * shouldn't be the case here.
  10.111+	 */
  10.112+	s = newseg(SG_STACK, USTKTOP-USTKSIZE, USTKSIZE/BY2PG);
  10.113+	s->flushme++;
  10.114+	p->seg[SSEG] = s;
  10.115+	pg = newpage(1, 0, USTKTOP-BY2PG);
  10.116+	segpage(s, pg);
  10.117+	k = kmap(pg);
  10.118+	memset((void*)VA(k), 0, BY2PG);
  10.119+	kunmap(k);
  10.120+
  10.121+	/*
  10.122+	 * Text
  10.123+	 */
  10.124+	s = newseg(SG_TEXT, UTZERO, 1);
  10.125+	p->seg[TSEG] = s;
  10.126+	pg = newpage(1, 0, UTZERO);
  10.127+	pg->txtflush = ~0;
  10.128+	segpage(s, pg);
  10.129+	k = kmap(s->map[0]->pages[0]);
  10.130+	memmove((void*)VA(k), initcode, sizeof initcode);
  10.131+	kunmap(k);
  10.132+
  10.133+	ready(p);
  10.134+}
  10.135+
  10.136+void
  10.137+confinit(void)
  10.138+{
  10.139+	int i, userpcnt;
  10.140+	ulong kpages;
  10.141+	uintptr pa;
  10.142+	char *p;
  10.143+
  10.144+	if(p = getconf("service")){
  10.145+		if(strcmp(p, "cpu") == 0)
  10.146+			cpuserver = 1;
  10.147+		else if(strcmp(p,"terminal") == 0)
  10.148+			cpuserver = 0;
  10.149+	}
  10.150+
  10.151+	if(p = getconf("*kernelpercent"))
  10.152+		userpcnt = 100 - strtol(p, 0, 0);
  10.153+	else
  10.154+		userpcnt = 0;
  10.155+
  10.156+	if((p = getconf("*maxmem")) != nil){
  10.157+		memsize = strtoul(p, 0, 0) - PHYSDRAM;
  10.158+		if (memsize < 16*MB)		/* sanity */
  10.159+			memsize = 16*MB;
  10.160+	}
  10.161+
  10.162+	getramsize(&conf.mem[0]);
  10.163+	if(conf.mem[0].limit == 0){
  10.164+		conf.mem[0].base = PHYSDRAM;
  10.165+		conf.mem[0].limit = PHYSDRAM + memsize;
  10.166+	}else if(p != nil)
  10.167+		conf.mem[0].limit = conf.mem[0].base + memsize;
  10.168+
  10.169+	conf.npage = 0;
  10.170+	pa = PADDR(PGROUND((uintptr)end));
  10.171+
  10.172+	/*
  10.173+	 *  we assume that the kernel is at the beginning of one of the
  10.174+	 *  contiguous chunks of memory and fits therein.
  10.175+	 */
  10.176+	for(i=0; i<nelem(conf.mem); i++){
  10.177+		/* take kernel out of allocatable space */
  10.178+		if(pa > conf.mem[i].base && pa < conf.mem[i].limit)
  10.179+			conf.mem[i].base = pa;
  10.180+
  10.181+		conf.mem[i].npage = (conf.mem[i].limit - conf.mem[i].base)/BY2PG;
  10.182+		conf.npage += conf.mem[i].npage;
  10.183+	}
  10.184+
  10.185+	if(userpcnt < 10)
  10.186+		userpcnt = 60 + cpuserver*10;
  10.187+	kpages = conf.npage - (conf.npage*userpcnt)/100;
  10.188+
  10.189+	/*
  10.190+	 * can't go past the end of virtual memory
  10.191+	 * (uintptr)-KZERO is 2^32 - KZERO
  10.192+	 */
  10.193+	if(kpages > ((uintptr)-KZERO)/BY2PG)
  10.194+		kpages = ((uintptr)-KZERO)/BY2PG;
  10.195+
  10.196+	conf.upages = conf.npage - kpages;
  10.197+	conf.ialloc = (kpages/2)*BY2PG;
  10.198+
  10.199+	conf.nmach = getncpus();
  10.200+
  10.201+	/* set up other configuration parameters */
  10.202+	conf.nproc = 100 + ((conf.npage*BY2PG)/MB)*5;
  10.203+	if(cpuserver)
  10.204+		conf.nproc *= 3;
  10.205+	if(conf.nproc > 2000)
  10.206+		conf.nproc = 2000;
  10.207+	conf.nswap = conf.npage*3;
  10.208+	conf.nswppo = 4096;
  10.209+	conf.nimage = 200;
  10.210+
  10.211+	conf.copymode = conf.nmach > 1;
  10.212+
  10.213+	/*
  10.214+	 * Guess how much is taken by the large permanent
  10.215+	 * datastructures. Mntcache and Mntrpc are not accounted for.
  10.216+	 */
  10.217+	kpages = conf.npage - conf.upages;
  10.218+	kpages *= BY2PG;
  10.219+	kpages -= conf.upages*sizeof(Page)
  10.220+		+ conf.nproc*sizeof(Proc)
  10.221+		+ conf.nimage*sizeof(Image)
  10.222+		+ conf.nswap
  10.223+		+ conf.nswppo*sizeof(Page*);
  10.224+	mainmem->maxsize = kpages;
  10.225+	if(!cpuserver)
  10.226+		/*
  10.227+		 * give terminals lots of image memory, too; the dynamic
  10.228+		 * allocation will balance the load properly, hopefully.
  10.229+		 * be careful with 32-bit overflow.
  10.230+		 */
  10.231+		imagmem->maxsize = kpages;
  10.232+}
  10.233+
  10.234+void
  10.235+machinit(void)
  10.236+{
  10.237+	m->ticks = 1;
  10.238+	m->perf.period = 1;
  10.239+	active.machs[m->machno] = 1;
  10.240+}
  10.241+
  10.242+void
  10.243+mpinit(void)
  10.244+{
  10.245+	extern void _start(void);
  10.246+	int i;
  10.247+
  10.248+	for(i = 0; i < MAXMACH; i++)
  10.249+		((uintptr*)SPINTABLE)[i] = 0;
  10.250+
  10.251+	for(i = 1; i < conf.nmach; i++)
  10.252+		MACHP(i)->machno = i;
  10.253+
  10.254+	cachedwbinv();
  10.255+
  10.256+	for(i = 1; i < conf.nmach; i++)
  10.257+		((uintptr*)SPINTABLE)[i] = PADDR(_start);
  10.258+
  10.259+	cachedwbinvse((void*)SPINTABLE, MAXMACH*8);
  10.260+	sev();
  10.261+	delay(100);
  10.262+	sev();
  10.263+	synccycles();
  10.264+}
  10.265+
  10.266+void
  10.267+idlehands(void)
  10.268+{
  10.269+}
  10.270+
  10.271+void
  10.272+main(void)
  10.273+{
  10.274+	machinit();
  10.275+	if(m->machno){
  10.276+		trapinit();
  10.277+		fpuinit();
  10.278+		clockinit();
  10.279+		cpuidprint();
  10.280+		synccycles();
  10.281+		timersinit();
  10.282+		flushtlb();
  10.283+		mmu1init();
  10.284+		delay(4000);	/* usb initilization is busted multicore, let cpu0 do it */
  10.285+		m->ticks = MACHP(0)->ticks;
  10.286+		schedinit();
  10.287+		return;
  10.288+	}
  10.289+	bootargsinit();
  10.290+	confinit();
  10.291+	xinit();
  10.292+	printinit();
  10.293+	fmtinstall('H', encodefmt);
  10.294+	quotefmtinstall();
  10.295+	uartconsinit();
  10.296+	screeninit();
  10.297+	print("\nPlan 9\n");
  10.298+	xsummary();
  10.299+
  10.300+	/* set clock rate to arm_freq from config.txt */
  10.301+	setclkrate(ClkArm, 0);
  10.302+
  10.303+	trapinit();
  10.304+	fpuinit();
  10.305+	clockinit();
  10.306+	cpuidprint();
  10.307+	timersinit();
  10.308+	pageinit();
  10.309+	procinit0();
  10.310+	initseg();
  10.311+	links();
  10.312+	chandevreset();
  10.313+	userinit();
  10.314+	mpinit();
  10.315+	mmu0clear((uintptr*)L1);
  10.316+	flushtlb();
  10.317+	mmu1init();
  10.318+	schedinit();
  10.319+}
  10.320+
  10.321+void
  10.322+exit(int)
  10.323+{
  10.324+	cpushutdown();
  10.325+	for(;;);
  10.326+}
  10.327+
  10.328+void
  10.329+reboot(void*, void*, ulong)
  10.330+{
  10.331+	error(Egreg);
  10.332+}
  10.333+
  10.334+/*
  10.335+ * stub for ../omap/devether.c
  10.336+ */
  10.337+int
  10.338+isaconfig(char *, int, ISAConf *)
  10.339+{
  10.340+	return 0;
  10.341+}
    11.1new file mode 100644
    11.2--- /dev/null
    11.3+++ b/sys/src/9/bcm64/mem.h
    11.4@@ -0,0 +1,139 @@
    11.5+/*
    11.6+ * Memory and machine-specific definitions.  Used in C and assembler.
    11.7+ */
    11.8+#define KiB		1024u			/* Kibi 0x0000000000000400 */
    11.9+#define MiB		1048576u		/* Mebi 0x0000000000100000 */
   11.10+#define GiB		1073741824u		/* Gibi 000000000040000000 */
   11.11+
   11.12+#define	HOWMANY(x,y)	(((x)+((y)-1))/(y))
   11.13+#define	ROUNDUP(x,y)	(HOWMANY((x),(y))*(y))
   11.14+#define	PGROUND(s)	ROUNDUP(s, BY2PG)
   11.15+#define	ROUND(s, sz)	(((s)+(sz-1))&~(sz-1))
   11.16+
   11.17+/*
   11.18+ * Sizes:
   11.19+ * 	L0	L1	L2	L3
   11.20+ *	4K	2M	1G	512G
   11.21+ *	16K	32M	64G	128T
   11.22+ *	64K	512M	4T	-
   11.23+ */
   11.24+#define	PGSHIFT		12		/* log(BY2PG) */
   11.25+#define	BY2PG		(1ULL<<PGSHIFT)	/* bytes per page */
   11.26+
   11.27+/* effective virtual address space */
   11.28+#define EVASHIFT	36
   11.29+#define EVAMASK		((1ULL<<EVASHIFT)-1)
   11.30+
   11.31+#define PTSHIFT		(PGSHIFT-3)
   11.32+#define PTLEVELS	HOWMANY(EVASHIFT-PGSHIFT, PTSHIFT)
   11.33+#define PTLX(v, l)	((((v) & EVAMASK) >> (PGSHIFT + (l)*PTSHIFT)) & ((1 << PTSHIFT)-1))
   11.34+#define PGLSZ(l)	(1ULL << (PGSHIFT + (l)*PTSHIFT))
   11.35+
   11.36+#define PTL1X(v, l)	(L1TABLEX(v, l) | PTLX(v, l))
   11.37+#define L1TABLEX(v, l)	(L1TABLE(v, l) << PTSHIFT)
   11.38+#define L1TABLES	HOWMANY(-KZERO, PGLSZ(2))
   11.39+#define L1TABLE(v, l)	(L1TABLES-1 - ((PTLX(v, 2) % L1TABLES) >> (((l)-1)*PTSHIFT)) + (l)-1)
   11.40+#define L1TOPSIZE	(1ULL << (EVASHIFT - PTLEVELS*PTSHIFT))
   11.41+
   11.42+#define	MAXMACH		4			/* max # cpus system can run */
   11.43+#define	MACHSIZE	(8*KiB)
   11.44+
   11.45+#define KSTACK		(8*KiB)
   11.46+#define STACKALIGN(sp)	((sp) & ~7)		/* bug: assure with alloc */
   11.47+#define TRAPFRAMESIZE	(38*8)
   11.48+
   11.49+/*
   11.50+ * Address spaces.
   11.51+ * KTZERO is used by kprof and dumpstack (if any).
   11.52+ *
   11.53+ * KZERO is mapped to physical 0 (start of ram).
   11.54+ */
   11.55+
   11.56+#define	KZERO		0xFFFFFFFF80000000ULL	/* kernel address space */
   11.57+
   11.58+#define SPINTABLE	(KZERO+0xd8)
   11.59+#define CONFADDR	(KZERO+0x100)
   11.60+#define	REBOOTADDR	(0x1c00)		/* reboot code - physical address */
   11.61+#define	VCBUFFER	(KZERO+0x3400)		/* videocore mailbox buffer */
   11.62+
   11.63+#define L1		(L1TOP-L1SIZE)
   11.64+#define L1SIZE		((L1TABLES+PTLEVELS-3)*BY2PG)
   11.65+#define L1TOP		((MACHADDR(MAXMACH-1)-L1TOPSIZE)&-BY2PG)
   11.66+
   11.67+#define MACHADDR(n)	(KTZERO-((n)+1)*MACHSIZE)
   11.68+
   11.69+#define	KTZERO		(KZERO+0x80000)		/* kernel text start */
   11.70+#define FRAMEBUFFER	0xFFFFFFFFC0000000ULL
   11.71+#define VIRTIO		0xFFFFFFFFE0000000ULL	/* i/o registers */
   11.72+#define	ARMLOCAL	(VIRTIO+IOSIZE)
   11.73+#define	VGPIO		0			/* virtual gpio for pi3 ACT LED */
   11.74+
   11.75+#define	UZERO		0ULL			/* user segment */
   11.76+#define	UTZERO		(UZERO+0x10000)		/* user text start */
   11.77+#define	USTKTOP		((EVAMASK>>1)-0xFFFF)	/* user segment end +1 */
   11.78+#define	USTKSIZE	(16*1024*1024)		/* user stack size */
   11.79+
   11.80+#define BLOCKALIGN	64			/* only used in allocb.c */
   11.81+
   11.82+/*
   11.83+ * Sizes
   11.84+ */
   11.85+#define BI2BY		8			/* bits per byte */
   11.86+#define BY2SE		4
   11.87+#define BY2WD		8
   11.88+#define BY2V		8			/* only used in xalloc.c */
   11.89+
   11.90+#define	PTEMAPMEM	(1024*1024)
   11.91+#define	PTEPERTAB	(PTEMAPMEM/BY2PG)
   11.92+#define	SEGMAPSIZE	1984
   11.93+#define	SSEGMAPSIZE	16
   11.94+#define	PPN(x)		((x)&~(BY2PG-1))
   11.95+
   11.96+#define SHARE_NONE	0
   11.97+#define SHARE_OUTER	2
   11.98+#define SHARE_INNER	3
   11.99+
  11.100+#define CACHE_UC	0
  11.101+#define CACHE_WB	1
  11.102+#define CACHE_WT	2
  11.103+#define CACHE_WB_NA	3
  11.104+
  11.105+#define MA_MEM_WB	0
  11.106+#define MA_MEM_WT	1
  11.107+#define MA_MEM_UC	2
  11.108+#define MA_DEV_nGnRnE	3
  11.109+#define MA_DEV_nGnRE	4
  11.110+#define MA_DEV_nGRE	5
  11.111+#define MA_DEV_GRE	6
  11.112+
  11.113+#define	PTEVALID	1
  11.114+#define PTEBLOCK	0
  11.115+#define PTETABLE	2
  11.116+#define PTEPAGE		2
  11.117+
  11.118+#define PTEMA(x)	((x)<<2)
  11.119+#define PTEAP(x)	((x)<<6)
  11.120+#define PTESH(x)	((x)<<8)
  11.121+
  11.122+#define PTEAF		(1<<10)
  11.123+#define PTENG		(1<<11)
  11.124+
  11.125+#define PTEKERNEL	PTEAP(0)
  11.126+#define PTEUSER		PTEAP(1)
  11.127+#define PTEWRITE	PTEAP(0)
  11.128+#define PTERONLY	PTEAP(2)
  11.129+
  11.130+#define PTEWT		PTEMA(MA_MEM_WT)
  11.131+#define	PTEUNCACHED	PTEMA(MA_MEM_UC)
  11.132+#define	PTEDEVICE	PTEMA(MA_DEV_nGnRnE)
  11.133+
  11.134+/*
  11.135+ * Physical machine information from here on.
  11.136+ *	PHYS addresses as seen from the arm cpu.
  11.137+ *	BUS  addresses as seen from the videocore gpu.
  11.138+ */
  11.139+#define	PHYSDRAM	0
  11.140+#define	IOSIZE		(16*MiB)
  11.141+
  11.142+#define MIN(a, b)	((a) < (b)? (a): (b))
  11.143+#define MAX(a, b)	((a) > (b)? (a): (b))
    12.1new file mode 100644
    12.2--- /dev/null
    12.3+++ b/sys/src/9/bcm64/mkfile
    12.4@@ -0,0 +1,146 @@
    12.5+CONF=pi3
    12.6+CONFLIST=pi3
    12.7+EXTRACOPIES=
    12.8+
    12.9+loadaddr=0xffffffff80080000
   12.10+
   12.11+objtype=arm64
   12.12+</$objtype/mkfile
   12.13+p=9
   12.14+
   12.15+OS=7
   12.16+
   12.17+DEVS=`{rc ../port/mkdevlist $CONF}
   12.18+
   12.19+PORT=\
   12.20+	alarm.$O\
   12.21+	alloc.$O\
   12.22+	allocb.$O\
   12.23+	auth.$O\
   12.24+	cache.$O\
   12.25+	chan.$O\
   12.26+	dev.$O\
   12.27+	edf.$O\
   12.28+	fault.$O\
   12.29+	mul64fract.$O\
   12.30+	page.$O\
   12.31+	parse.$O\
   12.32+	pgrp.$O\
   12.33+	portclock.$O\
   12.34+	print.$O\
   12.35+	proc.$O\
   12.36+	qio.$O\
   12.37+	qlock.$O\
   12.38+	rdb.$O\
   12.39+	rebootcmd.$O\
   12.40+	segment.$O\
   12.41+	syscallfmt.$O\
   12.42+	sysfile.$O\
   12.43+	sysproc.$O\
   12.44+	taslock.$O\
   12.45+	tod.$O\
   12.46+	xalloc.$O\
   12.47+
   12.48+OBJ=\
   12.49+	l.$O\
   12.50+	cache.v8.$O\
   12.51+	bootargs.$O\
   12.52+	clock.$O\
   12.53+	fpu.$O\
   12.54+	irq.$O\
   12.55+	main.$O\
   12.56+	mmu.$O\
   12.57+	sysreg.$O\
   12.58+	random.$O\
   12.59+	trap.$O\
   12.60+	$CONF.root.$O\
   12.61+	$CONF.rootc.$O\
   12.62+	$DEVS\
   12.63+	$PORT\
   12.64+
   12.65+# HFILES=
   12.66+
   12.67+LIB=\
   12.68+	/$objtype/lib/libmemlayer.a\
   12.69+	/$objtype/lib/libmemdraw.a\
   12.70+	/$objtype/lib/libdraw.a\
   12.71+	/$objtype/lib/libip.a\
   12.72+	/$objtype/lib/libsec.a\
   12.73+	/$objtype/lib/libmp.a\
   12.74+	/$objtype/lib/libc.a\
   12.75+
   12.76+9:V: $p$CONF s$p$CONF
   12.77+
   12.78+$p$CONF:DQ:	$CONF.c $OBJ $LIB mkfile
   12.79+	$CC $CFLAGS '-DKERNDATE='`{date -n} $CONF.c
   12.80+	echo '# linking raw kernel'	# H6: no headers, data segment aligned
   12.81+	$LD -l -o $target -H6 -R4096 -T$loadaddr $OBJ $CONF.$O $LIB
   12.82+
   12.83+s$p$CONF:DQ:	$CONF.$O $OBJ $LIB
   12.84+	echo '# linking kernel with symbols'
   12.85+	$LD -l -o $target -R4096 -T$loadaddr $OBJ $CONF.$O $LIB
   12.86+	size $target
   12.87+
   12.88+$p$CONF.gz:D:	$p$CONF
   12.89+	gzip -9 <$p$CONF >$target
   12.90+
   12.91+$OBJ: $HFILES
   12.92+
   12.93+install:V: /$objtype/$p$CONF
   12.94+
   12.95+/$objtype/$p$CONF:D: $p$CONF s$p$CONF
   12.96+	cp -x $p$CONF s$p$CONF /$objtype/ &
   12.97+	for(i in $EXTRACOPIES)
   12.98+		{ 9fs $i && cp $p$CONF s$p$CONF /n/$i/$objtype && echo -n $i... & }
   12.99+	wait
  12.100+	echo
  12.101+	touch $target
  12.102+
  12.103+
  12.104+REPCC=`{../port/mkfilelist ../bcm}
  12.105+^($REPCC)\.$O:R:	'../bcm/\1.c'
  12.106+	$CC $CFLAGS -I. -. ../bcm/$stem1.c
  12.107+
  12.108+<../boot/bootmkfile
  12.109+<../port/portmkfile
  12.110+<|../port/mkbootrules $CONF
  12.111+
  12.112+arch.$O clock.$O fpiarm.$O main.$O mmu.$O screen.$O syscall.$O trap.$O: \
  12.113+	/$objtype/include/ureg.h
  12.114+
  12.115+l.$O cache.v8.$O lexception.$O lproc.$O mmu.$O: mem.h
  12.116+l.$O cache.v8.$O archbcm3.$O clock.$O fpu.$O trap.$O mmu.$O: sysreg.h
  12.117+
  12.118+devmouse.$O mouse.$O screen.$O: screen.h
  12.119+usbdwc.$O: dwcotg.h ../port/usb.h
  12.120+
  12.121+io.h:D:	../bcm/io.h
  12.122+	echo '#include "../bcm/io.h"' > io.h
  12.123+screen.h:D: ../bcm/screen.h
  12.124+	echo '#include "../bcm/screen.h"' > screen.h
  12.125+dwcotg.h:D: ../bcm/dwcotg.h
  12.126+	echo '#include "../bcm/dwcotg.h"' > dwcotg.h
  12.127+
  12.128+init.h:D:	../port/initcode.c init9.s
  12.129+	$CC ../port/initcode.c
  12.130+	$AS init9.s
  12.131+	$LD -l -R1 -s -o init.out init9.$O initcode.$O /$objtype/lib/libc.a
  12.132+	{echo 'uchar initcode[]={'
  12.133+	 xd -1x <init.out |
  12.134+		sed -e 's/^[0-9a-f]+ //' -e 's/ ([0-9a-f][0-9a-f])/0x\1,/g'
  12.135+	 echo '};'} > init.h
  12.136+
  12.137+#reboot.h:D:	rebootcode.s arm.s arm.h mem.h
  12.138+#	$AS rebootcode.s
  12.139+#	# -T arg is REBOOTADDR
  12.140+#	$LD -l -s -T0x1c00 -R4 -o reboot.out rebootcode.$O
  12.141+#	{echo 'uchar rebootcode[]={'
  12.142+#	 xd -1x reboot.out |
  12.143+#		sed -e '1,2d' -e 's/^[0-9a-f]+ //' -e 's/ ([0-9a-f][0-9a-f])/0x\1,/g'
  12.144+#	 echo '};'} > reboot.h
  12.145+
  12.146+errstr.h:D:	../port/mkerrstr ../port/error.h
  12.147+	rc ../port/mkerrstr > errstr.h
  12.148+
  12.149+$CONF.clean:
  12.150+	rm -rf $p$CONF s$p$CONF errstr.h reboot.h io.h screen.h dwcotg.h $CONF.c boot$CONF.c
    13.1new file mode 100644
    13.2--- /dev/null
    13.3+++ b/sys/src/9/bcm64/mmu.c
    13.4@@ -0,0 +1,384 @@
    13.5+#include "u.h"
    13.6+#include "../port/lib.h"
    13.7+#include "mem.h"
    13.8+#include "dat.h"
    13.9+#include "fns.h"
   13.10+#include "sysreg.h"
   13.11+
   13.12+void
   13.13+mmu0init(uintptr *l1)
   13.14+{
   13.15+	uintptr va, pa, pe;
   13.16+
   13.17+	/* 0 identity map */
   13.18+	pe = PHYSDRAM + soc.dramsize;
   13.19+	for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(1))
   13.20+		l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
   13.21+			 | PTEKERNEL | PTESH(SHARE_INNER);
   13.22+	if(PTLEVELS > 2)
   13.23+	for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(2))
   13.24+		l1[PTL1X(pa, 2)] = (uintptr)&l1[L1TABLEX(pa, 1)] | PTEVALID | PTETABLE;
   13.25+	if(PTLEVELS > 3)
   13.26+	for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(3))
   13.27+		l1[PTL1X(pa, 3)] = (uintptr)&l1[L1TABLEX(pa, 2)] | PTEVALID | PTETABLE;
   13.28+
   13.29+	/* KZERO */
   13.30+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
   13.31+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
   13.32+			| PTEKERNEL | PTESH(SHARE_INNER);
   13.33+	if(PTLEVELS > 2)
   13.34+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
   13.35+		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
   13.36+	if(PTLEVELS > 3)
   13.37+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
   13.38+		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
   13.39+
   13.40+	/* VIRTIO */
   13.41+	pe = -VIRTIO + soc.physio;
   13.42+	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
   13.43+		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
   13.44+			| PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
   13.45+	if(PTLEVELS > 2)
   13.46+	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
   13.47+		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
   13.48+	if(PTLEVELS > 3)
   13.49+	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
   13.50+		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
   13.51+}
   13.52+
   13.53+void
   13.54+mmu0clear(uintptr *l1)
   13.55+{
   13.56+	uintptr va, pa, pe;
   13.57+
   13.58+	pe = PHYSDRAM + soc.dramsize;
   13.59+
   13.60+	if(PTLEVELS > 3)
   13.61+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
   13.62+		if(PTL1X(pa, 3) != PTL1X(va, 3))
   13.63+			l1[PTL1X(pa, 3)] = 0;
   13.64+	}
   13.65+	if(PTLEVELS > 2)
   13.66+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
   13.67+		if(PTL1X(pa, 2) != PTL1X(va, 2))
   13.68+			l1[PTL1X(pa, 2)] = 0;
   13.69+	}
   13.70+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
   13.71+		if(PTL1X(pa, 1) != PTL1X(va, 1))
   13.72+			l1[PTL1X(pa, 1)] = 0;
   13.73+	}
   13.74+}
   13.75+
   13.76+void
   13.77+mmu1init(void)
   13.78+{
   13.79+	m->mmul1 = mallocalign(L1SIZE+L1TOPSIZE, BY2PG, L1SIZE, 0);
   13.80+	if(m->mmul1 == nil)
   13.81+		panic("mmu1init: no memory for mmul1");
   13.82+	memset(m->mmul1, 0, L1SIZE+L1TOPSIZE);
   13.83+	mmuswitch(nil);
   13.84+}
   13.85+
   13.86+uintptr
   13.87+paddr(void *va)
   13.88+{
   13.89+	if((uintptr)va >= KZERO)
   13.90+		return (uintptr)va-KZERO;
   13.91+	panic("paddr: va=%#p pc=%#p", va, getcallerpc(&va));
   13.92+	return 0;
   13.93+}
   13.94+
   13.95+uintptr
   13.96+cankaddr(uintptr pa)
   13.97+{
   13.98+	if(pa < (uintptr)-KZERO)
   13.99+		return -KZERO - pa;
  13.100+	return 0;
  13.101+}
  13.102+
  13.103+void*
  13.104+kaddr(uintptr pa)
  13.105+{
  13.106+	if(pa < (uintptr)-KZERO)
  13.107+		return (void*)(pa + KZERO);
  13.108+	panic("kaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
  13.109+	return nil;
  13.110+}
  13.111+
  13.112+void
  13.113+kmapinval(void)
  13.114+{
  13.115+}
  13.116+
  13.117+KMap*
  13.118+kmap(Page *p)
  13.119+{
  13.120+	return kaddr(p->pa);
  13.121+}
  13.122+
  13.123+void
  13.124+kunmap(KMap*)
  13.125+{
  13.126+}
  13.127+
  13.128+uintptr
  13.129+mmukmap(uintptr va, uintptr pa, usize size)
  13.130+{
  13.131+	uintptr a, pe, off;
  13.132+
  13.133+	if(va == 0)
  13.134+		return 0;
  13.135+
  13.136+	assert((va % PGLSZ(1)) == 0);
  13.137+	off = pa % PGLSZ(1);
  13.138+	a = va + off;
  13.139+	pe = (pa + size + (PGLSZ(1)-1)) & -PGLSZ(1);
  13.140+	while(pa < pe){
  13.141+		((uintptr*)L1)[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
  13.142+			| PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
  13.143+		pa += PGLSZ(1);
  13.144+		va += PGLSZ(1);
  13.145+	}
  13.146+	flushtlb();
  13.147+	return a;
  13.148+}
  13.149+
  13.150+static uintptr*
  13.151+mmuwalk(uintptr va, int level)
  13.152+{
  13.153+	uintptr *table, pte;
  13.154+	Page *pg;
  13.155+	int i, x;
  13.156+
  13.157+	x = PTLX(va, PTLEVELS-1);
  13.158+	table = &m->mmul1[L1TABLEX(va, PTLEVELS-1)];
  13.159+	for(i = PTLEVELS-2; i >= level; i--){
  13.160+		pte = table[x];
  13.161+		if(pte & PTEVALID) {
  13.162+			if(pte & (0xFFFFULL<<48))
  13.163+				iprint("strange pte %#p va %#p\n", pte, va);
  13.164+			pte &= ~(0xFFFFULL<<48 | BY2PG-1);
  13.165+			table = KADDR(pte);
  13.166+		} else {
  13.167+			if(i < 2){
  13.168+				pg = up->mmufree;
  13.169+				if(pg == nil)
  13.170+					return nil;
  13.171+				up->mmufree = pg->next;
  13.172+				switch(i){
  13.173+				case 0:
  13.174+					pg->va = va & -PGLSZ(1);
  13.175+					if((pg->next = up->mmul1) == nil)
  13.176+						up->mmul1tail = pg;
  13.177+					up->mmul1 = pg;
  13.178+					break;
  13.179+				case 1:
  13.180+					pg->va = va & -PGLSZ(2);
  13.181+					if((pg->next = up->mmul2) == nil)
  13.182+						up->mmul2tail = pg;
  13.183+					up->mmul2 = pg;
  13.184+					break;
  13.185+				}
  13.186+				memset(KADDR(pg->pa), 0, BY2PG);
  13.187+				coherence();
  13.188+				table[x] = pg->pa | PTEVALID | PTETABLE;
  13.189+				table = KADDR(pg->pa);
  13.190+			} else {
  13.191+				table[x] = PADDR(&m->mmul1[L1TABLEX(va, 2)]) | PTEVALID | PTETABLE;
  13.192+				table = &m->mmul1[L1TABLEX(va, 2)];
  13.193+			}
  13.194+		}
  13.195+		x = PTLX(va, (uintptr)i);
  13.196+	}
  13.197+	return &table[x];
  13.198+}
  13.199+
  13.200+static Proc *asidlist[256];
  13.201+
  13.202+static int
  13.203+allocasid(Proc *p)
  13.204+{
  13.205+	static Lock lk;
  13.206+	Proc *x;
  13.207+	int a;
  13.208+
  13.209+	lock(&lk);
  13.210+	a = p->asid;
  13.211+	if(a < 0)
  13.212+		a = -a;
  13.213+	if(a == 0)
  13.214+		a = p->pid;
  13.215+	for(;; a++){
  13.216+		a %= nelem(asidlist);
  13.217+		if(a == 0)
  13.218+			continue;	// reserved
  13.219+		x = asidlist[a];
  13.220+		if(x == p || x == nil || (x->asid < 0 && x->mach == nil))
  13.221+			break;
  13.222+	}
  13.223+	p->asid = a;
  13.224+	asidlist[a] = p;
  13.225+	unlock(&lk);
  13.226+
  13.227+	return x != p;
  13.228+}
  13.229+
  13.230+static void
  13.231+freeasid(Proc *p)
  13.232+{
  13.233+	int a;
  13.234+
  13.235+	a = p->asid;
  13.236+	if(a < 0)
  13.237+		a = -a;
  13.238+	if(a > 0 && asidlist[a] == p)
  13.239+		asidlist[a] = nil;
  13.240+	p->asid = 0;
  13.241+}
  13.242+
  13.243+void
  13.244+putasid(Proc *p)
  13.245+{
  13.246+	/*
  13.247+	 * Prevent the following scenario:
  13.248+	 *	pX sleeps on cpuA, leaving its page tables in mmul1
  13.249+	 *	pX wakes up on cpuB, and exits, freeing its page tables
  13.250+	 *  pY on cpuB allocates a freed page table page and overwrites with data
  13.251+	 *  cpuA takes an interrupt, and is now running with bad page tables
  13.252+	 * In theory this shouldn't hurt because only user address space tables
  13.253+	 * are affected, and mmuswitch will clear mmul1 before a user process is
  13.254+	 * dispatched.  But empirically it correlates with weird problems, eg
  13.255+	 * resetting of the core clock at 0x4000001C which confuses local timers.
  13.256+	 */
  13.257+	if(conf.nmach > 1)
  13.258+		mmuswitch(nil);
  13.259+
  13.260+	if(p->asid > 0)
  13.261+		p->asid = -p->asid;
  13.262+}
  13.263+
  13.264+void
  13.265+putmmu(uintptr va, uintptr pa, Page *pg)
  13.266+{
  13.267+	uintptr *pte, old;
  13.268+	int s;
  13.269+
  13.270+// iprint("cpu%d: putmmu va %#p asid %d proc %lud %s\n", m->machno, va, up->asid, up->pid, up->text);
  13.271+	s = splhi();
  13.272+	while((pte = mmuwalk(va, 0)) == nil){
  13.273+		spllo();
  13.274+		assert(up->mmufree == nil);
  13.275+		up->mmufree = newpage(0, nil, 0);
  13.276+		splhi();
  13.277+	}
  13.278+	old = *pte;
  13.279+	*pte = 0;
  13.280+	if((old & PTEVALID) != 0)
  13.281+		flushasidvall((uvlong)up->asid<<48 | va>>12);
  13.282+	else
  13.283+		flushasidva((uvlong)up->asid<<48 | va>>12);
  13.284+	*pte = pa | PTEPAGE | PTEUSER | PTENG | PTEAF | PTESH(SHARE_INNER);
  13.285+	if(pg->txtflush & (1UL<<m->machno)){
  13.286+		/* pio() sets PG_TXTFLUSH whenever a text pg has been written */
  13.287+		cachedwbinvse((void*)KADDR(pg->pa), BY2PG);
  13.288+		cacheiinvse((void*)va, BY2PG);
  13.289+		pg->txtflush &= ~(1UL<<m->machno);
  13.290+	}
  13.291+	splx(s);
  13.292+}
  13.293+
  13.294+static void
  13.295+mmufree(Proc *p)
  13.296+{
  13.297+	freeasid(p);
  13.298+
  13.299+	if(p->mmul1 == nil){
  13.300+		assert(p->mmul2 == nil);
  13.301+		return;
  13.302+	}
  13.303+	p->mmul1tail->next = p->mmufree;
  13.304+	p->mmufree = p->mmul1;
  13.305+	p->mmul1 = p->mmul1tail = nil;
  13.306+
  13.307+	if(PTLEVELS > 2){
  13.308+		p->mmul2tail->next = p->mmufree;
  13.309+		p->mmufree = p->mmul2;
  13.310+		p->mmul2 = p->mmul2tail = nil;
  13.311+	}
  13.312+}
  13.313+
  13.314+void
  13.315+mmuswitch(Proc *p)
  13.316+{
  13.317+	uintptr va;
  13.318+	Page *t;
  13.319+
  13.320+	for(va = UZERO; va < USTKTOP; va += PGLSZ(PTLEVELS-1))
  13.321+		m->mmul1[PTL1X(va, PTLEVELS-1)] = 0;
  13.322+
  13.323+	if(p == nil){
  13.324+		setttbr(PADDR(&m->mmul1[L1TABLEX(0, PTLEVELS-1)]));
  13.325+		return;
  13.326+	}
  13.327+
  13.328+	if(p->newtlb){
  13.329+		mmufree(p);
  13.330+		p->newtlb = 0;
  13.331+	}
  13.332+
  13.333+	if(PTLEVELS == 2){
  13.334+		for(t = p->mmul1; t != nil; t = t->next){
  13.335+			va = t->va;
  13.336+			m->mmul1[PTL1X(va, 1)] = t->pa | PTEVALID | PTETABLE;
  13.337+		}
  13.338+	} else {
  13.339+		for(t = p->mmul2; t != nil; t = t->next){
  13.340+			va = t->va;
  13.341+			m->mmul1[PTL1X(va, 2)] = t->pa | PTEVALID | PTETABLE;
  13.342+			if(PTLEVELS > 3)
  13.343+				m->mmul1[PTL1X(va, 3)] = PADDR(&m->mmul1[L1TABLEX(va, 2)]) |
  13.344+					PTEVALID | PTETABLE;
  13.345+		}
  13.346+	}
  13.347+
  13.348+	if(allocasid(p))
  13.349+		flushasid((uvlong)p->asid<<48);
  13.350+
  13.351+// iprint("cpu%d: mmuswitch asid %d proc %lud %s\n", m->machno, p->asid, p->pid, p->text);
  13.352+	setttbr((uvlong)p->asid<<48 | PADDR(&m->mmul1[L1TABLEX(0, PTLEVELS-1)]));
  13.353+}
  13.354+
  13.355+void
  13.356+mmurelease(Proc *p)
  13.357+{
  13.358+	Page *t;
  13.359+
  13.360+	mmuswitch(nil);
  13.361+	mmufree(p);
  13.362+
  13.363+	if((t = p->mmufree) != nil){
  13.364+		do {
  13.365+			p->mmufree = t->next;
  13.366+			if(--t->ref != 0)
  13.367+				panic("mmurelease: bad page ref");
  13.368+			pagechainhead(t);
  13.369+		} while((t = p->mmufree) != nil);
  13.370+		pagechaindone();
  13.371+	}
  13.372+}
  13.373+
  13.374+void
  13.375+flushmmu(void)
  13.376+{
  13.377+	int x;
  13.378+
  13.379+	x = splhi();
  13.380+	up->newtlb = 1;
  13.381+	mmuswitch(up);
  13.382+	splx(x);
  13.383+}
  13.384+
  13.385+void
  13.386+checkmmu(uintptr, uintptr)
  13.387+{
  13.388+}
    14.1new file mode 100644
    14.2--- /dev/null
    14.3+++ b/sys/src/9/bcm64/pi3
    14.4@@ -0,0 +1,53 @@
    14.5+dev
    14.6+	root
    14.7+	cons
    14.8+	swap
    14.9+	env
   14.10+	pipe
   14.11+	proc
   14.12+	mnt
   14.13+	srv
   14.14+	shr
   14.15+	dup
   14.16+	arch
   14.17+	ssl
   14.18+	tls
   14.19+	cap
   14.20+	fs
   14.21+	ip		arp chandial ip ipv6 ipaux iproute netlog nullmedium pktmedium inferno
   14.22+	draw	screen swcursor
   14.23+	mouse	mouse
   14.24+	uart	gpio
   14.25+#	gpio	gpio
   14.26+	sd
   14.27+	usb
   14.28+
   14.29+link
   14.30+	loopbackmedium
   14.31+	ethermedium
   14.32+	archbcm3
   14.33+	usbdwc
   14.34+
   14.35+ip
   14.36+	tcp
   14.37+	udp
   14.38+	ipifc
   14.39+	icmp
   14.40+	icmp6
   14.41+	ipmux
   14.42+
   14.43+misc
   14.44+	uartmini
   14.45+	uartpl011
   14.46+	sdmmc	emmc
   14.47+	dma
   14.48+	vcore
   14.49+
   14.50+port
   14.51+	int cpuserver = 0;
   14.52+
   14.53+bootdir
   14.54+	/$objtype/bin/paqfs
   14.55+	/$objtype/bin/auth/factotum
   14.56+	bootfs.paq
   14.57+	boot
    15.1new file mode 100644
    15.2--- /dev/null
    15.3+++ b/sys/src/9/bcm64/sysreg.c
    15.4@@ -0,0 +1,58 @@
    15.5+/*
    15.6+ * ARMv8 system registers
    15.7+ * mainly to cope with arm hard-wiring register numbers into instructions.
    15.8+ *
    15.9+ * these routines must be callable from KZERO.
   15.10+ *
   15.11+ * on a multiprocessor, process switching to another cpu is assumed
   15.12+ * to be inhibited by the caller as these registers are local to the cpu.
   15.13+ */
   15.14+#include "u.h"
   15.15+#include "../port/lib.h"
   15.16+#include "mem.h"
   15.17+#include "dat.h"
   15.18+#include "fns.h"
   15.19+
   15.20+static void*
   15.21+mkinstr(ulong wd)
   15.22+{
   15.23+	static ulong ib[256], *ep[MAXMACH+1];
   15.24+	static Lock lk;
   15.25+	ulong *ip, *ie;
   15.26+
   15.27+	ie = ep[m->machno];
   15.28+	for(ip = ib; ip < ie; ip += 2)
   15.29+		if(*ip == wd)
   15.30+			return ip;
   15.31+
   15.32+	ilock(&lk);
   15.33+	ie = ep[MAXMACH];
   15.34+	for(; ip < ie; ip += 2)
   15.35+		if(*ip == wd)
   15.36+			goto Found;
   15.37+	if(ip >= &ib[nelem(ib)])
   15.38+		panic("mkinstr: out of instrucuction buffer");
   15.39+	ip[0] = wd;
   15.40+	ip[1] = 0xd65f03c0;	// RETURN
   15.41+	ep[MAXMACH] = ie = ip + 2;
   15.42+	cachedwbinvse(ip, 2*sizeof(*ip));
   15.43+Found:
   15.44+	iunlock(&lk);
   15.45+	cacheiinv();
   15.46+	ep[m->machno] = ie;
   15.47+	return ip;
   15.48+}
   15.49+
   15.50+uvlong
   15.51+sysrd(ulong spr)
   15.52+{
   15.53+	uvlong (*fp)(void) = mkinstr(0xd5380000UL | spr);
   15.54+	return fp();
   15.55+}
   15.56+
   15.57+void
   15.58+syswr(ulong spr, uvlong val)
   15.59+{
   15.60+	void (*fp)(uvlong) = mkinstr(0xd5180000UL | spr);
   15.61+	fp(val);
   15.62+}
    16.1new file mode 100644
    16.2--- /dev/null
    16.3+++ b/sys/src/9/bcm64/sysreg.h
    16.4@@ -0,0 +1,53 @@
    16.5+#define MIDR_EL1			SYSREG(3,0,0,0,0)
    16.6+#define MPIDR_EL1			SYSREG(3,0,0,0,5)
    16.7+#define ID_AA64MMFR0_EL1		SYSREG(3,0,0,7,0)
    16.8+#define SCTLR_EL1			SYSREG(3,0,1,0,0)
    16.9+#define CPACR_EL1			SYSREG(3,0,1,0,2)
   16.10+#define MAIR_EL1			SYSREG(3,0,10,2,0)
   16.11+#define TCR_EL1				SYSREG(3,0,2,0,2)
   16.12+#define TTBR0_EL1			SYSREG(3,0,2,0,0)
   16.13+#define TTBR1_EL1			SYSREG(3,0,2,0,1)
   16.14+#define ESR_EL1				SYSREG(3,0,5,2,0)
   16.15+#define FAR_EL1				SYSREG(3,0,6,0,0)
   16.16+#define VBAR_EL1			SYSREG(3,0,12,0,0)
   16.17+#define VTTBR_EL2			SYSREG(3,4,2,1,0)
   16.18+#define SP_EL0				SYSREG(3,0,4,1,0)
   16.19+#define SP_EL1				SYSREG(3,4,4,1,0)
   16.20+#define SP_EL2				SYSREG(3,6,4,1,0)
   16.21+#define SCTLR_EL2			SYSREG(3,4,1,0,0)
   16.22+#define HCR_EL2				SYSREG(3,4,1,1,0)
   16.23+#define MDCR_EL2			SYSREG(3,4,1,1,1)
   16.24+#define PMCR_EL0			SYSREG(3,3,9,12,0)
   16.25+#define PMCNTENSET			SYSREG(3,3,9,12,1)
   16.26+#define PMCCNTR_EL0			SYSREG(3,3,9,13,0)
   16.27+#define PMUSERENR_EL0			SYSREG(3,3,9,14,0)
   16.28+
   16.29+#define CNTP_TVAL_EL0			SYSREG(3,3,14,2,0)
   16.30+#define CNTP_CTL_EL0			SYSREG(3,3,14,2,1)
   16.31+#define CNTP_CVAL_EL0			SYSREG(3,3,14,2,2)
   16.32+
   16.33+#define TPIDR_EL0			SYSREG(3,3,13,0,2)
   16.34+#define TPIDR_EL1			SYSREG(3,0,13,0,4)
   16.35+
   16.36+#define CCSIDR_EL1			SYSREG(3,1,0,0,0)
   16.37+#define CSSELR_EL1			SYSREG(3,2,0,0,0)
   16.38+
   16.39+#define ACTLR_EL2			SYSREG(3,4,1,0,1)
   16.40+#define CPUACTLR_EL1			SYSREG(3,1,15,2,0)
   16.41+#define CPUECTLR_EL1			SYSREG(3,1,15,2,1)
   16.42+
   16.43+/* l.s redefines this for the assembler */
   16.44+#define SYSREG(op0,op1,Cn,Cm,op2)	((op0)<<19|(op1)<<16|(Cn)<<12|(Cm)<<8|(op2)<<5)
   16.45+
   16.46+#define	OSHLD	(0<<2 | 1)
   16.47+#define OSHST	(0<<2 | 2)
   16.48+#define	OSH	(0<<2 | 3)
   16.49+#define NSHLD	(1<<2 | 1)
   16.50+#define NSHST	(1<<2 | 2)
   16.51+#define NSH	(1<<2 | 3)
   16.52+#define ISHLD	(2<<2 | 1)
   16.53+#define ISHST	(2<<2 | 2)
   16.54+#define ISH	(2<<2 | 3)
   16.55+#define LD	(3<<2 | 1)
   16.56+#define ST	(3<<2 | 2)
   16.57+#define SY	(3<<2 | 3)
    17.1new file mode 100644
    17.2--- /dev/null
    17.3+++ b/sys/src/9/bcm64/trap.c
    17.4@@ -0,0 +1,752 @@
    17.5+#include "u.h"
    17.6+#include "../port/lib.h"
    17.7+#include "mem.h"
    17.8+#include "dat.h"
    17.9+#include "fns.h"
   17.10+#include "../port/error.h"
   17.11+#include "../port/systab.h"
   17.12+
   17.13+#include <tos.h>
   17.14+#include "ureg.h"
   17.15+#include "sysreg.h"
   17.16+
   17.17+/* SPSR bits user can modify */
   17.18+#define USPSRMASK	(0xFULL<<28)
   17.19+
   17.20+static void
   17.21+setupvector(u32int *v, void (*t)(void), void (*f)(void))
   17.22+{
   17.23+	int i;
   17.24+
   17.25+	for(i = 0; i < 0x80/4; i++){
   17.26+		v[i] = ((u32int*)t)[i];
   17.27+		if(v[i] == 0x14000000){
   17.28+			v[i] |= ((u32int*)f - &v[i]) & 0x3ffffff;
   17.29+			return;
   17.30+		}
   17.31+	}
   17.32+	panic("bug in vector code");
   17.33+}
   17.34+
   17.35+void
   17.36+trapinit(void)
   17.37+{
   17.38+	extern void vsys(void);
   17.39+	extern void vtrap(void);
   17.40+	extern void virq(void);
   17.41+	extern void vfiq(void);
   17.42+	extern void vserr(void);
   17.43+
   17.44+	extern void vsys0(void);
   17.45+	extern void vtrap0(void);
   17.46+	extern void vtrap1(void);
   17.47+
   17.48+	static u32int *v;
   17.49+
   17.50+	intrcpushutdown();
   17.51+	if(v == nil){
   17.52+		/* disable everything */
   17.53+		intrsoff();
   17.54+
   17.55+		v = mallocalign(0x80*4*4, 1<<11, 0, 0);
   17.56+		if(v == nil)
   17.57+			panic("no memory for vector table");
   17.58+
   17.59+		setupvector(&v[0x000/4], vtrap,	vtrap0);
   17.60+		setupvector(&v[0x080/4], virq,	vtrap0);
   17.61+		setupvector(&v[0x100/4], vfiq,	vtrap0);
   17.62+		setupvector(&v[0x180/4], vserr,	vtrap0);
   17.63+
   17.64+		setupvector(&v[0x200/4], vtrap,	vtrap1);
   17.65+		setupvector(&v[0x280/4], virq,	vtrap1);
   17.66+		setupvector(&v[0x300/4], vfiq,	vtrap1);
   17.67+		setupvector(&v[0x380/4], vserr,	vtrap1);
   17.68+
   17.69+		setupvector(&v[0x400/4], vsys,	vsys0);
   17.70+		setupvector(&v[0x480/4], virq,	vtrap0);
   17.71+		setupvector(&v[0x500/4], vfiq,	vtrap0);
   17.72+		setupvector(&v[0x580/4], vserr, vtrap0);
   17.73+
   17.74+		setupvector(&v[0x600/4], vtrap,	vtrap0);
   17.75+		setupvector(&v[0x680/4], virq,	vtrap0);
   17.76+		setupvector(&v[0x700/4], vfiq,	vtrap0);
   17.77+		setupvector(&v[0x780/4], vserr,	vtrap0);
   17.78+
   17.79+		cacheduwbse(v, 0x80*4*4);
   17.80+	}
   17.81+	cacheiinvse(v, 0x80*4*4);
   17.82+	syswr(VBAR_EL1, (uintptr)v);
   17.83+	splx(0x3<<6);	// unmask serr and debug
   17.84+}
   17.85+
   17.86+void
   17.87+kexit(Ureg*)
   17.88+{
   17.89+	Tos *tos;
   17.90+	uvlong t;
   17.91+
   17.92+	t = cycles();
   17.93+
   17.94+	tos = (Tos*)(USTKTOP-sizeof(Tos));
   17.95+	tos->kcycles += t - up->kentry;
   17.96+	tos->pcycles = t + up->pcycles;
   17.97+	tos->pid = up->pid;
   17.98+}
   17.99+
  17.100+static char *traps[64] = {
  17.101+	[0x00]	"sys: trap: unknown",
  17.102+	[0x01]	"sys: trap: WFI or WFE instruction execution",
  17.103+	[0x0E]	"sys: trap: illegal execution state",
  17.104+	[0x18]	"sys: trap: illegal MSR/MRS access",
  17.105+	[0x22]	"sys: trap: misaligned pc",
  17.106+	[0x26]	"sys: trap: stack pointer misaligned",
  17.107+	[0x30]	"sys: trap: breakpoint",
  17.108+	[0x32]	"sys: trap: software step",
  17.109+	[0x34]	"sys: trap: watchpoint",
  17.110+	[0x3C]	"sys: trap: BRK instruction",
  17.111+};
  17.112+
  17.113+void
  17.114+trap(Ureg *ureg)
  17.115+{
  17.116+	u32int type, intr;
  17.117+	
  17.118+	intr = ureg->type >> 32;
  17.119+	if(intr == 2){
  17.120+		fiq(ureg);
  17.121+		return;
  17.122+	}
  17.123+	splflo();
  17.124+	if(userureg(ureg)){
  17.125+		up->dbgreg = ureg;
  17.126+		up->kentry = cycles();
  17.127+	}
  17.128+	type = (u32int)ureg->type >> 26;
  17.129+	switch(type){
  17.130+	case 0x20:	// instruction abort from lower level
  17.131+	case 0x21:	// instruction abort from same level
  17.132+	case 0x24:	// data abort from lower level
  17.133+	case 0x25:	// data abort from same level
  17.134+		faultarm64(ureg);
  17.135+		break;
  17.136+	case 0x07:	// SIMD/FP
  17.137+	case 0x2C:	// FPU exception (A64 only)
  17.138+		mathtrap(ureg);
  17.139+		break;
  17.140+	case 0x00:	// unknown
  17.141+		if(intr == 1){
  17.142+			if(irq(ureg) && up != nil && up->delaysched)
  17.143+				sched();
  17.144+			break;
  17.145+		}
  17.146+		if(intr == 3){
  17.147+	case 0x2F:	// SError interrupt
  17.148+			dumpregs(ureg);
  17.149+			panic("SError interrupt");
  17.150+			break;
  17.151+		}
  17.152+		/* wet floor */
  17.153+	case 0x01:	// WFI or WFE instruction execution
  17.154+	case 0x03:	// MCR or MRC access to CP15 (A32 only)
  17.155+	case 0x04:	// MCRR or MRC access to CP15 (A32 only)
  17.156+	case 0x05:	// MCR or MRC access to CP14 (A32 only)
  17.157+	case 0x06:	// LDC or STD access to CP14 (A32 only)
  17.158+	case 0x08:	// MCR or MRC to CP10 (A32 only)
  17.159+	case 0x0C:	// MRC access to CP14 (A32 only)
  17.160+	case 0x0E:	// Illegal Execution State
  17.161+	case 0x11:	// SVC instruction execution (A32 only)
  17.162+	case 0x12:	// HVC instruction execution (A32 only)
  17.163+	case 0x13:	// SMC instruction execution (A32 only)
  17.164+	case 0x15:	// SVC instruction execution (A64 only)
  17.165+	case 0x16:	// HVC instruction execution (A64 only)
  17.166+	case 0x17:	// SMC instruction execution (A64 only)
  17.167+	case 0x18:	// MSR/MRS (A64)
  17.168+	case 0x22:	// misaligned pc
  17.169+	case 0x26:	// stack pointer misaligned
  17.170+	case 0x28:	// FPU exception (A32 only)
  17.171+	case 0x30:	// breakpoint from lower level
  17.172+	case 0x31:	// breakpoint from same level
  17.173+	case 0x32:	// software step from lower level
  17.174+	case 0x33:	// software step from same level
  17.175+	case 0x34:	// watchpoint execution from lower level
  17.176+	case 0x35:	// watchpoint exception from same level
  17.177+	case 0x38:	// breapoint (A32 only)
  17.178+	case 0x3A:	// vector catch exception (A32 only)
  17.179+	case 0x3C:	// BRK instruction (A64 only)
  17.180+	default:
  17.181+		if(!userureg(ureg)){
  17.182+			dumpregs(ureg);
  17.183+			panic("unhandled trap");
  17.184+		}
  17.185+		if(traps[type] == nil) type = 0;	// unknown
  17.186+		postnote(up, 1, traps[type], NDebug);
  17.187+		break;
  17.188+	}
  17.189+	splhi();
  17.190+	if(userureg(ureg)){
  17.191+		if(up->procctl || up->nnote)
  17.192+			notify(ureg);
  17.193+		kexit(ureg);
  17.194+	}
  17.195+}
  17.196+
  17.197+void
  17.198+syscall(Ureg *ureg)
  17.199+{
  17.200+	vlong startns, stopns;
  17.201+	uintptr sp, ret;
  17.202+	ulong scallnr;
  17.203+	int i, s;
  17.204+	char *e;
  17.205+
  17.206+	up->kentry = cycles();
  17.207+	
  17.208+	m->syscall++;
  17.209+	up->insyscall = 1;
  17.210+	up->pc = ureg->pc;
  17.211+	up->dbgreg = ureg;
  17.212+	
  17.213+	sp = ureg->sp;
  17.214+	up->scallnr = scallnr = ureg->r0;
  17.215+
  17.216+	spllo();
  17.217+	
  17.218+	up->nerrlab = 0;
  17.219+	startns = 0;
  17.220+	ret = -1;
  17.221+	if(!waserror()){
  17.222+		if(sp < USTKTOP - BY2PG || sp > USTKTOP - sizeof(Sargs) - BY2WD){
  17.223+			validaddr(sp, sizeof(Sargs)+BY2WD, 0);
  17.224+			evenaddr(sp);
  17.225+		}
  17.226+		up->s = *((Sargs*) (sp + BY2WD));
  17.227+
  17.228+		if(up->procctl == Proc_tracesyscall){
  17.229+			syscallfmt(scallnr, ureg->pc, (va_list) up->s.args);
  17.230+			s = splhi();
  17.231+			up->procctl = Proc_stopme;
  17.232+			procctl();
  17.233+			splx(s);
  17.234+			startns = todget(nil);
  17.235+		}
  17.236+		
  17.237+		if(scallnr >= nsyscall || systab[scallnr] == nil){
  17.238+			pprint("bad sys call number %lud pc %#p", scallnr, ureg->pc);
  17.239+			postnote(up, 1, "sys: bad sys call", NDebug);
  17.240+			error(Ebadarg);
  17.241+		}
  17.242+		up->psstate = sysctab[scallnr];
  17.243+		ret = systab[scallnr]((va_list)up->s.args);
  17.244+		poperror();
  17.245+	}else{
  17.246+		e = up->syserrstr;
  17.247+		up->syserrstr = up->errstr;
  17.248+		up->errstr = e;
  17.249+	}
  17.250+	if(up->nerrlab){
  17.251+		print("bad errstack [%lud]: %d extra\n", scallnr, up->nerrlab);
  17.252+		for(i = 0; i < NERR; i++)
  17.253+			print("sp=%#p pc=%#p\n", up->errlab[i].sp, up->errlab[i].pc);
  17.254+		panic("error stack");
  17.255+	}
  17.256+	ureg->r0 = ret;
  17.257+	if(up->procctl == Proc_tracesyscall){
  17.258+		stopns = todget(nil);
  17.259+		sysretfmt(scallnr, (va_list) up->s.args, ret, startns, stopns);
  17.260+		s = splhi();
  17.261+		up->procctl = Proc_stopme;
  17.262+		procctl();
  17.263+		splx(s);
  17.264+	}
  17.265+	
  17.266+	up->insyscall = 0;
  17.267+	up->psstate = 0;
  17.268+	if(scallnr == NOTED){
  17.269+		noted(ureg, *((ulong*) up->s.args));
  17.270+		/*
  17.271+		 * normally, syscall() returns to forkret()
  17.272+		 * not restoring general registers when going
  17.273+		 * to userspace. to completely restore the
  17.274+		 * interrupted context, we have to return thru
  17.275+		 * noteret(). we override return pc to jump to
  17.276+		 * to it when returning form syscall()
  17.277+		 */
  17.278+		returnto(noteret);
  17.279+	}
  17.280+
  17.281+	if(scallnr != RFORK && (up->procctl || up->nnote)){
  17.282+		splhi();
  17.283+		notify(ureg);
  17.284+	}
  17.285+	if(up->delaysched)
  17.286+		sched();
  17.287+	kexit(ureg);
  17.288+}
  17.289+
  17.290+int
  17.291+notify(Ureg *ureg)
  17.292+{
  17.293+	int l;
  17.294+	uintptr s, sp;
  17.295+	Note *n;
  17.296+
  17.297+	if(up->procctl)
  17.298+		procctl();
  17.299+	if(up->nnote == 0)
  17.300+		return 0;
  17.301+	if(up->fpstate == FPactive){
  17.302+		fpsave(up->fpsave);
  17.303+		up->fpstate = FPinactive;
  17.304+	}
  17.305+	up->fpstate |= FPillegal;
  17.306+
  17.307+	s = spllo();
  17.308+	qlock(&up->debug);
  17.309+	up->notepending = 0;
  17.310+	n = &up->note[0];
  17.311+	if(strncmp(n->msg, "sys:", 4) == 0){
  17.312+		l = strlen(n->msg);
  17.313+		if(l > ERRMAX-23)	/* " pc=0x0123456789abcdef\0" */
  17.314+			l = ERRMAX-23;
  17.315+		sprint(n->msg+l, " pc=%#p", ureg->pc);
  17.316+	}
  17.317+
  17.318+	if(n->flag!=NUser && (up->notified || up->notify==0)){
  17.319+		qunlock(&up->debug);
  17.320+		if(n->flag == NDebug)
  17.321+			pprint("suicide: %s\n", n->msg);
  17.322+		pexit(n->msg, n->flag!=NDebug);
  17.323+	}
  17.324+
  17.325+	if(up->notified){
  17.326+		qunlock(&up->debug);
  17.327+		splhi();
  17.328+		return 0;
  17.329+	}
  17.330+
  17.331+	if(!up->notify){
  17.332+		qunlock(&up->debug);
  17.333+		pexit(n->msg, n->flag!=NDebug);
  17.334+	}
  17.335+	sp = ureg->sp;
  17.336+	sp -= 256;	/* debugging: preserve context causing problem */
  17.337+	sp -= sizeof(Ureg);
  17.338+	sp = STACKALIGN(sp);
  17.339+
  17.340+	if(!okaddr((uintptr)up->notify, 1, 0)
  17.341+	|| !okaddr(sp-ERRMAX-4*BY2WD, sizeof(Ureg)+ERRMAX+4*BY2WD, 1)
  17.342+	|| ((uintptr) up->notify & 3) != 0
  17.343+	|| (sp & 7) != 0){
  17.344+		qunlock(&up->debug);
  17.345+		pprint("suicide: bad address in notify: handler=%#p sp=%#p\n",
  17.346+			up->notify, sp);
  17.347+		pexit("Suicide", 0);
  17.348+	}
  17.349+
  17.350+	memmove((Ureg*)sp, ureg, sizeof(Ureg));
  17.351+	*(Ureg**)(sp-BY2WD) = up->ureg;	/* word under Ureg is old up->ureg */
  17.352+	up->ureg = (void*)sp;
  17.353+	sp -= BY2WD+ERRMAX;
  17.354+	memmove((char*)sp, up->note[0].msg, ERRMAX);
  17.355+	sp -= 3*BY2WD;
  17.356+	*(uintptr*)(sp+2*BY2WD) = sp+3*BY2WD;
  17.357+	*(uintptr*)(sp+1*BY2WD) = (uintptr)up->ureg;
  17.358+	ureg->r0 = (uintptr) up->ureg;
  17.359+	ureg->sp = sp;
  17.360+	ureg->pc = (uintptr) up->notify;
  17.361+	ureg->link = 0;
  17.362+	up->notified = 1;
  17.363+	up->nnote--;
  17.364+	memmove(&up->lastnote, &up->note[0], sizeof(Note));
  17.365+	memmove(&up->note[0], &up->note[1], up->nnote*sizeof(Note));
  17.366+
  17.367+	qunlock(&up->debug);
  17.368+	splx(s);
  17.369+	return 1;
  17.370+}
  17.371+
  17.372+void
  17.373+noted(Ureg *ureg, ulong arg0)
  17.374+{
  17.375+	Ureg *nureg;
  17.376+	uintptr oureg, sp;
  17.377+	
  17.378+	qlock(&up->debug);
  17.379+	if(arg0 != NRSTR && !up->notified){
  17.380+		qunlock(&up->debug);
  17.381+		pprint("call to noted() when not notified\n");
  17.382+		pexit("Suicide", 0);
  17.383+	}
  17.384+	up->notified = 0;
  17.385+	
  17.386+	nureg = up->ureg;
  17.387+	up->fpstate &= ~FPillegal;
  17.388+	
  17.389+	oureg = (uintptr) nureg;
  17.390+	if(!okaddr(oureg - BY2WD, BY2WD + sizeof(Ureg), 0) || (oureg & 7) != 0){
  17.391+		qunlock(&up->debug);
  17.392+		pprint("bad ureg in noted or call to noted when not notified\n");
  17.393+		pexit("Suicide", 0);
  17.394+	}
  17.395+
  17.396+	nureg->psr = (nureg->psr & USPSRMASK) | (ureg->psr & ~USPSRMASK);
  17.397+	memmove(ureg, nureg, sizeof(Ureg));
  17.398+	
  17.399+	switch(arg0){
  17.400+	case NCONT: case NRSTR:
  17.401+		if(!okaddr(nureg->pc, BY2WD, 0) || !okaddr(nureg->sp, BY2WD, 0) ||
  17.402+				(nureg->pc & 3) != 0 || (nureg->sp & 7) != 0){
  17.403+			qunlock(&up->debug);
  17.404+			pprint("suicide: trap in noted\n");
  17.405+			pexit("Suicide", 0);
  17.406+		}
  17.407+		up->ureg = (Ureg *) (*(uintptr*) (oureg - BY2WD));
  17.408+		qunlock(&up->debug);
  17.409+		break;
  17.410+	
  17.411+	case NSAVE:
  17.412+		if(!okaddr(nureg->pc, BY2WD, 0) || !okaddr(nureg->sp, BY2WD, 0) ||
  17.413+				(nureg->pc & 3) != 0 || (nureg->sp & 7) != 0){
  17.414+			qunlock(&up->debug);
  17.415+			pprint("suicide: trap in noted\n");
  17.416+			pexit("Suicide", 0);
  17.417+		}
  17.418+		qunlock(&up->debug);
  17.419+		sp = oureg - 4 * BY2WD - ERRMAX;
  17.420+		splhi();
  17.421+		ureg->sp = sp;
  17.422+		ureg->r0 = (uintptr) oureg;
  17.423+		((uintptr *) sp)[1] = oureg;
  17.424+		((uintptr *) sp)[0] = 0;
  17.425+		break;
  17.426+	
  17.427+	default:
  17.428+		up->lastnote.flag = NDebug;
  17.429+	
  17.430+	case NDFLT:
  17.431+		qunlock(&up->debug);
  17.432+		if(up->lastnote.flag == NDebug)
  17.433+			pprint("suicide: %s\n", up->lastnote.msg);
  17.434+		pexit(up->lastnote.msg, up->lastnote.flag != NDebug);
  17.435+	}
  17.436+}
  17.437+
  17.438+void
  17.439+faultarm64(Ureg *ureg)
  17.440+{
  17.441+	extern void checkpages(void);
  17.442+	char buf[ERRMAX];
  17.443+	int read, insyscall;
  17.444+	uintptr addr;
  17.445+
  17.446+	insyscall = up->insyscall;
  17.447+	up->insyscall = 1;
  17.448+
  17.449+	if(!userureg(ureg) && waserror()){
  17.450+		if(up->nerrlab == 0){
  17.451+			pprint("suicide: sys: %s\n", up->errstr);
  17.452+			pexit(up->errstr, 1);
  17.453+		}
  17.454+		up->insyscall = insyscall;
  17.455+		nexterror();
  17.456+	}
  17.457+
  17.458+	addr = getfar();
  17.459+	read = (ureg->type & (1<<6)) == 0;
  17.460+
  17.461+	switch((u32int)ureg->type & 0x3F){
  17.462+	case  4: case  5: case  6: case  7:	// Tanslation fault.
  17.463+	case  8: case  9: case 10: case 11:	// Access flag fault.
  17.464+	case 12: case 13: case 14: case 15:	// Permission fault.
  17.465+	case 48:				// tlb conflict fault.
  17.466+		if(fault(addr, read) == 0)
  17.467+			break;
  17.468+
  17.469+		/* wet floor */
  17.470+	case  0: case  1: case  2: case  3:	// Address size fault.
  17.471+	case 16: 				// synchronous external abort.
  17.472+	case 24: 				// synchronous parity error on a memory access.
  17.473+	case 20: case 21: case 22: case 23:	// synchronous external abort on a table walk.
  17.474+	case 28: case 29: case 30: case 31:	// synchronous parity error on table walk.
  17.475+	case 33:				// alignment fault.
  17.476+	case 52:				// implementation defined, lockdown abort.
  17.477+	case 53:				// implementation defined, unsuppoted exclusive.
  17.478+	case 61:				// first level domain fault
  17.479+	case 62:				// second level domain fault
  17.480+	default:
  17.481+		if(!userureg(ureg)){
  17.482+			dumpregs(ureg);
  17.483+			panic("fault: %s addr=%#p", read ? "read" : "write", addr);
  17.484+		}
  17.485+		checkpages();
  17.486+		sprint(buf, "sys: trap: fault %s addr=%#p", read ? "read" : "write", addr);
  17.487+		postnote(up, 1, buf, NDebug);
  17.488+	}
  17.489+
  17.490+	if(!userureg(ureg))
  17.491+		poperror();
  17.492+
  17.493+	up->insyscall = insyscall;
  17.494+}
  17.495+
  17.496+int
  17.497+userureg(Ureg* ureg)
  17.498+{
  17.499+	return (ureg->psr & 15) == 0;
  17.500+}
  17.501+
  17.502+uintptr
  17.503+userpc(void)
  17.504+{
  17.505+	Ureg *ur = up->dbgreg;
  17.506+	return ur->pc;
  17.507+}
  17.508+
  17.509+uintptr
  17.510+dbgpc(Proc *)
  17.511+{
  17.512+	Ureg *ur = up->dbgreg;
  17.513+	if(ur == nil)
  17.514+		return 0;
  17.515+	return ur->pc;
  17.516+}
  17.517+
  17.518+void
  17.519+procfork(Proc *p)
  17.520+{
  17.521+	int s;
  17.522+
  17.523+	s = splhi();
  17.524+	switch(up->fpstate & ~FPillegal){
  17.525+	case FPactive:
  17.526+		fpsave(up->fpsave);
  17.527+		up->fpstate = FPinactive;
  17.528+	case FPinactive:
  17.529+		memmove(p->fpsave, up->fpsave, sizeof(FPsave));
  17.530+		p->fpstate = FPinactive;
  17.531+	}
  17.532+	splx(s);
  17.533+
  17.534+	p->tpidr = up->tpidr;
  17.535+
  17.536+	p->kentry = up->kentry;
  17.537+	p->pcycles = -p->kentry;
  17.538+}
  17.539+
  17.540+void
  17.541+procsetup(Proc *p)
  17.542+{
  17.543+	p->fpstate = FPinit;
  17.544+	fpoff();
  17.545+
  17.546+	p->tpidr = 0;
  17.547+	syswr(TPIDR_EL0, p->tpidr);
  17.548+
  17.549+	p->kentry = cycles();
  17.550+	p->pcycles = -p->kentry;
  17.551+}
  17.552+
  17.553+void
  17.554+procsave(Proc *p)
  17.555+{
  17.556+	uvlong t;
  17.557+
  17.558+	if(p->fpstate == FPactive){
  17.559+		if(p->state == Moribund)
  17.560+			fpclear();
  17.561+		else
  17.562+			fpsave(p->fpsave);
  17.563+		p->fpstate = FPinactive;
  17.564+	}
  17.565+
  17.566+	if(p->kp == 0)
  17.567+		p->tpidr = sysrd(TPIDR_EL0);
  17.568+
  17.569+	putasid(p);	// release asid
  17.570+
  17.571+	t = cycles();
  17.572+	p->kentry -= t;
  17.573+	p->pcycles += t;
  17.574+}
  17.575+
  17.576+void
  17.577+procrestore(Proc *p)
  17.578+{
  17.579+	uvlong t;
  17.580+
  17.581+	if(p->kp)
  17.582+		return;
  17.583+	
  17.584+	syswr(TPIDR_EL0, p->tpidr);
  17.585+
  17.586+	t = cycles();
  17.587+	p->kentry += t;
  17.588+	p->pcycles -= t;
  17.589+}
  17.590+
  17.591+static void
  17.592+linkproc(void)
  17.593+{
  17.594+	spllo();
  17.595+	up->kpfun(up->kparg);
  17.596+	pexit("kproc dying", 0);
  17.597+}
  17.598+
  17.599+void
  17.600+kprocchild(Proc* p, void (*func)(void*), void* arg)
  17.601+{
  17.602+	p->sched.pc = (uintptr) linkproc;
  17.603+	p->sched.sp = (uintptr) p->kstack + KSTACK - 16;
  17.604+	*(void**)p->sched.sp = kprocchild;	/* fake */
  17.605+
  17.606+	p->kpfun = func;
  17.607+	p->kparg = arg;
  17.608+}
  17.609+
  17.610+void
  17.611+forkchild(Proc *p, Ureg *ureg)
  17.612+{
  17.613+	Ureg *cureg;
  17.614+
  17.615+	p->sched.pc = (uintptr) forkret;
  17.616+	p->sched.sp = (uintptr) p->kstack + KSTACK - TRAPFRAMESIZE;
  17.617+
  17.618+	cureg = (Ureg*) (p->sched.sp + 16);
  17.619+	memmove(cureg, ureg, sizeof(Ureg));
  17.620+	cureg->r0 = 0;
  17.621+
  17.622+	p->psstate = 0;
  17.623+	p->insyscall = 0;
  17.624+}
  17.625+
  17.626+uintptr
  17.627+execregs(uintptr entry, ulong ssize, ulong nargs)
  17.628+{
  17.629+	uintptr *sp;
  17.630+	Ureg *ureg;
  17.631+
  17.632+	sp = (uintptr*)(USTKTOP - ssize);
  17.633+	*--sp = nargs;
  17.634+
  17.635+	ureg = up->dbgreg;
  17.636+	ureg->sp = (uintptr)sp;
  17.637+	ureg->pc = entry;
  17.638+	ureg->link = 0;
  17.639+	return USTKTOP-sizeof(Tos);
  17.640+}
  17.641+
  17.642+void
  17.643+evenaddr(uintptr addr)
  17.644+{
  17.645+	if(addr & 3){
  17.646+		postnote(up, 1, "sys: odd address", NDebug);
  17.647+		error(Ebadarg);
  17.648+	}
  17.649+}
  17.650+
  17.651+void
  17.652+callwithureg(void (*f) (Ureg *))
  17.653+{
  17.654+	Ureg u;
  17.655+	
  17.656+	u.pc = getcallerpc(&f);
  17.657+	u.sp = (uintptr) &f;
  17.658+	f(&u);
  17.659+}
  17.660+
  17.661+void
  17.662+setkernur(Ureg *ureg, Proc *p)
  17.663+{
  17.664+	ureg->pc = p->sched.pc;
  17.665+	ureg->sp = p->sched.sp;
  17.666+	ureg->link = (uintptr)sched;
  17.667+}
  17.668+
  17.669+void
  17.670+setupwatchpts(Proc*, Watchpt*, int)
  17.671+{
  17.672+}
  17.673+
  17.674+void
  17.675+setregisters(Ureg* ureg, char* pureg, char* uva, int n)
  17.676+{
  17.677+	ulong v;
  17.678+
  17.679+	v = ureg->psr;
  17.680+	memmove(pureg, uva, n);
  17.681+	ureg->psr = (ureg->psr & USPSRMASK) | (v & ~USPSRMASK);
  17.682+}
  17.683+
  17.684+static void
  17.685+dumpstackwithureg(Ureg *ureg)
  17.686+{
  17.687+	uintptr v, estack, sp;
  17.688+	char *s;
  17.689+	int i;
  17.690+
  17.691+	if((s = getconf("*nodumpstack")) != nil && strcmp(s, "0") != 0){
  17.692+		iprint("dumpstack disabled\n");
  17.693+		return;
  17.694+	}
  17.695+	iprint("ktrace /kernel/path %#p %#p %#p # pc, sp, link\n",
  17.696+		ureg->pc, ureg->sp, ureg->link);
  17.697+	delay(2000);
  17.698+
  17.699+	sp = ureg->sp;
  17.700+	if(sp < KZERO || (sp & 7) != 0)
  17.701+		sp = (uintptr)&ureg;
  17.702+
  17.703+	estack = (uintptr)m+MACHSIZE;
  17.704+	if(up != nil && sp <= (uintptr)up->kstack+KSTACK)
  17.705+		estack = (uintptr)up->kstack+KSTACK;
  17.706+
  17.707+	if(sp > estack){
  17.708+		if(up != nil)
  17.709+			iprint("&up->kstack %#p sp %#p\n", up->kstack, sp);
  17.710+		else
  17.711+			iprint("&m %#p sp %#p\n", m, sp);
  17.712+		return;
  17.713+	}
  17.714+
  17.715+	i = 0;
  17.716+	for(; sp < estack; sp += sizeof(uintptr)){
  17.717+		v = *(uintptr*)sp;
  17.718+		if(KTZERO < v && v < (uintptr)etext && (v & 3) == 0){
  17.719+			iprint("%#8.8lux=%#8.8lux ", (ulong)sp, (ulong)v);
  17.720+			i++;
  17.721+		}
  17.722+		if(i == 4){
  17.723+			i = 0;
  17.724+			iprint("\n");
  17.725+		}
  17.726+	}
  17.727+	if(i)
  17.728+		iprint("\n");
  17.729+}
  17.730+
  17.731+void
  17.732+dumpstack(void)
  17.733+{
  17.734+	callwithureg(dumpstackwithureg);
  17.735+}
  17.736+
  17.737+void
  17.738+dumpregs(Ureg *ureg)
  17.739+{
  17.740+	u64int *r;
  17.741+	int i, x;
  17.742+
  17.743+	x = splhi();
  17.744+	if(up != nil)
  17.745+		iprint("cpu%d: dumpregs ureg %#p process %lud: %s\n", m->machno, ureg,
  17.746+			up->pid, up->text);
  17.747+	else
  17.748+		iprint("cpu%d: dumpregs ureg %#p\n", m->machno, ureg);
  17.749+	r = &ureg->r0;
  17.750+	for(i = 0; i < 30; i += 3)
  17.751+		iprint("R%d %.16llux  R%d %.16llux  R%d %.16llux\n", i, r[i], i+1, r[i+1], i+2, r[i+2]);
  17.752+	iprint("PC %#p  SP %#p  LR %#p  PSR %llux  TYPE %llux\n",
  17.753+		ureg->pc, ureg->sp, ureg->link,
  17.754+		ureg->psr, ureg->type);
  17.755+	splx(x);
  17.756+}