changelog shortlog tags branches changeset file revisions annotate raw help

Mercurial > hg > plan9front / sys/src/9/bcm64/pci.c

revision 7315: 6e8b95bccec4
     1.1new file mode 100644
     1.2--- /dev/null
     1.3+++ b/sys/src/9/bcm64/pci.c
     1.4@@ -0,0 +1,1129 @@
     1.5+#include "u.h"
     1.6+#include "../port/lib.h"
     1.7+#include "mem.h"
     1.8+#include "dat.h"
     1.9+#include "fns.h"
    1.10+#include "io.h"
    1.11+
    1.12+/* bcmstb PCIe controller registers */
    1.13+enum{
    1.14+	RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1	= 0x0188/4,
    1.15+	RC_CFG_PRIV1_ID_VAL3			= 0x043c/4,
    1.16+	RC_DL_MDIO_ADDR				= 0x1100/4,
    1.17+	RC_DL_MDIO_WR_DATA			= 0x1104/4,
    1.18+	RC_DL_MDIO_RD_DATA			= 0x1108/4,
    1.19+	MISC_MISC_CTRL				= 0x4008/4,
    1.20+	MISC_CPU_2_PCIE_MEM_WIN0_LO		= 0x400c/4,
    1.21+	MISC_CPU_2_PCIE_MEM_WIN0_HI		= 0x4010/4,
    1.22+	MISC_RC_BAR1_CONFIG_LO			= 0x402c/4,
    1.23+	MISC_RC_BAR2_CONFIG_LO			= 0x4034/4,
    1.24+	MISC_RC_BAR2_CONFIG_HI			= 0x4038/4,
    1.25+	MISC_RC_BAR3_CONFIG_LO			= 0x403c/4,
    1.26+	MISC_MSI_BAR_CONFIG_LO			= 0x4044/4,
    1.27+	MISC_MSI_BAR_CONFIG_HI			= 0x4048/4,
    1.28+	MISC_MSI_DATA_CONFIG			= 0x404c/4,
    1.29+	MISC_EOI_CTRL				= 0x4060/4,
    1.30+	MISC_PCIE_CTRL				= 0x4064/4,
    1.31+	MISC_PCIE_STATUS			= 0x4068/4,
    1.32+	MISC_REVISION				= 0x406c/4,
    1.33+	MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT	= 0x4070/4,
    1.34+	MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI	= 0x4080/4,
    1.35+	MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI	= 0x4084/4,
    1.36+	MISC_HARD_PCIE_HARD_DEBUG		= 0x4204/4,
    1.37+
    1.38+	INTR2_CPU_BASE				= 0x4300/4,
    1.39+	MSI_INTR2_BASE				= 0x4500/4,
    1.40+		INTR_STATUS = 0,
    1.41+		INTR_SET,
    1.42+		INTR_CLR,
    1.43+		INTR_MASK_STATUS,
    1.44+		INTR_MASK_SET,
    1.45+		INTR_MASK_CLR,
    1.46+
    1.47+	EXT_CFG_INDEX				= 0x9000/4,
    1.48+	RGR1_SW_INIT_1				= 0x9210/4,
    1.49+	EXT_CFG_DATA				= 0x8000/4,
    1.50+
    1.51+};
    1.52+
    1.53+#define MSI_TARGET_ADDR		0xFFFFFFFFCULL
    1.54+
    1.55+static u32int *regs = (u32int*)(VIRTIO1 + 0x500000);
    1.56+
    1.57+static Lock pcicfglock;
    1.58+static int pcimaxbno = 0;
    1.59+static int pcimaxdno = 0;
    1.60+static Pcidev* pciroot;
    1.61+static Pcidev* pcilist;
    1.62+static Pcidev* pcitail;
    1.63+
    1.64+typedef struct Pcisiz Pcisiz;
    1.65+struct Pcisiz
    1.66+{
    1.67+	Pcidev*	dev;
    1.68+	int	siz;
    1.69+	int	bar;
    1.70+};
    1.71+
    1.72+enum
    1.73+{
    1.74+	MaxFNO		= 7,
    1.75+	MaxUBN		= 255,
    1.76+};
    1.77+
    1.78+static char* bustypes[] = {
    1.79+	"CBUSI",
    1.80+	"CBUSII",
    1.81+	"EISA",
    1.82+	"FUTURE",
    1.83+	"INTERN",
    1.84+	"ISA",
    1.85+	"MBI",
    1.86+	"MBII",
    1.87+	"MCA",
    1.88+	"MPI",
    1.89+	"MPSA",
    1.90+	"NUBUS",
    1.91+	"PCI",
    1.92+	"PCMCIA",
    1.93+	"TC",
    1.94+	"VL",
    1.95+	"VME",
    1.96+	"XPRESS",
    1.97+};
    1.98+
    1.99+static int
   1.100+tbdffmt(Fmt* fmt)
   1.101+{
   1.102+	char *p;
   1.103+	int l, r;
   1.104+	uint type, tbdf;
   1.105+
   1.106+	if((p = malloc(READSTR)) == nil)
   1.107+		return fmtstrcpy(fmt, "(tbdfconv)");
   1.108+
   1.109+	switch(fmt->r){
   1.110+	case 'T':
   1.111+		tbdf = va_arg(fmt->args, int);
   1.112+		if(tbdf == BUSUNKNOWN)
   1.113+			snprint(p, READSTR, "unknown");
   1.114+		else{
   1.115+			type = BUSTYPE(tbdf);
   1.116+			if(type < nelem(bustypes))
   1.117+				l = snprint(p, READSTR, bustypes[type]);
   1.118+			else
   1.119+				l = snprint(p, READSTR, "%d", type);
   1.120+			snprint(p+l, READSTR-l, ".%d.%d.%d",
   1.121+				BUSBNO(tbdf), BUSDNO(tbdf), BUSFNO(tbdf));
   1.122+		}
   1.123+		break;
   1.124+
   1.125+	default:
   1.126+		snprint(p, READSTR, "(tbdfconv)");
   1.127+		break;
   1.128+	}
   1.129+	r = fmtstrcpy(fmt, p);
   1.130+	free(p);
   1.131+
   1.132+	return r;
   1.133+}
   1.134+
   1.135+static void pcicfginit(void);
   1.136+
   1.137+static void*
   1.138+cfgaddr(int tbdf, int rno)
   1.139+{
   1.140+	if(BUSBNO(tbdf) == 0 && BUSDNO(tbdf) == 0)
   1.141+		return (uchar*)regs + rno;
   1.142+	regs[EXT_CFG_INDEX] = BUSBNO(tbdf) << 20 | BUSDNO(tbdf) << 15 | BUSFNO(tbdf) << 12;
   1.143+	coherence();
   1.144+	return ((uchar*)&regs[EXT_CFG_DATA]) + rno;
   1.145+}
   1.146+
   1.147+static int
   1.148+pcicfgrw32(int tbdf, int rno, int data, int read)
   1.149+{
   1.150+	int x = -1;
   1.151+	u32int *p;
   1.152+
   1.153+	ilock(&pcicfglock);
   1.154+	if((p = cfgaddr(tbdf, rno & ~3)) != nil){
   1.155+		if(read)
   1.156+			x = *p;
   1.157+		else
   1.158+			*p = data;
   1.159+	}
   1.160+	iunlock(&pcicfglock);
   1.161+	return x;
   1.162+}
   1.163+static int
   1.164+pcicfgrw16(int tbdf, int rno, int data, int read)
   1.165+{
   1.166+	int x = -1;
   1.167+	u16int *p;
   1.168+
   1.169+	ilock(&pcicfglock);
   1.170+	if((p = cfgaddr(tbdf, rno & ~1)) != nil){
   1.171+		if(read)
   1.172+			x = *p;
   1.173+		else
   1.174+			*p = data;
   1.175+	}
   1.176+	iunlock(&pcicfglock);
   1.177+	return x;
   1.178+}
   1.179+static int
   1.180+pcicfgrw8(int tbdf, int rno, int data, int read)
   1.181+{
   1.182+	int x = -1;
   1.183+	u8int *p;
   1.184+
   1.185+	ilock(&pcicfglock);
   1.186+	if((p = cfgaddr(tbdf, rno)) != nil){
   1.187+		if(read)
   1.188+			x = *p;
   1.189+		else
   1.190+			*p = data;
   1.191+	}
   1.192+	iunlock(&pcicfglock);
   1.193+	return x;
   1.194+}
   1.195+
   1.196+int
   1.197+pcicfgr32(Pcidev* pcidev, int rno)
   1.198+{
   1.199+	return pcicfgrw32(pcidev->tbdf, rno, 0, 1);
   1.200+}
   1.201+void
   1.202+pcicfgw32(Pcidev* pcidev, int rno, int data)
   1.203+{
   1.204+	pcicfgrw32(pcidev->tbdf, rno, data, 0);
   1.205+}
   1.206+int
   1.207+pcicfgr16(Pcidev* pcidev, int rno)
   1.208+{
   1.209+	return pcicfgrw16(pcidev->tbdf, rno, 0, 1);
   1.210+}
   1.211+void
   1.212+pcicfgw16(Pcidev* pcidev, int rno, int data)
   1.213+{
   1.214+	pcicfgrw16(pcidev->tbdf, rno, data, 0);
   1.215+}
   1.216+int
   1.217+pcicfgr8(Pcidev* pcidev, int rno)
   1.218+{
   1.219+	return pcicfgrw8(pcidev->tbdf, rno, 0, 1);
   1.220+}
   1.221+void
   1.222+pcicfgw8(Pcidev* pcidev, int rno, int data)
   1.223+{
   1.224+	pcicfgrw8(pcidev->tbdf, rno, data, 0);
   1.225+}
   1.226+
   1.227+Pcidev*
   1.228+pcimatch(Pcidev* prev, int vid, int did)
   1.229+{
   1.230+	if(prev == nil)
   1.231+		prev = pcilist;
   1.232+	else
   1.233+		prev = prev->list;
   1.234+
   1.235+	while(prev != nil){
   1.236+		if((vid == 0 || prev->vid == vid)
   1.237+		&& (did == 0 || prev->did == did))
   1.238+			break;
   1.239+		prev = prev->list;
   1.240+	}
   1.241+	return prev;
   1.242+}
   1.243+
   1.244+Pcidev*
   1.245+pcimatchtbdf(int tbdf)
   1.246+{
   1.247+	Pcidev *pcidev;
   1.248+
   1.249+	for(pcidev = pcilist; pcidev != nil; pcidev = pcidev->list) {
   1.250+		if(pcidev->tbdf == tbdf)
   1.251+			break;
   1.252+	}
   1.253+	return pcidev;
   1.254+}
   1.255+
   1.256+static u32int
   1.257+pcibarsize(Pcidev *p, int rno)
   1.258+{
   1.259+	u32int v, size;
   1.260+
   1.261+	v = pcicfgrw32(p->tbdf, rno, 0, 1);
   1.262+	pcicfgrw32(p->tbdf, rno, 0xFFFFFFF0, 0);
   1.263+	size = pcicfgrw32(p->tbdf, rno, 0, 1);
   1.264+	if(v & 1)
   1.265+		size |= 0xFFFF0000;
   1.266+	pcicfgrw32(p->tbdf, rno, v, 0);
   1.267+
   1.268+	return -(size & ~0x0F);
   1.269+}
   1.270+
   1.271+static int
   1.272+pcisizcmp(void *a, void *b)
   1.273+{
   1.274+	Pcisiz *aa, *bb;
   1.275+
   1.276+	aa = a;
   1.277+	bb = b;
   1.278+	return aa->siz - bb->siz;
   1.279+}
   1.280+
   1.281+static ulong
   1.282+pcimask(ulong v)
   1.283+{
   1.284+	ulong m;
   1.285+
   1.286+	m = BI2BY*sizeof(v);
   1.287+	for(m = 1<<(m-1); m != 0; m >>= 1) {
   1.288+		if(m & v)
   1.289+			break;
   1.290+	}
   1.291+
   1.292+	m--;
   1.293+	if((v & m) == 0)
   1.294+		return v;
   1.295+
   1.296+	v |= m;
   1.297+	return v+1;
   1.298+}
   1.299+
   1.300+static void
   1.301+pcibusmap(Pcidev *root, uintptr *pmema, uintptr *pioa, int wrreg)
   1.302+{
   1.303+	Pcidev *p;
   1.304+	int ntb, i, size, rno, hole;
   1.305+	uintptr v, mema, ioa, sioa, smema, base, limit;
   1.306+	Pcisiz *table, *tptr, *mtb, *itb;
   1.307+
   1.308+	ioa = *pioa;
   1.309+	mema = *pmema;
   1.310+
   1.311+	ntb = 0;
   1.312+	for(p = root; p != nil; p = p->link)
   1.313+		ntb++;
   1.314+
   1.315+	ntb *= (PciCIS-PciBAR0)/4;
   1.316+	table = malloc(2*ntb*sizeof(Pcisiz));
   1.317+	if(table == nil)
   1.318+		panic("pcibusmap: can't allocate memory");
   1.319+	itb = table;
   1.320+	mtb = table+ntb;
   1.321+
   1.322+	/*
   1.323+	 * Build a table of sizes
   1.324+	 */
   1.325+	for(p = root; p != nil; p = p->link) {
   1.326+		if(p->ccrb == 0x06) {
   1.327+			if(p->ccru != 0x04 || p->bridge == nil)
   1.328+				continue;
   1.329+
   1.330+			sioa = ioa;
   1.331+			smema = mema;
   1.332+			pcibusmap(p->bridge, &smema, &sioa, 0);
   1.333+
   1.334+			hole = pcimask(smema-mema);
   1.335+			if(hole < (1<<20))
   1.336+				hole = 1<<20;
   1.337+			p->mema.size = hole;
   1.338+
   1.339+			hole = pcimask(sioa-ioa);
   1.340+			if(hole < (1<<12))
   1.341+				hole = 1<<12;
   1.342+
   1.343+			p->ioa.size = hole;
   1.344+
   1.345+			itb->dev = p;
   1.346+			itb->bar = -1;
   1.347+			itb->siz = p->ioa.size;
   1.348+			itb++;
   1.349+
   1.350+			mtb->dev = p;
   1.351+			mtb->bar = -1;
   1.352+			mtb->siz = p->mema.size;
   1.353+			mtb++;
   1.354+			continue;
   1.355+		}
   1.356+
   1.357+		for(i = 0; i <= 5; i++) {
   1.358+			rno = PciBAR0 + i*4;
   1.359+			v = pcicfgrw32(p->tbdf, rno, 0, 1);
   1.360+			size = pcibarsize(p, rno);
   1.361+			if(size == 0)
   1.362+				continue;
   1.363+
   1.364+			p->mem[i].size = size;
   1.365+			if(v & 1) {
   1.366+				itb->dev = p;
   1.367+				itb->bar = i;
   1.368+				itb->siz = size;
   1.369+				itb++;
   1.370+			}
   1.371+			else {
   1.372+				mtb->dev = p;
   1.373+				mtb->bar = i;
   1.374+				mtb->siz = size;
   1.375+				mtb++;
   1.376+
   1.377+				if((v & 7) == 4)
   1.378+					i++;
   1.379+			}
   1.380+		}
   1.381+	}
   1.382+
   1.383+	/*
   1.384+	 * Sort both tables IO smallest first, Memory largest
   1.385+	 */
   1.386+	qsort(table, itb-table, sizeof(Pcisiz), pcisizcmp);
   1.387+	tptr = table+ntb;
   1.388+	qsort(tptr, mtb-tptr, sizeof(Pcisiz), pcisizcmp);
   1.389+
   1.390+	/*
   1.391+	 * Allocate IO address space on this bus
   1.392+	 */
   1.393+	for(tptr = table; tptr < itb; tptr++) {
   1.394+		hole = tptr->siz;
   1.395+		if(tptr->bar == -1)
   1.396+			hole = 1<<12;
   1.397+		ioa = (ioa+hole-1) & ~(hole-1);
   1.398+
   1.399+		p = tptr->dev;
   1.400+		if(tptr->bar == -1)
   1.401+			p->ioa.bar = ioa;
   1.402+		else {
   1.403+			p->pcr |= IOen;
   1.404+			p->mem[tptr->bar].bar = ioa|1;
   1.405+			if(wrreg)
   1.406+				pcicfgrw32(p->tbdf, PciBAR0+(tptr->bar*4), ioa|1, 0);
   1.407+		}
   1.408+
   1.409+		ioa += tptr->siz;
   1.410+	}
   1.411+
   1.412+	/*
   1.413+	 * Allocate Memory address space on this bus
   1.414+	 */
   1.415+	for(tptr = table+ntb; tptr < mtb; tptr++) {
   1.416+		hole = tptr->siz;
   1.417+		if(tptr->bar == -1)
   1.418+			hole = 1<<20;
   1.419+		mema = (mema+hole-1) & ~(hole-1);
   1.420+
   1.421+		p = tptr->dev;
   1.422+		if(tptr->bar == -1)
   1.423+			p->mema.bar = mema;
   1.424+		else {
   1.425+			p->pcr |= MEMen;
   1.426+			p->mem[tptr->bar].bar = mema;
   1.427+			if(wrreg){
   1.428+				rno = PciBAR0+(tptr->bar*4);
   1.429+				if((mema >> 32) != 0){
   1.430+					pcicfgrw32(p->tbdf, rno, mema|4, 0);
   1.431+					pcicfgrw32(p->tbdf, rno+4, mema >> 32, 0);
   1.432+				} else {
   1.433+					pcicfgrw32(p->tbdf, rno, mema, 0);
   1.434+				}
   1.435+			}
   1.436+		}
   1.437+		mema += tptr->siz;
   1.438+	}
   1.439+
   1.440+	*pmema = mema;
   1.441+	*pioa = ioa;
   1.442+	free(table);
   1.443+
   1.444+	if(wrreg == 0)
   1.445+		return;
   1.446+
   1.447+	/*
   1.448+	 * Finally set all the bridge addresses & registers
   1.449+	 */
   1.450+	for(p = root; p != nil; p = p->link) {
   1.451+		if(p->bridge == nil) {
   1.452+			if(p->cls == 0){
   1.453+				p->cls = 64;
   1.454+				pcicfgw8(p, PciCLS, p->cls);
   1.455+			}
   1.456+			pcicfgrw8(p->tbdf, PciLTR, 64, 0);
   1.457+			p->pcr |= MASen;
   1.458+			pcicfgrw16(p->tbdf, PciPCR, p->pcr, 0);
   1.459+			continue;
   1.460+		}
   1.461+
   1.462+		if(p == pciroot){
   1.463+			base = p->mema.bar;
   1.464+			limit = base+p->mema.size-1;
   1.465+			regs[MISC_CPU_2_PCIE_MEM_WIN0_LO] = base;
   1.466+			regs[MISC_CPU_2_PCIE_MEM_WIN0_HI] = base >> 32;
   1.467+			base >>= 20, limit >>= 20;
   1.468+			regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT] = (base & 0xFFF) << 4 | (limit & 0xFFF) << 20;
   1.469+			regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI] = base >> 12;
   1.470+			regs[MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI] = limit >> 12;
   1.471+		}
   1.472+
   1.473+		base = p->ioa.bar;
   1.474+		limit = base+p->ioa.size-1;
   1.475+		v = pcicfgrw32(p->tbdf, PciIBR, 0, 1);
   1.476+		v = (v&0xFFFF0000)|(limit & 0xF000)|((base & 0xF000)>>8);
   1.477+		pcicfgrw32(p->tbdf, PciIBR, v, 0);
   1.478+		v = (limit & 0xFFFF0000)|(base>>16);
   1.479+		pcicfgrw32(p->tbdf, PciIUBR, v, 0);
   1.480+
   1.481+		base = p->mema.bar;
   1.482+		limit = base+p->mema.size-1;
   1.483+		v = (limit & 0xFFF00000)|((base & 0xFFF00000)>>16);
   1.484+		pcicfgrw32(p->tbdf, PciMBR, v, 0);
   1.485+
   1.486+		/*
   1.487+		 * Disable memory prefetch
   1.488+		 */
   1.489+		pcicfgrw32(p->tbdf, PciPMBR, 0x0000FFFF, 0);
   1.490+		pcicfgrw8(p->tbdf, PciLTR, 64, 0);
   1.491+
   1.492+		/*
   1.493+		 * Enable the bridge
   1.494+		 */
   1.495+		p->pcr |= IOen|MEMen|MASen;
   1.496+		pcicfgrw32(p->tbdf, PciPCR, 0xFFFF0000|p->pcr, 0);
   1.497+
   1.498+		sioa = p->ioa.bar;
   1.499+		smema = p->mema.bar;
   1.500+		pcibusmap(p->bridge, &smema, &sioa, 1);
   1.501+	}
   1.502+}
   1.503+
   1.504+static int
   1.505+pcilscan(int bno, Pcidev** list, Pcidev *parent)
   1.506+{
   1.507+	Pcidev *p, *head, *tail;
   1.508+	int dno, fno, i, hdt, l, maxfno, maxubn, rno, sbn, tbdf, ubn;
   1.509+
   1.510+	maxubn = bno;
   1.511+	head = nil;
   1.512+	tail = nil;
   1.513+	for(dno = 0; dno <= pcimaxdno; dno++){
   1.514+		maxfno = 0;
   1.515+		for(fno = 0; fno <= maxfno; fno++){
   1.516+			/*
   1.517+			 * For this possible device, form the
   1.518+			 * bus+device+function triplet needed to address it
   1.519+			 * and try to read the vendor and device ID.
   1.520+			 * If successful, allocate a device struct and
   1.521+			 * start to fill it in with some useful information
   1.522+			 * from the device's configuration space.
   1.523+			 */
   1.524+			tbdf = MKBUS(BusPCI, bno, dno, fno);
   1.525+			l = pcicfgrw32(tbdf, PciVID, 0, 1);
   1.526+			if(l == 0xFFFFFFFF || l == 0)
   1.527+				continue;
   1.528+			p = malloc(sizeof(*p));
   1.529+			if(p == nil)
   1.530+				panic("pcilscan: no memory");
   1.531+			p->tbdf = tbdf;
   1.532+			p->vid = l;
   1.533+			p->did = l>>16;
   1.534+
   1.535+			if(pcilist != nil)
   1.536+				pcitail->list = p;
   1.537+			else
   1.538+				pcilist = p;
   1.539+			pcitail = p;
   1.540+
   1.541+			p->pcr = pcicfgr16(p, PciPCR);
   1.542+			p->rid = pcicfgr8(p, PciRID);
   1.543+			p->ccrp = pcicfgr8(p, PciCCRp);
   1.544+			p->ccru = pcicfgr8(p, PciCCRu);
   1.545+			p->ccrb = pcicfgr8(p, PciCCRb);
   1.546+			p->cls = pcicfgr8(p, PciCLS);
   1.547+			p->ltr = pcicfgr8(p, PciLTR);
   1.548+
   1.549+			p->intl = pcicfgr8(p, PciINTL);
   1.550+
   1.551+			/*
   1.552+			 * If the device is a multi-function device adjust the
   1.553+			 * loop count so all possible functions are checked.
   1.554+			 */
   1.555+			hdt = pcicfgr8(p, PciHDT);
   1.556+			if(hdt & 0x80)
   1.557+				maxfno = MaxFNO;
   1.558+
   1.559+			/*
   1.560+			 * If appropriate, read the base address registers
   1.561+			 * and work out the sizes.
   1.562+			 */
   1.563+			switch(p->ccrb) {
   1.564+			case 0x00:		/* prehistoric */
   1.565+			case 0x01:		/* mass storage controller */
   1.566+			case 0x02:		/* network controller */
   1.567+			case 0x03:		/* display controller */
   1.568+			case 0x04:		/* multimedia device */
   1.569+			case 0x07:		/* simple comm. controllers */
   1.570+			case 0x08:		/* base system peripherals */
   1.571+			case 0x09:		/* input devices */
   1.572+			case 0x0A:		/* docking stations */
   1.573+			case 0x0B:		/* processors */
   1.574+			case 0x0C:		/* serial bus controllers */
   1.575+			case 0x0D:		/* wireless controllers */
   1.576+			case 0x0E:		/* intelligent I/O controllers */
   1.577+			case 0x0F:		/* sattelite communication controllers */
   1.578+			case 0x10:		/* encryption/decryption controllers */
   1.579+			case 0x11:		/* signal processing controllers */
   1.580+				if((hdt & 0x7F) != 0)
   1.581+					break;
   1.582+				rno = PciBAR0;
   1.583+				for(i = 0; i <= 5; i++) {
   1.584+					p->mem[i].bar = pcicfgr32(p, rno);
   1.585+					p->mem[i].size = pcibarsize(p, rno);
   1.586+					if((p->mem[i].bar & 7) == 4 && i < 5){
   1.587+						rno += 4;
   1.588+						p->mem[i].bar |= (uintptr)pcicfgr32(p, rno) << 32;
   1.589+						i++;
   1.590+					}
   1.591+					rno += 4;
   1.592+				}
   1.593+				break;
   1.594+
   1.595+			case 0x05:		/* memory controller */
   1.596+			case 0x06:		/* bridge device */
   1.597+			default:
   1.598+				break;
   1.599+			}
   1.600+
   1.601+			p->parent = parent;
   1.602+			if(head != nil)
   1.603+				tail->link = p;
   1.604+			else
   1.605+				head = p;
   1.606+			tail = p;
   1.607+		}
   1.608+	}
   1.609+
   1.610+	*list = head;
   1.611+	for(p = head; p != nil; p = p->link){
   1.612+		/*
   1.613+		 * Find PCI-PCI bridges and recursively descend the tree.
   1.614+		 */
   1.615+		if(p->ccrb != 0x06 || p->ccru != 0x04)
   1.616+			continue;
   1.617+
   1.618+		/*
   1.619+		 * If the secondary or subordinate bus number is not
   1.620+		 * initialised try to do what the PCI BIOS should have
   1.621+		 * done and fill in the numbers as the tree is descended.
   1.622+		 * On the way down the subordinate bus number is set to
   1.623+		 * the maximum as it's not known how many buses are behind
   1.624+		 * this one; the final value is set on the way back up.
   1.625+		 */
   1.626+		sbn = pcicfgr8(p, PciSBN);
   1.627+		ubn = pcicfgr8(p, PciUBN);
   1.628+
   1.629+		if(sbn == 0 || ubn == 0) {
   1.630+			sbn = maxubn+1;
   1.631+			/*
   1.632+			 * Make sure memory, I/O and master enables are
   1.633+			 * off, set the primary, secondary and subordinate
   1.634+			 * bus numbers and clear the secondary status before
   1.635+			 * attempting to scan the secondary bus.
   1.636+			 *
   1.637+			 * Initialisation of the bridge should be done here.
   1.638+			 */
   1.639+			pcicfgw32(p, PciPCR, 0xFFFF0000);
   1.640+			l = (MaxUBN<<16)|(sbn<<8)|bno;
   1.641+			pcicfgw32(p, PciPBN, l);
   1.642+			pcicfgw16(p, PciSPSR, 0xFFFF);
   1.643+			maxubn = pcilscan(sbn, &p->bridge, p);
   1.644+			l = (maxubn<<16)|(sbn<<8)|bno;
   1.645+
   1.646+			pcicfgw32(p, PciPBN, l);
   1.647+		}
   1.648+		else {
   1.649+			if(ubn > maxubn)
   1.650+				maxubn = ubn;
   1.651+			pcilscan(sbn, &p->bridge, p);
   1.652+		}
   1.653+	}
   1.654+
   1.655+	return maxubn;
   1.656+}
   1.657+
   1.658+static void
   1.659+pcicfginit(void)
   1.660+{
   1.661+	uintptr mema, ioa;
   1.662+
   1.663+	fmtinstall('T', tbdffmt);
   1.664+
   1.665+	pcilscan(0, &pciroot, nil);
   1.666+
   1.667+	/*
   1.668+	 * Work out how big the top bus is
   1.669+	 */
   1.670+	ioa = 0;
   1.671+	mema = 0;
   1.672+	pcibusmap(pciroot, &mema, &ioa, 0);
   1.673+
   1.674+	/*
   1.675+	 * Align the windows and map it
   1.676+	 */
   1.677+	ioa = 0;
   1.678+	mema = soc.pciwin;
   1.679+	pcibusmap(pciroot, &mema, &ioa, 1);
   1.680+}
   1.681+
   1.682+static void
   1.683+pcilhinv(Pcidev* p)
   1.684+{
   1.685+	int i;
   1.686+	Pcidev *t;
   1.687+
   1.688+	if(p == nil) {
   1.689+		p = pciroot;
   1.690+		print("bus dev type vid  did intl memory\n");
   1.691+	}
   1.692+	for(t = p; t != nil; t = t->link) {
   1.693+		print("%d  %2d/%d %.2ux %.2ux %.2ux %.4ux %.4ux %3d  ",
   1.694+			BUSBNO(t->tbdf), BUSDNO(t->tbdf), BUSFNO(t->tbdf),
   1.695+			t->ccrb, t->ccru, t->ccrp, t->vid, t->did, t->intl);
   1.696+
   1.697+		for(i = 0; i < nelem(p->mem); i++) {
   1.698+			if(t->mem[i].size == 0)
   1.699+				continue;
   1.700+			print("%d:%llux %d ", i,
   1.701+				(uvlong)t->mem[i].bar, t->mem[i].size);
   1.702+		}
   1.703+		if(t->bridge)
   1.704+			print("->%d", BUSBNO(t->bridge->tbdf));
   1.705+		print("\n");
   1.706+	}
   1.707+	while(p != nil) {
   1.708+		if(p->bridge != nil)
   1.709+			pcilhinv(p->bridge);
   1.710+		p = p->link;
   1.711+	}
   1.712+}
   1.713+
   1.714+static void
   1.715+pcihinv(Pcidev* p)
   1.716+{
   1.717+	pcilhinv(p);
   1.718+}
   1.719+
   1.720+void
   1.721+pcisetioe(Pcidev* p)
   1.722+{
   1.723+	p->pcr |= IOen;
   1.724+	pcicfgw16(p, PciPCR, p->pcr);
   1.725+}
   1.726+
   1.727+void
   1.728+pciclrioe(Pcidev* p)
   1.729+{
   1.730+	p->pcr &= ~IOen;
   1.731+	pcicfgw16(p, PciPCR, p->pcr);
   1.732+}
   1.733+
   1.734+void
   1.735+pcisetbme(Pcidev* p)
   1.736+{
   1.737+	p->pcr |= MASen;
   1.738+	pcicfgw16(p, PciPCR, p->pcr);
   1.739+}
   1.740+
   1.741+void
   1.742+pciclrbme(Pcidev* p)
   1.743+{
   1.744+	p->pcr &= ~MASen;
   1.745+	pcicfgw16(p, PciPCR, p->pcr);
   1.746+}
   1.747+
   1.748+void
   1.749+pcisetmwi(Pcidev* p)
   1.750+{
   1.751+	p->pcr |= MemWrInv;
   1.752+	pcicfgw16(p, PciPCR, p->pcr);
   1.753+}
   1.754+
   1.755+void
   1.756+pciclrmwi(Pcidev* p)
   1.757+{
   1.758+	p->pcr &= ~MemWrInv;
   1.759+	pcicfgw16(p, PciPCR, p->pcr);
   1.760+}
   1.761+
   1.762+static int
   1.763+enumcaps(Pcidev *p, int (*fmatch)(Pcidev*, int, int, int), int arg)
   1.764+{
   1.765+	int i, r, cap, off;
   1.766+
   1.767+	/* status register bit 4 has capabilities */
   1.768+	if((pcicfgr16(p, PciPSR) & 1<<4) == 0)
   1.769+		return -1;      
   1.770+	switch(pcicfgr8(p, PciHDT) & 0x7F){
   1.771+	default:
   1.772+		return -1;
   1.773+	case 0:                         /* etc */
   1.774+	case 1:                         /* pci to pci bridge */
   1.775+		off = 0x34;
   1.776+		break;
   1.777+	case 2:                         /* cardbus bridge */
   1.778+		off = 0x14;
   1.779+		break;
   1.780+	}
   1.781+	for(i = 48; i--;){
   1.782+		off = pcicfgr8(p, off);
   1.783+		if(off < 0x40 || (off & 3))
   1.784+			break;
   1.785+		off &= ~3;
   1.786+		cap = pcicfgr8(p, off);
   1.787+		if(cap == 0xff)
   1.788+			break;
   1.789+		r = (*fmatch)(p, cap, off, arg);
   1.790+		if(r < 0)
   1.791+			break;
   1.792+		if(r == 0)
   1.793+			return off;
   1.794+		off++;
   1.795+	}
   1.796+	return -1;
   1.797+}
   1.798+
   1.799+static int
   1.800+matchcap(Pcidev *, int cap, int, int arg)
   1.801+{
   1.802+	return cap != arg;
   1.803+}
   1.804+
   1.805+static int
   1.806+matchhtcap(Pcidev *p, int cap, int off, int arg)
   1.807+{
   1.808+	int mask;
   1.809+
   1.810+	if(cap != PciCapHTC)
   1.811+		return 1;
   1.812+	if(arg == 0x00 || arg == 0x20)
   1.813+		mask = 0xE0;
   1.814+	else
   1.815+		mask = 0xF8;
   1.816+	cap = pcicfgr8(p, off+3);
   1.817+	return (cap & mask) != arg;
   1.818+}
   1.819+
   1.820+int
   1.821+pcicap(Pcidev *p, int cap)
   1.822+{
   1.823+	return enumcaps(p, matchcap, cap);
   1.824+}
   1.825+
   1.826+int
   1.827+pcinextcap(Pcidev *pci, int offset)
   1.828+{
   1.829+	if(offset == 0) {
   1.830+		if((pcicfgr16(pci, PciPSR) & (1<<4)) == 0)
   1.831+			return 0; /* no capabilities */
   1.832+		offset = PciCAP-1;
   1.833+	}
   1.834+	return pcicfgr8(pci, offset+1) & ~3;
   1.835+}
   1.836+
   1.837+int
   1.838+pcihtcap(Pcidev *p, int cap)
   1.839+{
   1.840+	return enumcaps(p, matchhtcap, cap);
   1.841+}
   1.842+
   1.843+static int
   1.844+pcigetpmrb(Pcidev* p)
   1.845+{
   1.846+        if(p->pmrb != 0)
   1.847+                return p->pmrb;
   1.848+        return p->pmrb = pcicap(p, PciCapPMG);
   1.849+}
   1.850+
   1.851+int
   1.852+pcigetpms(Pcidev* p)
   1.853+{
   1.854+	int pmcsr, ptr;
   1.855+
   1.856+	if((ptr = pcigetpmrb(p)) == -1)
   1.857+		return -1;
   1.858+
   1.859+	/*
   1.860+	 * Power Management Register Block:
   1.861+	 *  offset 0:	Capability ID
   1.862+	 *	   1:	next item pointer
   1.863+	 *	   2:	capabilities
   1.864+	 *	   4:	control/status
   1.865+	 *	   6:	bridge support extensions
   1.866+	 *	   7:	data
   1.867+	 */
   1.868+	pmcsr = pcicfgr16(p, ptr+4);
   1.869+
   1.870+	return pmcsr & 0x0003;
   1.871+}
   1.872+
   1.873+int
   1.874+pcisetpms(Pcidev* p, int state)
   1.875+{
   1.876+	int ostate, pmc, pmcsr, ptr;
   1.877+
   1.878+	if((ptr = pcigetpmrb(p)) == -1)
   1.879+		return -1;
   1.880+
   1.881+	pmc = pcicfgr16(p, ptr+2);
   1.882+	pmcsr = pcicfgr16(p, ptr+4);
   1.883+	ostate = pmcsr & 0x0003;
   1.884+	pmcsr &= ~0x0003;
   1.885+
   1.886+	switch(state){
   1.887+	default:
   1.888+		return -1;
   1.889+	case 0:
   1.890+		break;
   1.891+	case 1:
   1.892+		if(!(pmc & 0x0200))
   1.893+			return -1;
   1.894+		break;
   1.895+	case 2:
   1.896+		if(!(pmc & 0x0400))
   1.897+			return -1;
   1.898+		break;
   1.899+	case 3:
   1.900+		break;
   1.901+	}
   1.902+	pmcsr |= state;
   1.903+	pcicfgw16(p, ptr+4, pmcsr);
   1.904+
   1.905+	return ostate;
   1.906+}
   1.907+
   1.908+void
   1.909+pcienable(Pcidev *p)
   1.910+{
   1.911+	uint pcr;
   1.912+	int i;
   1.913+
   1.914+	if(p == nil)
   1.915+		return;
   1.916+
   1.917+	pcienable(p->parent);
   1.918+
   1.919+	switch(pcisetpms(p, 0)){
   1.920+	case 1:
   1.921+		print("pcienable %T: wakeup from D1\n", p->tbdf);
   1.922+		break;
   1.923+	case 2:
   1.924+		print("pcienable %T: wakeup from D2\n", p->tbdf);
   1.925+		if(p->bridge != nil)
   1.926+			delay(100);	/* B2: minimum delay 50ms */
   1.927+		else
   1.928+			delay(1);	/* D2: minimum delay 200┬Ás */
   1.929+		break;
   1.930+	case 3:
   1.931+		print("pcienable %T: wakeup from D3\n", p->tbdf);
   1.932+		delay(100);		/* D3: minimum delay 50ms */
   1.933+
   1.934+		/* restore registers */
   1.935+		for(i = 0; i < 6; i++)
   1.936+			pcicfgw32(p, PciBAR0+i*4, p->mem[i].bar);
   1.937+		pcicfgw8(p, PciINTL, p->intl);
   1.938+		pcicfgw8(p, PciLTR, p->ltr);
   1.939+		pcicfgw8(p, PciCLS, p->cls);
   1.940+		pcicfgw16(p, PciPCR, p->pcr);
   1.941+		break;
   1.942+	}
   1.943+
   1.944+	if(p->bridge != nil)
   1.945+		pcr = IOen|MEMen|MASen;
   1.946+	else {
   1.947+		pcr = 0;
   1.948+		for(i = 0; i < 6; i++){
   1.949+			if(p->mem[i].size == 0)
   1.950+				continue;
   1.951+			if(p->mem[i].bar & 1)
   1.952+				pcr |= IOen;
   1.953+			else
   1.954+				pcr |= MEMen;
   1.955+		}
   1.956+	}
   1.957+
   1.958+	if((p->pcr & pcr) != pcr){
   1.959+		print("pcienable %T: pcr %ux->%ux\n", p->tbdf, p->pcr, p->pcr|pcr);
   1.960+		p->pcr |= pcr;
   1.961+		pcicfgrw32(p->tbdf, PciPCR, 0xFFFF0000|p->pcr, 0);
   1.962+	}
   1.963+}
   1.964+
   1.965+void
   1.966+pcidisable(Pcidev *p)
   1.967+{
   1.968+	if(p == nil)
   1.969+		return;
   1.970+	pciclrbme(p);
   1.971+}
   1.972+
   1.973+enum {
   1.974+	MSICtrl = 0x02, /* message control register (16 bit) */
   1.975+	MSIAddr = 0x04, /* message address register (64 bit) */
   1.976+	MSIData32 = 0x08, /* message data register for 32 bit MSI (16 bit) */
   1.977+	MSIData64 = 0x0C, /* message data register for 64 bit MSI (16 bit) */
   1.978+};
   1.979+
   1.980+typedef struct Pciisr Pciisr;
   1.981+struct Pciisr {
   1.982+	void	(*f)(Ureg*, void*);
   1.983+	void	*a;
   1.984+	Pcidev	*p;
   1.985+};
   1.986+
   1.987+static Pciisr pciisr[32];
   1.988+static Lock pciisrlk;
   1.989+
   1.990+void
   1.991+pciintrenable(int tbdf, void (*f)(Ureg*, void*), void *a)
   1.992+{
   1.993+	int cap, ok64;
   1.994+	u32int dat;
   1.995+	u64int adr;
   1.996+	Pcidev *p;
   1.997+	Pciisr *isr;
   1.998+
   1.999+	if((p = pcimatchtbdf(tbdf)) == nil){
  1.1000+		print("pciintrenable: %T: unknown device\n", tbdf);
  1.1001+		return;
  1.1002+	}
  1.1003+	if((cap = pcicap(p, PciCapMSI)) < 0){
  1.1004+		print("pciintrenable: %T: no MSI cap\n", tbdf);
  1.1005+		return;
  1.1006+	}
  1.1007+
  1.1008+	lock(&pciisrlk);
  1.1009+	for(isr = pciisr; isr < &pciisr[nelem(pciisr)]; isr++){
  1.1010+		if(isr->p == p){
  1.1011+			isr->p = nil;
  1.1012+			regs[MSI_INTR2_BASE + INTR_MASK_SET] = 1 << (isr-pciisr);
  1.1013+			break;
  1.1014+		}
  1.1015+	}
  1.1016+	for(isr = pciisr; isr < &pciisr[nelem(pciisr)]; isr++){
  1.1017+		if(isr->p == nil){
  1.1018+			isr->p = p;
  1.1019+			isr->a = a;
  1.1020+			isr->f = f;
  1.1021+			regs[MSI_INTR2_BASE + INTR_CLR] = 1 << (isr-pciisr);
  1.1022+			regs[MSI_INTR2_BASE + INTR_MASK_CLR] = 1 << (isr-pciisr);
  1.1023+			break;
  1.1024+		}
  1.1025+	}
  1.1026+	unlock(&pciisrlk);
  1.1027+
  1.1028+	if(isr >= &pciisr[nelem(pciisr)]){
  1.1029+		print("pciintrenable: %T: out of isr slots\n", tbdf);
  1.1030+		return;
  1.1031+	}
  1.1032+
  1.1033+	adr = MSI_TARGET_ADDR;
  1.1034+	ok64 = (pcicfgr16(p, cap + MSICtrl) & (1<<7)) != 0;
  1.1035+	pcicfgw32(p, cap + MSIAddr, adr);
  1.1036+	if(ok64) pcicfgw32(p, cap + MSIAddr + 4, adr>>32);
  1.1037+	dat = regs[MISC_MSI_DATA_CONFIG];
  1.1038+	dat = ((dat >> 16) & (dat & 0xFFFF)) | (isr-pciisr);
  1.1039+	pcicfgw16(p, cap + (ok64 ? MSIData64 : MSIData32), dat);
  1.1040+	pcicfgw16(p, cap + MSICtrl, 1);
  1.1041+}
  1.1042+
  1.1043+void
  1.1044+pciintrdisable(int tbdf, void (*f)(Ureg*, void*), void *a)
  1.1045+{
  1.1046+	Pciisr *isr;
  1.1047+
  1.1048+	lock(&pciisrlk);
  1.1049+	for(isr = pciisr; isr < &pciisr[nelem(pciisr)]; isr++){
  1.1050+		if(isr->p != nil && isr->p->tbdf == tbdf && isr->f == f && isr->a == a){
  1.1051+			regs[MSI_INTR2_BASE + INTR_MASK_SET] = 1 << (isr-pciisr);
  1.1052+			isr->p = nil;
  1.1053+			isr->f = nil;
  1.1054+			isr->a = nil;
  1.1055+			break;
  1.1056+		}
  1.1057+	}
  1.1058+	unlock(&pciisrlk);
  1.1059+}
  1.1060+
  1.1061+static void
  1.1062+pciinterrupt(Ureg *ureg, void*)
  1.1063+{
  1.1064+	Pciisr *isr;
  1.1065+	u32int sts;
  1.1066+
  1.1067+	sts = regs[MSI_INTR2_BASE + INTR_STATUS];
  1.1068+	if(sts == 0)
  1.1069+		return;
  1.1070+	regs[MSI_INTR2_BASE + INTR_CLR] = sts;
  1.1071+	for(isr = pciisr; sts != 0 && isr < &pciisr[nelem(pciisr)]; isr++, sts>>=1){
  1.1072+		if((sts & 1) != 0 && isr->f != nil)
  1.1073+			(*isr->f)(ureg, isr->a);
  1.1074+	}
  1.1075+	regs[MISC_EOI_CTRL] = 1;
  1.1076+}
  1.1077+
  1.1078+void
  1.1079+pcilink(void)
  1.1080+{
  1.1081+	int log2dmasize = 30;	// 1GB
  1.1082+
  1.1083+	regs[RGR1_SW_INIT_1] |= 3;
  1.1084+	delay(200);
  1.1085+	regs[RGR1_SW_INIT_1] &= ~2;
  1.1086+	regs[MISC_PCIE_CTRL] &= ~5;
  1.1087+	delay(200);
  1.1088+
  1.1089+	regs[MISC_HARD_PCIE_HARD_DEBUG] &= ~0x08000000;
  1.1090+	delay(200);
  1.1091+
  1.1092+	regs[MSI_INTR2_BASE + INTR_CLR] = -1;
  1.1093+	regs[MSI_INTR2_BASE + INTR_MASK_SET] = -1;
  1.1094+
  1.1095+	regs[MISC_CPU_2_PCIE_MEM_WIN0_LO] = 0;
  1.1096+	regs[MISC_CPU_2_PCIE_MEM_WIN0_HI] = 0;
  1.1097+	regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT] = 0;
  1.1098+	regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI] = 0;
  1.1099+	regs[MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI] = 0;
  1.1100+
  1.1101+	// SCB_ACCESS_EN, CFG_READ_UR_MODE, MAX_BURST_SIZE_128, SCB0SIZE
  1.1102+	regs[MISC_MISC_CTRL] = 1<<12 | 1<<13 | 0<<20 | (log2dmasize-15)<<27;
  1.1103+
  1.1104+	regs[MISC_RC_BAR2_CONFIG_LO] = (log2dmasize-15);
  1.1105+	regs[MISC_RC_BAR2_CONFIG_HI] = 0;
  1.1106+
  1.1107+	regs[MISC_RC_BAR1_CONFIG_LO] = 0;
  1.1108+	regs[MISC_RC_BAR3_CONFIG_LO] = 0;
  1.1109+
  1.1110+	regs[MISC_MSI_BAR_CONFIG_LO] = MSI_TARGET_ADDR | 1;
  1.1111+	regs[MISC_MSI_BAR_CONFIG_HI] = MSI_TARGET_ADDR>>32;
  1.1112+	regs[MISC_MSI_DATA_CONFIG] = 0xFFF86540;
  1.1113+	intrenable(IRQpci, pciinterrupt, nil, BUSUNKNOWN, "pci");
  1.1114+
  1.1115+	// force to GEN2
  1.1116+	regs[(0xAC + 12)/4] = (regs[(0xAC + 12)/4] & ~15) | 2;	// linkcap
  1.1117+	regs[(0xAC + 48)/4] = (regs[(0xAC + 48)/4] & ~15) | 2;	// linkctl2
  1.1118+
  1.1119+	regs[RGR1_SW_INIT_1] &= ~1;
  1.1120+	delay(500);
  1.1121+
  1.1122+	if((regs[MISC_PCIE_STATUS] & 0x30) != 0x30){
  1.1123+		print("pcireset: phy link is down\n");
  1.1124+		return;
  1.1125+	}
  1.1126+
  1.1127+	regs[RC_CFG_PRIV1_ID_VAL3] = 0x060400;
  1.1128+	regs[RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1] &= ~0xC;
  1.1129+	regs[MISC_HARD_PCIE_HARD_DEBUG] |= 2;
  1.1130+
  1.1131+	pcicfginit();
  1.1132+	pcihinv(nil);
  1.1133+}